Visualization#

add_geometry.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import open3d as o3d
  9import open3d.visualization.gui as gui
 10import open3d.visualization.rendering as rendering
 11import platform
 12import random
 13import threading
 14import time
 15
 16isMacOS = (platform.system() == "Darwin")
 17
 18
 19# This example shows two methods of adding geometry to an existing scene.
 20# 1) add via a UI callback (in this case a menu, but a button would be similar,
 21#    you would call `button.set_on_clicked(self.on_menu_sphere_)` when
 22#    configuring the button. See `on_menu_sphere()`.
 23# 2) add asynchronously by polling from another thread. GUI functions must be
 24#    called from the UI thread, so use Application.post_to_main_thread().
 25#    See `on_menu_random()`.
 26# Running the example will show a simple window with a Debug menu item with the
 27# two different options. The second method will add random spheres for
 28# 20 seconds, during which time you can be interacting with the scene, rotating,
 29# etc.
 30class SpheresApp:
 31    MENU_SPHERE = 1
 32    MENU_RANDOM = 2
 33    MENU_QUIT = 3
 34
 35    def __init__(self):
 36        self._id = 0
 37        self.window = gui.Application.instance.create_window(
 38            "Add Spheres Example", 1024, 768)
 39        self.scene = gui.SceneWidget()
 40        self.scene.scene = rendering.Open3DScene(self.window.renderer)
 41        self.scene.scene.set_background([1, 1, 1, 1])
 42        self.scene.scene.scene.set_sun_light(
 43            [-1, -1, -1],  # direction
 44            [1, 1, 1],  # color
 45            100000)  # intensity
 46        self.scene.scene.scene.enable_sun_light(True)
 47        bbox = o3d.geometry.AxisAlignedBoundingBox([-10, -10, -10],
 48                                                   [10, 10, 10])
 49        self.scene.setup_camera(60, bbox, [0, 0, 0])
 50
 51        self.window.add_child(self.scene)
 52
 53        # The menu is global (because the macOS menu is global), so only create
 54        # it once, no matter how many windows are created
 55        if gui.Application.instance.menubar is None:
 56            if isMacOS:
 57                app_menu = gui.Menu()
 58                app_menu.add_item("Quit", SpheresApp.MENU_QUIT)
 59            debug_menu = gui.Menu()
 60            debug_menu.add_item("Add Sphere", SpheresApp.MENU_SPHERE)
 61            debug_menu.add_item("Add Random Spheres", SpheresApp.MENU_RANDOM)
 62            if not isMacOS:
 63                debug_menu.add_separator()
 64                debug_menu.add_item("Quit", SpheresApp.MENU_QUIT)
 65
 66            menu = gui.Menu()
 67            if isMacOS:
 68                # macOS will name the first menu item for the running application
 69                # (in our case, probably "Python"), regardless of what we call
 70                # it. This is the application menu, and it is where the
 71                # About..., Preferences..., and Quit menu items typically go.
 72                menu.add_menu("Example", app_menu)
 73                menu.add_menu("Debug", debug_menu)
 74            else:
 75                menu.add_menu("Debug", debug_menu)
 76            gui.Application.instance.menubar = menu
 77
 78        # The menubar is global, but we need to connect the menu items to the
 79        # window, so that the window can call the appropriate function when the
 80        # menu item is activated.
 81        self.window.set_on_menu_item_activated(SpheresApp.MENU_SPHERE,
 82                                               self._on_menu_sphere)
 83        self.window.set_on_menu_item_activated(SpheresApp.MENU_RANDOM,
 84                                               self._on_menu_random)
 85        self.window.set_on_menu_item_activated(SpheresApp.MENU_QUIT,
 86                                               self._on_menu_quit)
 87
 88    def add_sphere(self):
 89        self._id += 1
 90        mat = rendering.MaterialRecord()
 91        mat.base_color = [
 92            random.random(),
 93            random.random(),
 94            random.random(), 1.0
 95        ]
 96        mat.shader = "defaultLit"
 97        sphere = o3d.geometry.TriangleMesh.create_sphere(0.5)
 98        sphere.compute_vertex_normals()
 99        sphere.translate([
100            10.0 * random.uniform(-1.0, 1.0), 10.0 * random.uniform(-1.0, 1.0),
101            10.0 * random.uniform(-1.0, 1.0)
102        ])
103        self.scene.scene.add_geometry("sphere" + str(self._id), sphere, mat)
104
105    def _on_menu_sphere(self):
106        # GUI callbacks happen on the main thread, so we can do everything
107        # normally here.
108        self.add_sphere()
109
110    def _on_menu_random(self):
111        # This adds spheres asynchronously. This pattern is useful if you have
112        # data coming in from another source than user interaction.
113        def thread_main():
114            for _ in range(0, 20):
115                # We can only modify GUI objects on the main thread, so we
116                # need to post the function to call to the main thread.
117                gui.Application.instance.post_to_main_thread(
118                    self.window, self.add_sphere)
119                time.sleep(1)
120
121        threading.Thread(target=thread_main).start()
122
123    def _on_menu_quit(self):
124        gui.Application.instance.quit()
125
126
127def main():
128    gui.Application.instance.initialize()
129    SpheresApp()
130    gui.Application.instance.run()
131
132
133if __name__ == "__main__":
134    main()

all_widgets.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import open3d.visualization.gui as gui
  9import os.path
 10
 11basedir = os.path.dirname(os.path.realpath(__file__))
 12
 13
 14class ExampleWindow:
 15    MENU_CHECKABLE = 1
 16    MENU_DISABLED = 2
 17    MENU_QUIT = 3
 18
 19    def __init__(self):
 20        self.window = gui.Application.instance.create_window("Test", 400, 768)
 21        # self.window = gui.Application.instance.create_window("Test", 400, 768,
 22        #                                                        x=50, y=100)
 23        w = self.window  # for more concise code
 24
 25        # Rather than specifying sizes in pixels, which may vary in size based
 26        # on the monitor, especially on macOS which has 220 dpi monitors, use
 27        # the em-size. This way sizings will be proportional to the font size,
 28        # which will create a more visually consistent size across platforms.
 29        em = w.theme.font_size
 30
 31        # Widgets are laid out in layouts: gui.Horiz, gui.Vert,
 32        # gui.CollapsableVert, and gui.VGrid. By nesting the layouts we can
 33        # achieve complex designs. Usually we use a vertical layout as the
 34        # topmost widget, since widgets tend to be organized from top to bottom.
 35        # Within that, we usually have a series of horizontal layouts for each
 36        # row.
 37        layout = gui.Vert(0, gui.Margins(0.5 * em, 0.5 * em, 0.5 * em,
 38                                         0.5 * em))
 39
 40        # Create the menu. The menu is global (because the macOS menu is global),
 41        # so only create it once.
 42        if gui.Application.instance.menubar is None:
 43            menubar = gui.Menu()
 44            test_menu = gui.Menu()
 45            test_menu.add_item("An option", ExampleWindow.MENU_CHECKABLE)
 46            test_menu.set_checked(ExampleWindow.MENU_CHECKABLE, True)
 47            test_menu.add_item("Unavailable feature",
 48                               ExampleWindow.MENU_DISABLED)
 49            test_menu.set_enabled(ExampleWindow.MENU_DISABLED, False)
 50            test_menu.add_separator()
 51            test_menu.add_item("Quit", ExampleWindow.MENU_QUIT)
 52            # On macOS the first menu item is the application menu item and will
 53            # always be the name of the application (probably "Python"),
 54            # regardless of what you pass in here. The application menu is
 55            # typically where About..., Preferences..., and Quit go.
 56            menubar.add_menu("Test", test_menu)
 57            gui.Application.instance.menubar = menubar
 58
 59        # Each window needs to know what to do with the menu items, so we need
 60        # to tell the window how to handle menu items.
 61        w.set_on_menu_item_activated(ExampleWindow.MENU_CHECKABLE,
 62                                     self._on_menu_checkable)
 63        w.set_on_menu_item_activated(ExampleWindow.MENU_QUIT,
 64                                     self._on_menu_quit)
 65
 66        # Create a file-chooser widget. One part will be a text edit widget for
 67        # the filename and clicking on the button will let the user choose using
 68        # the file dialog.
 69        self._fileedit = gui.TextEdit()
 70        filedlgbutton = gui.Button("...")
 71        filedlgbutton.horizontal_padding_em = 0.5
 72        filedlgbutton.vertical_padding_em = 0
 73        filedlgbutton.set_on_clicked(self._on_filedlg_button)
 74
 75        # (Create the horizontal widget for the row. This will make sure the
 76        # text editor takes up as much space as it can.)
 77        fileedit_layout = gui.Horiz()
 78        fileedit_layout.add_child(gui.Label("Model file"))
 79        fileedit_layout.add_child(self._fileedit)
 80        fileedit_layout.add_fixed(0.25 * em)
 81        fileedit_layout.add_child(filedlgbutton)
 82        # add to the top-level (vertical) layout
 83        layout.add_child(fileedit_layout)
 84
 85        # Create a collapsible vertical widget, which takes up enough vertical
 86        # space for all its children when open, but only enough for text when
 87        # closed. This is useful for property pages, so the user can hide sets
 88        # of properties they rarely use. All layouts take a spacing parameter,
 89        # which is the spacinging between items in the widget, and a margins
 90        # parameter, which specifies the spacing of the left, top, right,
 91        # bottom margins. (This acts like the 'padding' property in CSS.)
 92        collapse = gui.CollapsableVert("Widgets", 0.33 * em,
 93                                       gui.Margins(em, 0, 0, 0))
 94        self._label = gui.Label("Lorem ipsum dolor")
 95        self._label.text_color = gui.Color(1.0, 0.5, 0.0)
 96        collapse.add_child(self._label)
 97
 98        # Create a checkbox. Checking or unchecking would usually be used to set
 99        # a binary property, but in this case it will show a simple message box,
100        # which illustrates how to create simple dialogs.
101        cb = gui.Checkbox("Enable some really cool effect")
102        cb.set_on_checked(self._on_cb)  # set the callback function
103        collapse.add_child(cb)
104
105        # Create a color editor. We will change the color of the orange label
106        # above when the color changes.
107        color = gui.ColorEdit()
108        color.color_value = self._label.text_color
109        color.set_on_value_changed(self._on_color)
110        collapse.add_child(color)
111
112        # This is a combobox, nothing fancy here, just set a simple function to
113        # handle the user selecting an item.
114        combo = gui.Combobox()
115        combo.add_item("Show point labels")
116        combo.add_item("Show point velocity")
117        combo.add_item("Show bounding boxes")
118        combo.set_on_selection_changed(self._on_combo)
119        collapse.add_child(combo)
120
121        # This is a toggle switch, which is similar to a checkbox. To my way of
122        # thinking the difference is subtle: a checkbox toggles properties
123        # (for example, purely visual changes like enabling lighting) while a
124        # toggle switch is better for changing the behavior of the app (for
125        # example, turning on processing from the camera).
126        switch = gui.ToggleSwitch("Continuously update from camera")
127        switch.set_on_clicked(self._on_switch)
128        collapse.add_child(switch)
129
130        self.logo_idx = 0
131        proxy = gui.WidgetProxy()
132
133        def switch_proxy():
134            self.logo_idx += 1
135            if self.logo_idx % 3 == 0:
136                proxy.set_widget(None)
137            elif self.logo_idx % 3 == 1:
138                # Add a simple image
139                logo = gui.ImageWidget(basedir + "/icon-32.png")
140                proxy.set_widget(logo)
141            else:
142                label = gui.Label(
143                    'Open3D: A Modern Library for 3D Data Processing')
144                proxy.set_widget(label)
145            w.set_needs_layout()
146
147        logo_btn = gui.Button('Switch Logo By WidgetProxy')
148        logo_btn.vertical_padding_em = 0
149        logo_btn.background_color = gui.Color(r=0, b=0.5, g=0)
150        logo_btn.set_on_clicked(switch_proxy)
151        collapse.add_child(logo_btn)
152        collapse.add_child(proxy)
153
154        # Widget stack demo
155        self._widget_idx = 0
156        hz = gui.Horiz(spacing=5)
157        push_widget_btn = gui.Button('Push widget')
158        push_widget_btn.vertical_padding_em = 0
159        pop_widget_btn = gui.Button('Pop widget')
160        pop_widget_btn.vertical_padding_em = 0
161        stack = gui.WidgetStack()
162        stack.set_on_top(lambda w: print(f'New widget is: {w.text}'))
163        hz.add_child(gui.Label('WidgetStack '))
164        hz.add_child(push_widget_btn)
165        hz.add_child(pop_widget_btn)
166        hz.add_child(stack)
167        collapse.add_child(hz)
168
169        def push_widget():
170            self._widget_idx += 1
171            stack.push_widget(gui.Label(f'Widget {self._widget_idx}'))
172
173        push_widget_btn.set_on_clicked(push_widget)
174        pop_widget_btn.set_on_clicked(stack.pop_widget)
175
176        # Add a list of items
177        lv = gui.ListView()
178        lv.set_items(["Ground", "Trees", "Buildings", "Cars", "People", "Cats"])
179        lv.selected_index = lv.selected_index + 2  # initially is -1, so now 1
180        lv.set_max_visible_items(4)
181        lv.set_on_selection_changed(self._on_list)
182        collapse.add_child(lv)
183
184        # Add a tree view
185        tree = gui.TreeView()
186        tree.add_text_item(tree.get_root_item(), "Camera")
187        geo_id = tree.add_text_item(tree.get_root_item(), "Geometries")
188        mesh_id = tree.add_text_item(geo_id, "Mesh")
189        tree.add_text_item(mesh_id, "Triangles")
190        tree.add_text_item(mesh_id, "Albedo texture")
191        tree.add_text_item(mesh_id, "Normal map")
192        points_id = tree.add_text_item(geo_id, "Points")
193        tree.can_select_items_with_children = True
194        tree.set_on_selection_changed(self._on_tree)
195        # does not call on_selection_changed: user did not change selection
196        tree.selected_item = points_id
197        collapse.add_child(tree)
198
199        # Add two number editors, one for integers and one for floating point
200        # Number editor can clamp numbers to a range, although this is more
201        # useful for integers than for floating point.
202        intedit = gui.NumberEdit(gui.NumberEdit.INT)
203        intedit.int_value = 0
204        intedit.set_limits(1, 19)  # value coerced to 1
205        intedit.int_value = intedit.int_value + 2  # value should be 3
206        doubleedit = gui.NumberEdit(gui.NumberEdit.DOUBLE)
207        numlayout = gui.Horiz()
208        numlayout.add_child(gui.Label("int"))
209        numlayout.add_child(intedit)
210        numlayout.add_fixed(em)  # manual spacing (could set it in Horiz() ctor)
211        numlayout.add_child(gui.Label("double"))
212        numlayout.add_child(doubleedit)
213        collapse.add_child(numlayout)
214
215        # Create a progress bar. It ranges from 0.0 to 1.0.
216        self._progress = gui.ProgressBar()
217        self._progress.value = 0.25  # 25% complete
218        self._progress.value = self._progress.value + 0.08  # 0.25 + 0.08 = 33%
219        prog_layout = gui.Horiz(em)
220        prog_layout.add_child(gui.Label("Progress..."))
221        prog_layout.add_child(self._progress)
222        collapse.add_child(prog_layout)
223
224        # Create a slider. It acts very similar to NumberEdit except that the
225        # user moves a slider and cannot type the number.
226        slider = gui.Slider(gui.Slider.INT)
227        slider.set_limits(5, 13)
228        slider.set_on_value_changed(self._on_slider)
229        collapse.add_child(slider)
230
231        # Create a text editor. The placeholder text (if not empty) will be
232        # displayed when there is no text, as concise help, or visible tooltip.
233        tedit = gui.TextEdit()
234        tedit.placeholder_text = "Edit me some text here"
235
236        # on_text_changed fires whenever the user changes the text (but not if
237        # the text_value property is assigned to).
238        tedit.set_on_text_changed(self._on_text_changed)
239
240        # on_value_changed fires whenever the user signals that they are finished
241        # editing the text, either by pressing return or by clicking outside of
242        # the text editor, thus losing text focus.
243        tedit.set_on_value_changed(self._on_value_changed)
244        collapse.add_child(tedit)
245
246        # Create a widget for showing/editing a 3D vector
247        vedit = gui.VectorEdit()
248        vedit.vector_value = [1, 2, 3]
249        vedit.set_on_value_changed(self._on_vedit)
250        collapse.add_child(vedit)
251
252        # Create a VGrid layout. This layout specifies the number of columns
253        # (two, in this case), and will place the first child in the first
254        # column, the second in the second, the third in the first, the fourth
255        # in the second, etc.
256        # So:
257        #      2 cols             3 cols                  4 cols
258        #   |  1  |  2  |   |  1  |  2  |  3  |   |  1  |  2  |  3  |  4  |
259        #   |  3  |  4  |   |  4  |  5  |  6  |   |  5  |  6  |  7  |  8  |
260        #   |  5  |  6  |   |  7  |  8  |  9  |   |  9  | 10  | 11  | 12  |
261        #   |    ...    |   |       ...       |   |         ...           |
262        vgrid = gui.VGrid(2)
263        vgrid.add_child(gui.Label("Trees"))
264        vgrid.add_child(gui.Label("12 items"))
265        vgrid.add_child(gui.Label("People"))
266        vgrid.add_child(gui.Label("2 (93% certainty)"))
267        vgrid.add_child(gui.Label("Cars"))
268        vgrid.add_child(gui.Label("5 (87% certainty)"))
269        collapse.add_child(vgrid)
270
271        # Create a tab control. This is really a set of N layouts on top of each
272        # other, but with only one selected.
273        tabs = gui.TabControl()
274        tab1 = gui.Vert()
275        tab1.add_child(gui.Checkbox("Enable option 1"))
276        tab1.add_child(gui.Checkbox("Enable option 2"))
277        tab1.add_child(gui.Checkbox("Enable option 3"))
278        tabs.add_tab("Options", tab1)
279        tab2 = gui.Vert()
280        tab2.add_child(gui.Label("No plugins detected"))
281        tab2.add_stretch()
282        tabs.add_tab("Plugins", tab2)
283        tab3 = gui.RadioButton(gui.RadioButton.VERT)
284        tab3.set_items(["Apple", "Orange"])
285
286        def vt_changed(idx):
287            print(f"current cargo: {tab3.selected_value}")
288
289        tab3.set_on_selection_changed(vt_changed)
290        tabs.add_tab("Cargo", tab3)
291        tab4 = gui.RadioButton(gui.RadioButton.HORIZ)
292        tab4.set_items(["Air plane", "Train", "Bus"])
293
294        def hz_changed(idx):
295            print(f"current traffic plan: {tab4.selected_value}")
296
297        tab4.set_on_selection_changed(hz_changed)
298        tabs.add_tab("Traffic", tab4)
299        collapse.add_child(tabs)
300
301        # Quit button. (Typically this is a menu item)
302        button_layout = gui.Horiz()
303        ok_button = gui.Button("Ok")
304        ok_button.set_on_clicked(self._on_ok)
305        button_layout.add_stretch()
306        button_layout.add_child(ok_button)
307
308        layout.add_child(collapse)
309        layout.add_child(button_layout)
310
311        # We're done, set the window's layout
312        w.add_child(layout)
313
314    def _on_filedlg_button(self):
315        filedlg = gui.FileDialog(gui.FileDialog.OPEN, "Select file",
316                                 self.window.theme)
317        filedlg.add_filter(".obj .ply .stl", "Triangle mesh (.obj, .ply, .stl)")
318        filedlg.add_filter("", "All files")
319        filedlg.set_on_cancel(self._on_filedlg_cancel)
320        filedlg.set_on_done(self._on_filedlg_done)
321        self.window.show_dialog(filedlg)
322
323    def _on_filedlg_cancel(self):
324        self.window.close_dialog()
325
326    def _on_filedlg_done(self, path):
327        self._fileedit.text_value = path
328        self.window.close_dialog()
329
330    def _on_cb(self, is_checked):
331        if is_checked:
332            text = "Sorry, effects are unimplemented"
333        else:
334            text = "Good choice"
335
336        self.show_message_dialog("There might be a problem...", text)
337
338    def _on_switch(self, is_on):
339        if is_on:
340            print("Camera would now be running")
341        else:
342            print("Camera would now be off")
343
344    # This function is essentially the same as window.show_message_box(),
345    # so for something this simple just use that, but it illustrates making a
346    # dialog.
347    def show_message_dialog(self, title, message):
348        # A Dialog is just a widget, so you make its child a layout just like
349        # a Window.
350        dlg = gui.Dialog(title)
351
352        # Add the message text
353        em = self.window.theme.font_size
354        dlg_layout = gui.Vert(em, gui.Margins(em, em, em, em))
355        dlg_layout.add_child(gui.Label(message))
356
357        # Add the Ok button. We need to define a callback function to handle
358        # the click.
359        ok_button = gui.Button("Ok")
360        ok_button.set_on_clicked(self._on_dialog_ok)
361
362        # We want the Ok button to be an the right side, so we need to add
363        # a stretch item to the layout, otherwise the button will be the size
364        # of the entire row. A stretch item takes up as much space as it can,
365        # which forces the button to be its minimum size.
366        button_layout = gui.Horiz()
367        button_layout.add_stretch()
368        button_layout.add_child(ok_button)
369
370        # Add the button layout,
371        dlg_layout.add_child(button_layout)
372        # ... then add the layout as the child of the Dialog
373        dlg.add_child(dlg_layout)
374        # ... and now we can show the dialog
375        self.window.show_dialog(dlg)
376
377    def _on_dialog_ok(self):
378        self.window.close_dialog()
379
380    def _on_color(self, new_color):
381        self._label.text_color = new_color
382
383    def _on_combo(self, new_val, new_idx):
384        print(new_idx, new_val)
385
386    def _on_list(self, new_val, is_dbl_click):
387        print(new_val)
388
389    def _on_tree(self, new_item_id):
390        print(new_item_id)
391
392    def _on_slider(self, new_val):
393        self._progress.value = new_val / 20.0
394
395    def _on_text_changed(self, new_text):
396        print("edit:", new_text)
397
398    def _on_value_changed(self, new_text):
399        print("value:", new_text)
400
401    def _on_vedit(self, new_val):
402        print(new_val)
403
404    def _on_ok(self):
405        gui.Application.instance.quit()
406
407    def _on_menu_checkable(self):
408        gui.Application.instance.menubar.set_checked(
409            ExampleWindow.MENU_CHECKABLE,
410            not gui.Application.instance.menubar.is_checked(
411                ExampleWindow.MENU_CHECKABLE))
412
413    def _on_menu_quit(self):
414        gui.Application.instance.quit()
415
416
417# This class is essentially the same as window.show_message_box(),
418# so for something this simple just use that, but it illustrates making a
419# dialog.
420class MessageBox:
421
422    def __init__(self, title, message):
423        self._window = None
424
425        # A Dialog is just a widget, so you make its child a layout just like
426        # a Window.
427        dlg = gui.Dialog(title)
428
429        # Add the message text
430        em = self.window.theme.font_size
431        dlg_layout = gui.Vert(em, gui.Margins(em, em, em, em))
432        dlg_layout.add_child(gui.Label(message))
433
434        # Add the Ok button. We need to define a callback function to handle
435        # the click.
436        ok_button = gui.Button("Ok")
437        ok_button.set_on_clicked(self._on_ok)
438
439        # We want the Ok button to be an the right side, so we need to add
440        # a stretch item to the layout, otherwise the button will be the size
441        # of the entire row. A stretch item takes up as much space as it can,
442        # which forces the button to be its minimum size.
443        button_layout = gui.Horiz()
444        button_layout.add_stretch()
445        button_layout.add_child(ok_button)
446
447        # Add the button layout,
448        dlg_layout.add_child(button_layout)
449        # ... then add the layout as the child of the Dialog
450        dlg.add_child(dlg_layout)
451
452    def show(self, window):
453        self._window = window
454
455    def _on_ok(self):
456        self._window.close_dialog()
457
458
459def main():
460    # We need to initialize the application, which finds the necessary shaders for
461    # rendering and prepares the cross-platform window abstraction.
462    gui.Application.instance.initialize()
463
464    w = ExampleWindow()
465
466    # Run the event loop. This will not return until the last window is closed.
467    gui.Application.instance.run()
468
469
470if __name__ == "__main__":
471    main()

customized_visualization.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import os
  9import open3d as o3d
 10import numpy as np
 11import matplotlib.pyplot as plt
 12
 13pyexample_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 14test_data_path = os.path.join(os.path.dirname(pyexample_path), 'test_data')
 15
 16
 17def custom_draw_geometry(pcd):
 18    # The following code achieves the same effect as:
 19    # o3d.visualization.draw_geometries([pcd])
 20    vis = o3d.visualization.Visualizer()
 21    vis.create_window()
 22    vis.add_geometry(pcd)
 23    vis.run()
 24    vis.destroy_window()
 25
 26
 27def custom_draw_geometry_with_custom_fov(pcd, fov_step):
 28    vis = o3d.visualization.Visualizer()
 29    vis.create_window()
 30    vis.add_geometry(pcd)
 31    ctr = vis.get_view_control()
 32    print("Field of view (before changing) %.2f" % ctr.get_field_of_view())
 33    ctr.change_field_of_view(step=fov_step)
 34    print("Field of view (after changing) %.2f" % ctr.get_field_of_view())
 35    vis.run()
 36    vis.destroy_window()
 37
 38
 39def custom_draw_geometry_with_rotation(pcd):
 40
 41    def rotate_view(vis):
 42        ctr = vis.get_view_control()
 43        ctr.rotate(10.0, 0.0)
 44        return False
 45
 46    o3d.visualization.draw_geometries_with_animation_callback([pcd],
 47                                                              rotate_view)
 48
 49
 50def custom_draw_geometry_load_option(pcd, render_option_path):
 51    vis = o3d.visualization.Visualizer()
 52    vis.create_window()
 53    vis.add_geometry(pcd)
 54    vis.get_render_option().load_from_json(render_option_path)
 55    vis.run()
 56    vis.destroy_window()
 57
 58
 59def custom_draw_geometry_with_key_callback(pcd, render_option_path):
 60
 61    def change_background_to_black(vis):
 62        opt = vis.get_render_option()
 63        opt.background_color = np.asarray([0, 0, 0])
 64        return False
 65
 66    def load_render_option(vis):
 67        vis.get_render_option().load_from_json(render_option_path)
 68        return False
 69
 70    def capture_depth(vis):
 71        depth = vis.capture_depth_float_buffer()
 72        plt.imshow(np.asarray(depth))
 73        plt.show()
 74        return False
 75
 76    def capture_image(vis):
 77        image = vis.capture_screen_float_buffer()
 78        plt.imshow(np.asarray(image))
 79        plt.show()
 80        return False
 81
 82    key_to_callback = {}
 83    key_to_callback[ord("K")] = change_background_to_black
 84    key_to_callback[ord("R")] = load_render_option
 85    key_to_callback[ord(",")] = capture_depth
 86    key_to_callback[ord(".")] = capture_image
 87    o3d.visualization.draw_geometries_with_key_callbacks([pcd], key_to_callback)
 88
 89
 90def custom_draw_geometry_with_camera_trajectory(pcd, render_option_path,
 91                                                camera_trajectory_path):
 92    custom_draw_geometry_with_camera_trajectory.index = -1
 93    custom_draw_geometry_with_camera_trajectory.trajectory =\
 94        o3d.io.read_pinhole_camera_trajectory(camera_trajectory_path)
 95    custom_draw_geometry_with_camera_trajectory.vis = o3d.visualization.Visualizer(
 96    )
 97    image_path = os.path.join(test_data_path, 'image')
 98    if not os.path.exists(image_path):
 99        os.makedirs(image_path)
100    depth_path = os.path.join(test_data_path, 'depth')
101    if not os.path.exists(depth_path):
102        os.makedirs(depth_path)
103
104    def move_forward(vis):
105        # This function is called within the o3d.visualization.Visualizer::run() loop
106        # The run loop calls the function, then re-render
107        # So the sequence in this function is to:
108        # 1. Capture frame
109        # 2. index++, check ending criteria
110        # 3. Set camera
111        # 4. (Re-render)
112        ctr = vis.get_view_control()
113        glb = custom_draw_geometry_with_camera_trajectory
114        if glb.index >= 0:
115            print("Capture image {:05d}".format(glb.index))
116            depth = vis.capture_depth_float_buffer(False)
117            image = vis.capture_screen_float_buffer(False)
118            plt.imsave(os.path.join(depth_path, '{:05d}.png'.format(glb.index)),
119                       np.asarray(depth),
120                       dpi=1)
121            plt.imsave(os.path.join(image_path, '{:05d}.png'.format(glb.index)),
122                       np.asarray(image),
123                       dpi=1)
124            # vis.capture_depth_image("depth/{:05d}.png".format(glb.index), False)
125            # vis.capture_screen_image("image/{:05d}.png".format(glb.index), False)
126        glb.index = glb.index + 1
127        if glb.index < len(glb.trajectory.parameters):
128            ctr.convert_from_pinhole_camera_parameters(
129                glb.trajectory.parameters[glb.index], allow_arbitrary=True)
130        else:
131            custom_draw_geometry_with_camera_trajectory.vis.\
132                register_animation_callback(None)
133        return False
134
135    vis = custom_draw_geometry_with_camera_trajectory.vis
136    vis.create_window()
137    vis.add_geometry(pcd)
138    vis.get_render_option().load_from_json(render_option_path)
139    vis.register_animation_callback(move_forward)
140    vis.run()
141    vis.destroy_window()
142
143
144if __name__ == "__main__":
145    sample_data = o3d.data.DemoCustomVisualization()
146    pcd_flipped = o3d.io.read_point_cloud(sample_data.point_cloud_path)
147    # Flip it, otherwise the pointcloud will be upside down
148    pcd_flipped.transform([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0],
149                           [0, 0, 0, 1]])
150
151    print("1. Customized visualization to mimic DrawGeometry")
152    custom_draw_geometry(pcd_flipped)
153
154    print("2. Changing field of view")
155    custom_draw_geometry_with_custom_fov(pcd_flipped, 90.0)
156    custom_draw_geometry_with_custom_fov(pcd_flipped, -90.0)
157
158    print("3. Customized visualization with a rotating view")
159    custom_draw_geometry_with_rotation(pcd_flipped)
160
161    print("4. Customized visualization showing normal rendering")
162    custom_draw_geometry_load_option(pcd_flipped,
163                                     sample_data.render_option_path)
164
165    print("5. Customized visualization with key press callbacks")
166    print("   Press 'K' to change background color to black")
167    print("   Press 'R' to load a customized render option, showing normals")
168    print("   Press ',' to capture the depth buffer and show it")
169    print("   Press '.' to capture the screen and show it")
170    custom_draw_geometry_with_key_callback(pcd_flipped,
171                                           sample_data.render_option_path)
172
173    pcd = o3d.io.read_point_cloud(sample_data.point_cloud_path)
174    print("6. Customized visualization playing a camera trajectory")
175    custom_draw_geometry_with_camera_trajectory(
176        pcd, sample_data.render_option_path, sample_data.camera_trajectory_path)

customized_visualization_key_action.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import open3d as o3d
 9
10
11def custom_key_action_without_kb_repeat_delay(pcd):
12    rotating = False
13
14    vis = o3d.visualization.VisualizerWithKeyCallback()
15
16    def key_action_callback(vis, action, mods):
17        nonlocal rotating
18        print(action)
19        if action == 1:  # key down
20            rotating = True
21        elif action == 0:  # key up
22            rotating = False
23        elif action == 2:  # key repeat
24            pass
25        return True
26
27    def animation_callback(vis):
28        nonlocal rotating
29        if rotating:
30            ctr = vis.get_view_control()
31            ctr.rotate(10.0, 0.0)
32
33    # key_action_callback will be triggered when there's a keyboard press, release or repeat event
34    vis.register_key_action_callback(32, key_action_callback)  # space
35
36    # animation_callback is always repeatedly called by the visualizer
37    vis.register_animation_callback(animation_callback)
38
39    vis.create_window()
40    vis.add_geometry(pcd)
41    vis.run()
42
43
44if __name__ == "__main__":
45    ply_data = o3d.data.PLYPointCloud()
46    pcd = o3d.io.read_point_cloud(ply_data.path)
47
48    print(
49        "Customized visualization with smooth key action (without keyboard repeat delay)"
50    )
51    custom_key_action_without_kb_repeat_delay(pcd)

demo_scene.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7"""Demo scene demonstrating models, built-in shapes, and materials"""
  8
  9import math
 10import numpy as np
 11import os
 12import open3d as o3d
 13import open3d.visualization as vis
 14
 15
 16def convert_material_record(mat_record):
 17    mat = vis.Material('defaultLit')
 18    # Convert scalar parameters
 19    mat.vector_properties['base_color'] = mat_record.base_color
 20    mat.scalar_properties['metallic'] = mat_record.base_metallic
 21    mat.scalar_properties['roughness'] = mat_record.base_roughness
 22    mat.scalar_properties['reflectance'] = mat_record.base_reflectance
 23    mat.texture_maps['albedo'] = o3d.t.geometry.Image.from_legacy(
 24        mat_record.albedo_img)
 25    mat.texture_maps['normal'] = o3d.t.geometry.Image.from_legacy(
 26        mat_record.normal_img)
 27    mat.texture_maps['ao_rough_metal'] = o3d.t.geometry.Image.from_legacy(
 28        mat_record.ao_rough_metal_img)
 29    return mat
 30
 31
 32def create_scene():
 33    '''
 34    Creates the geometry and materials for the demo scene and returns a dictionary suitable for draw call
 35    '''
 36    # Create some shapes for our scene
 37    a_cube = o3d.geometry.TriangleMesh.create_box(2,
 38                                                  4,
 39                                                  4,
 40                                                  create_uv_map=True,
 41                                                  map_texture_to_each_face=True)
 42    a_cube.compute_triangle_normals()
 43    a_cube.translate((-5, 0, -2))
 44    a_cube = o3d.t.geometry.TriangleMesh.from_legacy(a_cube)
 45
 46    a_sphere = o3d.geometry.TriangleMesh.create_sphere(2.5,
 47                                                       resolution=40,
 48                                                       create_uv_map=True)
 49    a_sphere.compute_vertex_normals()
 50    rotate_90 = o3d.geometry.get_rotation_matrix_from_xyz((-math.pi / 2, 0, 0))
 51    a_sphere.rotate(rotate_90)
 52    a_sphere.translate((5, 2.4, 0))
 53    a_sphere = o3d.t.geometry.TriangleMesh.from_legacy(a_sphere)
 54
 55    a_cylinder = o3d.geometry.TriangleMesh.create_cylinder(
 56        1.0, 4.0, 30, 4, True)
 57    a_cylinder.compute_triangle_normals()
 58    a_cylinder.rotate(rotate_90)
 59    a_cylinder.translate((10, 2, 0))
 60    a_cylinder = o3d.t.geometry.TriangleMesh.from_legacy(a_cylinder)
 61
 62    a_ico = o3d.geometry.TriangleMesh.create_icosahedron(1.25,
 63                                                         create_uv_map=True)
 64    a_ico.compute_triangle_normals()
 65    a_ico.translate((-10, 2, 0))
 66    a_ico = o3d.t.geometry.TriangleMesh.from_legacy(a_ico)
 67
 68    # Load an OBJ model for our scene
 69    helmet_data = o3d.data.FlightHelmetModel()
 70    helmet = o3d.io.read_triangle_model(helmet_data.path)
 71    helmet_parts = []
 72    for m in helmet.meshes:
 73        # m.mesh.paint_uniform_color((1.0, 0.75, 0.3))
 74        m.mesh.scale(10.0, (0.0, 0.0, 0.0))
 75        helmet_parts.append(m)
 76
 77    # Create a ground plane
 78    ground_plane = o3d.geometry.TriangleMesh.create_box(
 79        50.0, 0.1, 50.0, create_uv_map=True, map_texture_to_each_face=True)
 80    ground_plane.compute_triangle_normals()
 81    rotate_180 = o3d.geometry.get_rotation_matrix_from_xyz((-math.pi, 0, 0))
 82    ground_plane.rotate(rotate_180)
 83    ground_plane.translate((-25.0, -0.1, -25.0))
 84    ground_plane.paint_uniform_color((1, 1, 1))
 85    ground_plane = o3d.t.geometry.TriangleMesh.from_legacy(ground_plane)
 86
 87    # Material to make ground plane more interesting - a rough piece of glass
 88    ground_plane.material = vis.Material("defaultLitSSR")
 89    ground_plane.material.scalar_properties['roughness'] = 0.15
 90    ground_plane.material.scalar_properties['reflectance'] = 0.72
 91    ground_plane.material.scalar_properties['transmission'] = 0.6
 92    ground_plane.material.scalar_properties['thickness'] = 0.3
 93    ground_plane.material.scalar_properties['absorption_distance'] = 0.1
 94    ground_plane.material.vector_properties['absorption_color'] = np.array(
 95        [0.82, 0.98, 0.972, 1.0])
 96    painted_plaster_texture_data = o3d.data.PaintedPlasterTexture()
 97    ground_plane.material.texture_maps['albedo'] = o3d.t.io.read_image(
 98        painted_plaster_texture_data.albedo_texture_path)
 99    ground_plane.material.texture_maps['normal'] = o3d.t.io.read_image(
100        painted_plaster_texture_data.normal_texture_path)
101    ground_plane.material.texture_maps['roughness'] = o3d.t.io.read_image(
102        painted_plaster_texture_data.roughness_texture_path)
103
104    # Load textures and create materials for each of our demo items
105    wood_floor_texture_data = o3d.data.WoodFloorTexture()
106    a_cube.material = vis.Material('defaultLit')
107    a_cube.material.texture_maps['albedo'] = o3d.t.io.read_image(
108        wood_floor_texture_data.albedo_texture_path)
109    a_cube.material.texture_maps['normal'] = o3d.t.io.read_image(
110        wood_floor_texture_data.normal_texture_path)
111    a_cube.material.texture_maps['roughness'] = o3d.t.io.read_image(
112        wood_floor_texture_data.roughness_texture_path)
113
114    tiles_texture_data = o3d.data.TilesTexture()
115    a_sphere.material = vis.Material('defaultLit')
116    a_sphere.material.texture_maps['albedo'] = o3d.t.io.read_image(
117        tiles_texture_data.albedo_texture_path)
118    a_sphere.material.texture_maps['normal'] = o3d.t.io.read_image(
119        tiles_texture_data.normal_texture_path)
120    a_sphere.material.texture_maps['roughness'] = o3d.t.io.read_image(
121        tiles_texture_data.roughness_texture_path)
122
123    terrazzo_texture_data = o3d.data.TerrazzoTexture()
124    a_ico.material = vis.Material('defaultLit')
125    a_ico.material.texture_maps['albedo'] = o3d.t.io.read_image(
126        terrazzo_texture_data.albedo_texture_path)
127    a_ico.material.texture_maps['normal'] = o3d.t.io.read_image(
128        terrazzo_texture_data.normal_texture_path)
129    a_ico.material.texture_maps['roughness'] = o3d.t.io.read_image(
130        terrazzo_texture_data.roughness_texture_path)
131
132    metal_texture_data = o3d.data.MetalTexture()
133    a_cylinder.material = vis.Material('defaultLit')
134    a_cylinder.material.texture_maps['albedo'] = o3d.t.io.read_image(
135        metal_texture_data.albedo_texture_path)
136    a_cylinder.material.texture_maps['normal'] = o3d.t.io.read_image(
137        metal_texture_data.normal_texture_path)
138    a_cylinder.material.texture_maps['roughness'] = o3d.t.io.read_image(
139        metal_texture_data.roughness_texture_path)
140    a_cylinder.material.texture_maps['metallic'] = o3d.t.io.read_image(
141        metal_texture_data.metallic_texture_path)
142
143    geoms = [{
144        "name": "plane",
145        "geometry": ground_plane
146    }, {
147        "name": "cube",
148        "geometry": a_cube
149    }, {
150        "name": "cylinder",
151        "geometry": a_cylinder
152    }, {
153        "name": "ico",
154        "geometry": a_ico
155    }, {
156        "name": "sphere",
157        "geometry": a_sphere
158    }]
159    # Load the helmet
160    for part in helmet_parts:
161        name = part.mesh_name
162        tgeom = o3d.t.geometry.TriangleMesh.from_legacy(part.mesh)
163        tgeom.material = convert_material_record(
164            helmet.materials[part.material_idx])
165        geoms.append({"name": name, "geometry": tgeom})
166    return geoms
167
168
169if __name__ == "__main__":
170    geoms = create_scene()
171    vis.draw(geoms,
172             bg_color=(0.8, 0.9, 0.9, 1.0),
173             show_ui=True,
174             width=1920,
175             height=1080)

draw.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import math
  9import numpy as np
 10import open3d as o3d
 11import open3d.visualization as vis
 12import os
 13import random
 14
 15pyexample_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 16test_data_path = os.path.join(os.path.dirname(pyexample_path), 'test_data')
 17
 18
 19def normalize(v):
 20    a = 1.0 / math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2])
 21    return (a * v[0], a * v[1], a * v[2])
 22
 23
 24def make_point_cloud(npts, center, radius, colorize):
 25    pts = np.random.uniform(-radius, radius, size=[npts, 3]) + center
 26    cloud = o3d.geometry.PointCloud()
 27    cloud.points = o3d.utility.Vector3dVector(pts)
 28    if colorize:
 29        colors = np.random.uniform(0.0, 1.0, size=[npts, 3])
 30        cloud.colors = o3d.utility.Vector3dVector(colors)
 31    return cloud
 32
 33
 34def single_object():
 35    # No colors, no normals, should appear unlit black
 36    cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4)
 37    vis.draw(cube)
 38
 39
 40def multi_objects():
 41    pc_rad = 1.0
 42    pc_nocolor = make_point_cloud(100, (0, -2, 0), pc_rad, False)
 43    pc_color = make_point_cloud(100, (3, -2, 0), pc_rad, True)
 44    r = 0.4
 45    sphere_unlit = o3d.geometry.TriangleMesh.create_sphere(r)
 46    sphere_unlit.translate((0, 1, 0))
 47    sphere_colored_unlit = o3d.geometry.TriangleMesh.create_sphere(r)
 48    sphere_colored_unlit.paint_uniform_color((1.0, 0.0, 0.0))
 49    sphere_colored_unlit.translate((2, 1, 0))
 50    sphere_lit = o3d.geometry.TriangleMesh.create_sphere(r)
 51    sphere_lit.compute_vertex_normals()
 52    sphere_lit.translate((4, 1, 0))
 53    sphere_colored_lit = o3d.geometry.TriangleMesh.create_sphere(r)
 54    sphere_colored_lit.compute_vertex_normals()
 55    sphere_colored_lit.paint_uniform_color((0.0, 1.0, 0.0))
 56    sphere_colored_lit.translate((6, 1, 0))
 57    big_bbox = o3d.geometry.AxisAlignedBoundingBox((-pc_rad, -3, -pc_rad),
 58                                                   (6.0 + r, 1.0 + r, pc_rad))
 59    big_bbox.color = (0.0, 0.0, 0.0)
 60    sphere_bbox = sphere_unlit.get_axis_aligned_bounding_box()
 61    sphere_bbox.color = (1.0, 0.5, 0.0)
 62    lines = o3d.geometry.LineSet.create_from_axis_aligned_bounding_box(
 63        sphere_lit.get_axis_aligned_bounding_box())
 64    lines.paint_uniform_color((0.0, 1.0, 0.0))
 65    lines_colored = o3d.geometry.LineSet.create_from_axis_aligned_bounding_box(
 66        sphere_colored_lit.get_axis_aligned_bounding_box())
 67    lines_colored.paint_uniform_color((0.0, 0.0, 1.0))
 68
 69    vis.draw([
 70        pc_nocolor, pc_color, sphere_unlit, sphere_colored_unlit, sphere_lit,
 71        sphere_colored_lit, big_bbox, sphere_bbox, lines, lines_colored
 72    ])
 73
 74
 75def actions():
 76    SOURCE_NAME = "Source"
 77    RESULT_NAME = "Result (Poisson reconstruction)"
 78    TRUTH_NAME = "Ground truth"
 79
 80    bunny = o3d.data.BunnyMesh()
 81    bunny_mesh = o3d.io.read_triangle_mesh(bunny.path)
 82    bunny_mesh.compute_vertex_normals()
 83
 84    bunny_mesh.paint_uniform_color((1, 0.75, 0))
 85    bunny_mesh.compute_vertex_normals()
 86    cloud = o3d.geometry.PointCloud()
 87    cloud.points = bunny_mesh.vertices
 88    cloud.normals = bunny_mesh.vertex_normals
 89
 90    def make_mesh(o3dvis):
 91        # TODO: call o3dvis.get_geometry instead of using bunny_mesh
 92        mesh, _ = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
 93            cloud)
 94        mesh.paint_uniform_color((1, 1, 1))
 95        mesh.compute_vertex_normals()
 96        o3dvis.add_geometry({"name": RESULT_NAME, "geometry": mesh})
 97        o3dvis.show_geometry(SOURCE_NAME, False)
 98
 99    def toggle_result(o3dvis):
100        truth_vis = o3dvis.get_geometry(TRUTH_NAME).is_visible
101        o3dvis.show_geometry(TRUTH_NAME, not truth_vis)
102        o3dvis.show_geometry(RESULT_NAME, truth_vis)
103
104    vis.draw([{
105        "name": SOURCE_NAME,
106        "geometry": cloud
107    }, {
108        "name": TRUTH_NAME,
109        "geometry": bunny_mesh,
110        "is_visible": False
111    }],
112             actions=[("Create Mesh", make_mesh),
113                      ("Toggle truth/result", toggle_result)])
114
115
116def get_icp_transform(source, target, source_indices, target_indices):
117    corr = np.zeros((len(source_indices), 2))
118    corr[:, 0] = source_indices
119    corr[:, 1] = target_indices
120
121    # Estimate rough transformation using correspondences
122    p2p = o3d.pipelines.registration.TransformationEstimationPointToPoint()
123    trans_init = p2p.compute_transformation(source, target,
124                                            o3d.utility.Vector2iVector(corr))
125
126    # Point-to-point ICP for refinement
127    threshold = 0.03  # 3cm distance threshold
128    reg_p2p = o3d.pipelines.registration.registration_icp(
129        source, target, threshold, trans_init,
130        o3d.pipelines.registration.TransformationEstimationPointToPoint())
131
132    return reg_p2p.transformation
133
134
135def selections():
136    pcd_fragments_data = o3d.data.DemoICPPointClouds()
137    source = o3d.io.read_point_cloud(pcd_fragments_data.paths[0])
138    target = o3d.io.read_point_cloud(pcd_fragments_data.paths[1])
139    source.paint_uniform_color([1, 0.706, 0])
140    target.paint_uniform_color([0, 0.651, 0.929])
141
142    source_name = "Source (yellow)"
143    target_name = "Target (blue)"
144
145    def do_icp_one_set(o3dvis):
146        # sets: [name: [{ "index": int, "order": int, "point": (x, y, z)}, ...],
147        #        ...]
148        sets = o3dvis.get_selection_sets()
149        source_picked = sorted(list(sets[0][source_name]),
150                               key=lambda x: x.order)
151        target_picked = sorted(list(sets[0][target_name]),
152                               key=lambda x: x.order)
153        source_indices = [idx.index for idx in source_picked]
154        target_indices = [idx.index for idx in target_picked]
155
156        t = get_icp_transform(source, target, source_indices, target_indices)
157        source.transform(t)
158
159        # Update the source geometry
160        o3dvis.remove_geometry(source_name)
161        o3dvis.add_geometry({"name": source_name, "geometry": source})
162
163    def do_icp_two_sets(o3dvis):
164        sets = o3dvis.get_selection_sets()
165        source_set = sets[0][source_name]
166        target_set = sets[1][target_name]
167        source_picked = sorted(list(source_set), key=lambda x: x.order)
168        target_picked = sorted(list(target_set), key=lambda x: x.order)
169        source_indices = [idx.index for idx in source_picked]
170        target_indices = [idx.index for idx in target_picked]
171
172        t = get_icp_transform(source, target, source_indices, target_indices)
173        source.transform(t)
174
175        # Update the source geometry
176        o3dvis.remove_geometry(source_name)
177        o3dvis.add_geometry({"name": source_name, "geometry": source})
178
179    vis.draw([{
180        "name": source_name,
181        "geometry": source
182    }, {
183        "name": target_name,
184        "geometry": target
185    }],
186             actions=[("ICP Registration (one set)", do_icp_one_set),
187                      ("ICP Registration (two sets)", do_icp_two_sets)],
188             show_ui=True)
189
190
191def time_animation():
192    orig = make_point_cloud(200, (0, 0, 0), 1.0, True)
193    clouds = [{"name": "t=0", "geometry": orig, "time": 0}]
194    drift_dir = (1.0, 0.0, 0.0)
195    expand = 1.0
196    n = 20
197    for i in range(1, n):
198        amount = float(i) / float(n - 1)
199        cloud = o3d.geometry.PointCloud()
200        pts = np.asarray(orig.points)
201        pts = pts * (1.0 + amount * expand) + [amount * v for v in drift_dir]
202        cloud.points = o3d.utility.Vector3dVector(pts)
203        cloud.colors = orig.colors
204        clouds.append({
205            "name": "points at t=" + str(i),
206            "geometry": cloud,
207            "time": i
208        })
209
210    vis.draw(clouds)
211
212
213def groups():
214    building_mat = vis.rendering.MaterialRecord()
215    building_mat.shader = "defaultLit"
216    building_mat.base_color = (1.0, .90, .75, 1.0)
217    building_mat.base_reflectance = 0.1
218    midrise_mat = vis.rendering.MaterialRecord()
219    midrise_mat.shader = "defaultLit"
220    midrise_mat.base_color = (.475, .450, .425, 1.0)
221    midrise_mat.base_reflectance = 0.1
222    skyscraper_mat = vis.rendering.MaterialRecord()
223    skyscraper_mat.shader = "defaultLit"
224    skyscraper_mat.base_color = (.05, .20, .55, 1.0)
225    skyscraper_mat.base_reflectance = 0.9
226    skyscraper_mat.base_roughness = 0.01
227
228    buildings = []
229    size = 10.0
230    half = size / 2.0
231    min_height = 1.0
232    max_height = 20.0
233    for z in range(0, 10):
234        for x in range(0, 10):
235            max_h = max_height * (1.0 - abs(half - x) / half) * (
236                1.0 - abs(half - z) / half)
237            h = random.uniform(min_height, max(max_h, min_height + 1.0))
238            box = o3d.geometry.TriangleMesh.create_box(0.9, h, 0.9)
239            box.compute_triangle_normals()
240            box.translate((x + 0.05, 0.0, z + 0.05))
241            if h > 0.333 * max_height:
242                mat = skyscraper_mat
243            elif h > 0.1 * max_height:
244                mat = midrise_mat
245            else:
246                mat = building_mat
247            buildings.append({
248                "name": "building_" + str(x) + "_" + str(z),
249                "geometry": box,
250                "material": mat,
251                "group": "buildings"
252            })
253
254    haze = make_point_cloud(5000, (half, 0.333 * max_height, half),
255                            1.414 * half, False)
256    haze.paint_uniform_color((0.8, 0.8, 0.8))
257
258    smog = make_point_cloud(10000, (half, 0.25 * max_height, half), 1.2 * half,
259                            False)
260    smog.paint_uniform_color((0.95, 0.85, 0.75))
261
262    vis.draw(buildings + [{
263        "name": "haze",
264        "geometry": haze,
265        "group": "haze"
266    }, {
267        "name": "smog",
268        "geometry": smog,
269        "group": "smog"
270    }])
271
272
273def remove():
274
275    def make_sphere(name, center, color, group, time):
276        sphere = o3d.geometry.TriangleMesh.create_sphere(0.5)
277        sphere.compute_vertex_normals()
278        sphere.translate(center)
279
280        mat = vis.rendering.Material()
281        mat.shader = "defaultLit"
282        mat.base_color = color
283
284        return {
285            "name": name,
286            "geometry": sphere,
287            "material": mat,
288            "group": group,
289            "time": time
290        }
291
292    red = make_sphere("red", (0, 0, 0), (1.0, 0.0, 0.0, 1.0), "spheres", 0)
293    green = make_sphere("green", (2, 0, 0), (0.0, 1.0, 0.0, 1.0), "spheres", 0)
294    blue = make_sphere("blue", (4, 0, 0), (0.0, 0.0, 1.0, 1.0), "spheres", 0)
295    yellow = make_sphere("yellow", (0, 0, 0), (1.0, 1.0, 0.0, 1.0), "spheres",
296                         1)
297    bbox = {
298        "name": "bbox",
299        "geometry": red["geometry"].get_axis_aligned_bounding_box()
300    }
301
302    def remove_green(visdraw):
303        visdraw.remove_geometry("green")
304
305    def remove_yellow(visdraw):
306        visdraw.remove_geometry("yellow")
307
308    def remove_bbox(visdraw):
309        visdraw.remove_geometry("bbox")
310
311    vis.draw([red, green, blue, yellow, bbox],
312             actions=[("Remove Green", remove_green),
313                      ("Remove Yellow", remove_yellow),
314                      ("Remove Bounds", remove_bbox)])
315
316
317def main():
318    single_object()
319    multi_objects()
320    actions()
321    selections()
322    groups()
323    time_animation()
324
325
326if __name__ == "__main__":
327    main()

draw_webrtc.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import open3d as o3d
 9
10if __name__ == "__main__":
11    o3d.visualization.webrtc_server.enable_webrtc()
12    cube_red = o3d.geometry.TriangleMesh.create_box(1, 2, 4)
13    cube_red.compute_vertex_normals()
14    cube_red.paint_uniform_color((1.0, 0.0, 0.0))
15    o3d.visualization.draw(cube_red)

headless_rendering.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import os
 9import open3d as o3d
10import numpy as np
11import matplotlib.pyplot as plt
12
13
14def custom_draw_geometry_with_camera_trajectory(pcd, camera_trajectory_path,
15                                                render_option_path,
16                                                output_path):
17    custom_draw_geometry_with_camera_trajectory.index = -1
18    custom_draw_geometry_with_camera_trajectory.trajectory =\
19        o3d.io.read_pinhole_camera_trajectory(camera_trajectory_path)
20    custom_draw_geometry_with_camera_trajectory.vis = o3d.visualization.Visualizer(
21    )
22    image_path = os.path.join(output_path, 'image')
23    if not os.path.exists(image_path):
24        os.makedirs(image_path)
25    depth_path = os.path.join(output_path, 'depth')
26    if not os.path.exists(depth_path):
27        os.makedirs(depth_path)
28
29    print("Saving color images in " + image_path)
30    print("Saving depth images in " + depth_path)
31
32    def move_forward(vis):
33        # This function is called within the o3d.visualization.Visualizer::run() loop
34        # The run loop calls the function, then re-render
35        # So the sequence in this function is to:
36        # 1. Capture frame
37        # 2. index++, check ending criteria
38        # 3. Set camera
39        # 4. (Re-render)
40        ctr = vis.get_view_control()
41        glb = custom_draw_geometry_with_camera_trajectory
42        if glb.index >= 0:
43            print("Capture image {:05d}".format(glb.index))
44            # Capture and save image using Open3D.
45            vis.capture_depth_image(
46                os.path.join(depth_path, "{:05d}.png".format(glb.index)), False)
47            vis.capture_screen_image(
48                os.path.join(image_path, "{:05d}.png".format(glb.index)), False)
49
50            # Example to save image using matplotlib.
51            '''
52            depth = vis.capture_depth_float_buffer()
53            image = vis.capture_screen_float_buffer()
54            plt.imsave(os.path.join(depth_path, "{:05d}.png".format(glb.index)),
55                       np.asarray(depth),
56                       dpi=1)
57            plt.imsave(os.path.join(image_path, "{:05d}.png".format(glb.index)),
58                       np.asarray(image),
59                       dpi=1)
60            '''
61
62        glb.index = glb.index + 1
63        if glb.index < len(glb.trajectory.parameters):
64            ctr.convert_from_pinhole_camera_parameters(
65                glb.trajectory.parameters[glb.index])
66        else:
67            custom_draw_geometry_with_camera_trajectory.vis.destroy_window()
68
69        # Return false as we don't need to call UpdateGeometry()
70        return False
71
72    vis = custom_draw_geometry_with_camera_trajectory.vis
73    vis.create_window()
74    vis.add_geometry(pcd)
75    vis.get_render_option().load_from_json(render_option_path)
76    vis.register_animation_callback(move_forward)
77    vis.run()
78
79
80if __name__ == "__main__":
81    if not o3d._build_config['ENABLE_HEADLESS_RENDERING']:
82        print("Headless rendering is not enabled. "
83              "Please rebuild Open3D with ENABLE_HEADLESS_RENDERING=ON")
84        exit(1)
85
86    sample_data = o3d.data.DemoCustomVisualization()
87    pcd = o3d.io.read_point_cloud(sample_data.point_cloud_path)
88    print("Customized visualization playing a camera trajectory. "
89          "Press ctrl+z to terminate.")
90    custom_draw_geometry_with_camera_trajectory(
91        pcd, sample_data.camera_trajectory_path, sample_data.render_option_path,
92        'HeadlessRenderingOutput')

interactive_visualization.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8# examples/python/visualization/interactive_visualization.py
 9
10import numpy as np
11import copy
12import open3d as o3d
13
14
15def demo_crop_geometry():
16    print("Demo for manual geometry cropping")
17    print(
18        "1) Press 'Y' twice to align geometry with negative direction of y-axis"
19    )
20    print("2) Press 'K' to lock screen and to switch to selection mode")
21    print("3) Drag for rectangle selection,")
22    print("   or use ctrl + left click for polygon selection")
23    print("4) Press 'C' to get a selected geometry")
24    print("5) Press 'S' to save the selected geometry")
25    print("6) Press 'F' to switch to freeview mode")
26    pcd_data = o3d.data.DemoICPPointClouds()
27    pcd = o3d.io.read_point_cloud(pcd_data.paths[0])
28    o3d.visualization.draw_geometries_with_editing([pcd])
29
30
31def draw_registration_result(source, target, transformation):
32    source_temp = copy.deepcopy(source)
33    target_temp = copy.deepcopy(target)
34    source_temp.paint_uniform_color([1, 0.706, 0])
35    target_temp.paint_uniform_color([0, 0.651, 0.929])
36    source_temp.transform(transformation)
37    o3d.visualization.draw_geometries([source_temp, target_temp])
38
39
40def prepare_data():
41    pcd_data = o3d.data.DemoICPPointClouds()
42    source = o3d.io.read_point_cloud(pcd_data.paths[0])
43    target = o3d.io.read_point_cloud(pcd_data.paths[2])
44    print("Visualization of two point clouds before manual alignment")
45    draw_registration_result(source, target, np.identity(4))
46    return source, target
47
48
49def pick_points(pcd):
50    print("")
51    print(
52        "1) Please pick at least three correspondences using [shift + left click]"
53    )
54    print("   Press [shift + right click] to undo point picking")
55    print("2) After picking points, press 'Q' to close the window")
56    vis = o3d.visualization.VisualizerWithEditing()
57    vis.create_window()
58    vis.add_geometry(pcd)
59    vis.run()  # user picks points
60    vis.destroy_window()
61    print("")
62    return vis.get_picked_points()
63
64
65def register_via_correspondences(source, target, source_points, target_points):
66    corr = np.zeros((len(source_points), 2))
67    corr[:, 0] = source_points
68    corr[:, 1] = target_points
69    # estimate rough transformation using correspondences
70    print("Compute a rough transform using the correspondences given by user")
71    p2p = o3d.pipelines.registration.TransformationEstimationPointToPoint()
72    trans_init = p2p.compute_transformation(source, target,
73                                            o3d.utility.Vector2iVector(corr))
74    # point-to-point ICP for refinement
75    print("Perform point-to-point ICP refinement")
76    threshold = 0.03  # 3cm distance threshold
77    reg_p2p = o3d.pipelines.registration.registration_icp(
78        source, target, threshold, trans_init,
79        o3d.pipelines.registration.TransformationEstimationPointToPoint())
80    draw_registration_result(source, target, reg_p2p.transformation)
81
82
83def demo_manual_registration():
84    print("Demo for manual ICP")
85    source, target = prepare_data()
86
87    # pick points from two point clouds and builds correspondences
88    source_points = pick_points(source)
89    target_points = pick_points(target)
90    assert (len(source_points) >= 3 and len(target_points) >= 3)
91    assert (len(source_points) == len(target_points))
92    register_via_correspondences(source, target, source_points, target_points)
93    print("")
94
95
96if __name__ == "__main__":
97    demo_crop_geometry()
98    demo_manual_registration()

line_width.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import open3d as o3d
 9import random
10
11NUM_LINES = 10
12
13
14def random_point():
15    return [5 * random.random(), 5 * random.random(), 5 * random.random()]
16
17
18def main():
19    pts = [random_point() for _ in range(0, 2 * NUM_LINES)]
20    line_indices = [[2 * i, 2 * i + 1] for i in range(0, NUM_LINES)]
21    colors = [[0.0, 0.0, 0.0] for _ in range(0, NUM_LINES)]
22
23    lines = o3d.geometry.LineSet()
24    lines.points = o3d.utility.Vector3dVector(pts)
25    lines.lines = o3d.utility.Vector2iVector(line_indices)
26    # The default color of the lines is white, which will be invisible on the
27    # default white background. So we either need to set the color of the lines
28    # or the base_color of the material.
29    lines.colors = o3d.utility.Vector3dVector(colors)
30
31    # Some platforms do not require OpenGL implementations to support wide lines,
32    # so the renderer requires a custom shader to implement this: "unlitLine".
33    # The line_width field is only used by this shader; all other shaders ignore
34    # it.
35    mat = o3d.visualization.rendering.MaterialRecord()
36    mat.shader = "unlitLine"
37    mat.line_width = 10  # note that this is scaled with respect to pixels,
38    # so will give different results depending on the
39    # scaling values of your system
40    o3d.visualization.draw({
41        "name": "lines",
42        "geometry": lines,
43        "material": mat
44    })
45
46
47if __name__ == "__main__":
48    main()

load_save_viewpoint.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import open3d as o3d
 9
10
11def save_view_point(pcd, filename):
12    vis = o3d.visualization.Visualizer()
13    vis.create_window()
14    vis.add_geometry(pcd)
15    vis.run()  # user changes the view and press "q" to terminate
16    param = vis.get_view_control().convert_to_pinhole_camera_parameters()
17    o3d.io.write_pinhole_camera_parameters(filename, param)
18    vis.destroy_window()
19
20
21def load_view_point(pcd, filename):
22    vis = o3d.visualization.Visualizer()
23    vis.create_window()
24    ctr = vis.get_view_control()
25    param = o3d.io.read_pinhole_camera_parameters(filename)
26    vis.add_geometry(pcd)
27    ctr.convert_from_pinhole_camera_parameters(param)
28    vis.run()
29    vis.destroy_window()
30
31
32if __name__ == "__main__":
33    pcd_data = o3d.data.PCDPointCloud()
34    pcd = o3d.io.read_point_cloud(pcd_data.path)
35    save_view_point(pcd, "viewpoint.json")
36    load_view_point(pcd, "viewpoint.json")

mitsuba_material_estimation.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import sys
  9import argparse
 10from pathlib import Path
 11import open3d as o3d
 12import mitsuba as mi
 13import drjit as dr
 14import numpy as np
 15import math
 16
 17
 18def make_mitsuba_scene(mesh, cam_xform, fov, width, height, principle_pts,
 19                       envmap):
 20    # Camera transform
 21    t_from_np = mi.ScalarTransform4f(cam_xform)
 22    # Transform necessary to get from Open3D's environment map coordinate system
 23    # to Mitsuba's
 24    env_t = mi.ScalarTransform4f.rotate(axis=[0, 0, 1],
 25                                        angle=90).rotate(axis=[1, 0, 0],
 26                                                         angle=90)
 27    scene_dict = {
 28        "type": "scene",
 29        "integrator": {
 30            'type': 'path'
 31        },
 32        "light": {
 33            "type": "envmap",
 34            "to_world": env_t,
 35            "bitmap": mi.Bitmap(envmap),
 36        },
 37        "sensor": {
 38            "type": "perspective",
 39            "fov": fov,
 40            "to_world": t_from_np,
 41            "principal_point_offset_x": principle_pts[0],
 42            "principal_point_offset_y": principle_pts[1],
 43            "thefilm": {
 44                "type": "hdrfilm",
 45                "width": width,
 46                "height": height,
 47            },
 48            "thesampler": {
 49                "type": "multijitter",
 50                "sample_count": 64,
 51            },
 52        },
 53        "themesh": mesh,
 54    }
 55
 56    scene = mi.load_dict(scene_dict)
 57    return scene
 58
 59
 60def run_estimation(mesh, cam_info, ref_image, env_width, iterations, tv_alpha):
 61    # Make Mitsuba mesh from Open3D mesh -- conversion will attach a Mitsuba
 62    # Principled BSDF to the mesh
 63    mesh_opt = mesh.to_mitsuba('themesh')
 64
 65    # Prepare empty environment map
 66    empty_envmap = np.ones((int(env_width / 2), env_width, 3))
 67
 68    # Create Mitsuba scene
 69    scene = make_mitsuba_scene(mesh_opt, cam_info[0], cam_info[1], cam_info[2],
 70                               cam_info[3], cam_info[4], empty_envmap)
 71
 72    def total_variation(image, alpha):
 73        diff1 = image[1:, :, :] - image[:-1, :, :]
 74        diff2 = image[:, 1:, :] - image[:, :-1, :]
 75        return alpha * (dr.sum(dr.abs(diff1)) / len(diff1) +
 76                        dr.sum(dr.abs(diff2)) / len(diff2))
 77
 78    def mse(image, ref_img):
 79        return dr.mean(dr.sqr(image - ref_img))
 80
 81    params = mi.traverse(scene)
 82    print(params)
 83
 84    # Create a Mitsuba Optimizer and configure it to optimize albedo and
 85    # environment maps
 86    opt = mi.ad.Adam(lr=0.05, mask_updates=True)
 87    opt['themesh.bsdf.base_color.data'] = params['themesh.bsdf.base_color.data']
 88    opt['light.data'] = params['light.data']
 89    params.update(opt)
 90
 91    integrator = mi.load_dict({'type': 'prb'})
 92    for i in range(iterations):
 93        img = mi.render(scene, params, spp=8, seed=i, integrator=integrator)
 94
 95        # Compute loss
 96        loss = mse(img, ref_image)
 97        # Apply TV regularization if requested
 98        if tv_alpha > 0.0:
 99            loss = loss + total_variation(opt['themesh.bsdf.base_color.data'],
100                                          tv_alpha)
101
102        # Backpropogate and step. Note: if we were optimizing over a larger set
103        # of inputs not just a single image we might want to step only every x
104        # number of inputs
105        dr.backward(loss)
106        opt.step()
107
108        # Make sure albedo values stay in allowed range
109        opt['themesh.bsdf.base_color.data'] = dr.clamp(
110            opt['themesh.bsdf.base_color.data'], 0.0, 1.0)
111        params.update(opt)
112        print(f'Iteration {i} complete')
113
114    # Done! Return the estimated maps
115    albedo_img = params['themesh.bsdf.base_color.data'].numpy()
116    envmap_img = params['light.data'].numpy()
117    return (albedo_img, envmap_img)
118
119
120def load_input_mesh(model_path, tex_dim):
121    mesh = o3d.t.io.read_triangle_mesh(str(model_path))
122    mesh.material.set_default_properties()
123    mesh.material.material_name = 'defaultLit'  # note: ignored by Mitsuba, just used to visualize in Open3D
124    mesh.material.texture_maps['albedo'] = o3d.t.geometry.Image(0.5 + np.zeros(
125        (tex_dim, tex_dim, 3), dtype=np.float32))
126    return mesh
127
128
129def load_input_data(object, camera_pose, input_image, tex_dim):
130    print(f'Loading {object}...')
131    mesh = load_input_mesh(object, tex_dim)
132
133    print(f'Loading camera pose from {camera_pose}...')
134    cam_npz = np.load(camera_pose)
135    img_width = cam_npz['width'].item()
136    img_height = cam_npz['height'].item()
137    cam_xform = np.linalg.inv(cam_npz['T'])
138    cam_xform = np.matmul(
139        cam_xform,
140        np.array([[-1, 0, 0, 0], [0, -1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
141                 dtype=np.float32))
142    fov = 2 * np.arctan(0.5 * img_width / cam_npz['K'][0, 0])
143    fov = (180.0 / math.pi) * fov.item()
144    camera = (cam_xform, fov, img_width, img_height, (0.0, 0.0))
145
146    print(f'Loading reference image from {input_image}...')
147    ref_img = o3d.t.io.read_image(str(input_image))
148    ref_img = ref_img.as_tensor()[:, :, 0:3].to(o3d.core.Dtype.Float32) / 255.0
149    bmp = mi.Bitmap(ref_img.numpy()).convert(srgb_gamma=False)
150    ref_img = mi.TensorXf(bmp)
151    return (mesh, camera, ref_img)
152
153
154if __name__ == '__main__':
155    parser = argparse.ArgumentParser(
156        description=
157        "Script that estimates texture and environment map from an input image and geometry. You can find data to test this script here: https://github.com/isl-org/open3d_downloads/releases/download/mitsuba-demos/raven_mitsuba.zip.",
158        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
159    parser.add_argument(
160        'object_path',
161        type=Path,
162        help=
163        "Path to geometry for which to estimate albedo. It is assumed that in the same directory will be an object-name.npz which contains the camera pose information and an object-name.png which is the input image"
164    )
165    parser.add_argument('--env-width', type=int, default=1024)
166    parser.add_argument('--tex-width',
167                        type=int,
168                        default=2048,
169                        help="The dimensions of the texture")
170    parser.add_argument(
171        '--device',
172        default='cuda' if o3d.core.cuda.is_available() else 'cpu',
173        choices=('cpu', 'cuda'),
174        help="Run Mitsuba on 'cuda' or 'cpu'")
175    parser.add_argument('--iterations',
176                        type=int,
177                        default=40,
178                        help="Number of iterations")
179    parser.add_argument(
180        '--total-variation',
181        type=float,
182        default=0.01,
183        help="Factor to apply to total_variation loss. 0.0 disables TV")
184
185    if len(sys.argv) < 2:
186        parser.print_help(sys.stderr)
187        sys.exit(1)
188    args = parser.parse_args()
189    print("Arguments: ", vars(args))
190
191    # Initialize Mitsuba
192    if args.device == 'cpu':
193        mi.set_variant('llvm_ad_rgb')
194    else:
195        mi.set_variant('cuda_ad_rgb')
196
197    # Confirm that the 3 required inputs exist
198    object_path = args.object_path
199    object_name = object_path.stem
200    datadir = args.object_path.parent
201    camera_pose = datadir / (object_name + '.npz')
202    input_image = datadir / (object_name + '.png')
203    if not object_path.exists():
204        print(f'{object_path} does not exist!')
205        sys.exit()
206    if not camera_pose.exists():
207        print(f'{camera_pose} does not exist!')
208        sys.exit()
209    if not input_image.exists():
210        print(f'{input_image} does not exist!')
211        sys.exit()
212
213    # Load input data
214    mesh, cam_info, input_image = load_input_data(object_path, camera_pose,
215                                                  input_image, args.tex_width)
216
217    # Estimate albedo map
218    print('Running material estimation...')
219    albedo, envmap = run_estimation(mesh, cam_info, input_image, args.env_width,
220                                    args.iterations, args.total_variation)
221
222    # Save maps
223    def save_image(img, name, output_dir):
224        # scale to 0-255
225        texture = o3d.core.Tensor(img * 255.0).to(o3d.core.Dtype.UInt8)
226        texture = o3d.t.geometry.Image(texture)
227        o3d.t.io.write_image(str(output_dir / name), texture)
228
229    print('Saving final results...')
230    save_image(albedo, 'estimated_albedo.png', datadir)
231    mi.Bitmap(envmap).write(str(datadir / 'predicted_envmap.exr'))
232
233    # Visualize result with Open3D
234    mesh.material.texture_maps['albedo'] = o3d.t.io.read_image(
235        str(datadir / 'estimated_albedo.png'))
236    o3d.visualization.draw(mesh)

mouse_and_point_coord.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import numpy as np
  9import open3d as o3d
 10import open3d.visualization.gui as gui
 11import open3d.visualization.rendering as rendering
 12
 13
 14# This example displays a point cloud and if you Ctrl-click on a point
 15# (Cmd-click on macOS) it will show the coordinates of the point.
 16# This example illustrates:
 17# - custom mouse handling on SceneWidget
 18# - getting a the depth value of a point (OpenGL depth)
 19# - converting from a window point + OpenGL depth to world coordinate
 20class ExampleApp:
 21
 22    def __init__(self, cloud):
 23        # We will create a SceneWidget that fills the entire window, and then
 24        # a label in the lower left on top of the SceneWidget to display the
 25        # coordinate.
 26        app = gui.Application.instance
 27        self.window = app.create_window("Open3D - GetCoord Example", 1024, 768)
 28        # Since we want the label on top of the scene, we cannot use a layout,
 29        # so we need to manually layout the window's children.
 30        self.window.set_on_layout(self._on_layout)
 31        self.widget3d = gui.SceneWidget()
 32        self.window.add_child(self.widget3d)
 33        self.info = gui.Label("")
 34        self.info.visible = False
 35        self.window.add_child(self.info)
 36
 37        self.widget3d.scene = rendering.Open3DScene(self.window.renderer)
 38
 39        mat = rendering.MaterialRecord()
 40        mat.shader = "defaultUnlit"
 41        # Point size is in native pixels, but "pixel" means different things to
 42        # different platforms (macOS, in particular), so multiply by Window scale
 43        # factor.
 44        mat.point_size = 3 * self.window.scaling
 45        self.widget3d.scene.add_geometry("Point Cloud", cloud, mat)
 46
 47        bounds = self.widget3d.scene.bounding_box
 48        center = bounds.get_center()
 49        self.widget3d.setup_camera(60, bounds, center)
 50        self.widget3d.look_at(center, center - [0, 0, 3], [0, -1, 0])
 51
 52        self.widget3d.set_on_mouse(self._on_mouse_widget3d)
 53
 54    def _on_layout(self, layout_context):
 55        r = self.window.content_rect
 56        self.widget3d.frame = r
 57        pref = self.info.calc_preferred_size(layout_context,
 58                                             gui.Widget.Constraints())
 59        self.info.frame = gui.Rect(r.x,
 60                                   r.get_bottom() - pref.height, pref.width,
 61                                   pref.height)
 62
 63    def _on_mouse_widget3d(self, event):
 64        # We could override BUTTON_DOWN without a modifier, but that would
 65        # interfere with manipulating the scene.
 66        if event.type == gui.MouseEvent.Type.BUTTON_DOWN and event.is_modifier_down(
 67                gui.KeyModifier.CTRL):
 68
 69            def depth_callback(depth_image):
 70                # Coordinates are expressed in absolute coordinates of the
 71                # window, but to dereference the image correctly we need them
 72                # relative to the origin of the widget. Note that even if the
 73                # scene widget is the only thing in the window, if a menubar
 74                # exists it also takes up space in the window (except on macOS).
 75                x = event.x - self.widget3d.frame.x
 76                y = event.y - self.widget3d.frame.y
 77                # Note that np.asarray() reverses the axes.
 78                depth = np.asarray(depth_image)[y, x]
 79
 80                if depth == 1.0:  # clicked on nothing (i.e. the far plane)
 81                    text = ""
 82                else:
 83                    world = self.widget3d.scene.camera.unproject(
 84                        x, y, depth, self.widget3d.frame.width,
 85                        self.widget3d.frame.height)
 86                    text = "({:.3f}, {:.3f}, {:.3f})".format(
 87                        world[0], world[1], world[2])
 88
 89                # This is not called on the main thread, so we need to
 90                # post to the main thread to safely access UI items.
 91                def update_label():
 92                    self.info.text = text
 93                    self.info.visible = (text != "")
 94                    # We are sizing the info label to be exactly the right size,
 95                    # so since the text likely changed width, we need to
 96                    # re-layout to set the new frame.
 97                    self.window.set_needs_layout()
 98
 99                gui.Application.instance.post_to_main_thread(
100                    self.window, update_label)
101
102            self.widget3d.scene.scene.render_to_depth_image(depth_callback)
103            return gui.Widget.EventCallbackResult.HANDLED
104        return gui.Widget.EventCallbackResult.IGNORED
105
106
107def main():
108    app = gui.Application.instance
109    app.initialize()
110
111    # This example will also work with a triangle mesh, or any 3D object.
112    # If you use a triangle mesh you will probably want to set the material
113    # shader to "defaultLit" instead of "defaultUnlit".
114    pcd_data = o3d.data.DemoICPPointClouds()
115    cloud = o3d.io.read_point_cloud(pcd_data.paths[0])
116    ex = ExampleApp(cloud)
117
118    app.run()
119
120
121if __name__ == "__main__":
122    main()

multiple_windows.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import numpy as np
  9import open3d as o3d
 10import threading
 11import time
 12
 13CLOUD_NAME = "points"
 14
 15
 16def main():
 17    MultiWinApp().run()
 18
 19
 20class MultiWinApp:
 21
 22    def __init__(self):
 23        self.is_done = False
 24        self.n_snapshots = 0
 25        self.cloud = None
 26        self.main_vis = None
 27        self.snapshot_pos = None
 28
 29    def run(self):
 30        app = o3d.visualization.gui.Application.instance
 31        app.initialize()
 32
 33        self.main_vis = o3d.visualization.O3DVisualizer(
 34            "Open3D - Multi-Window Demo")
 35        self.main_vis.add_action("Take snapshot in new window",
 36                                 self.on_snapshot)
 37        self.main_vis.set_on_close(self.on_main_window_closing)
 38
 39        app.add_window(self.main_vis)
 40        self.snapshot_pos = (self.main_vis.os_frame.x, self.main_vis.os_frame.y)
 41
 42        threading.Thread(target=self.update_thread).start()
 43
 44        app.run()
 45
 46    def on_snapshot(self, vis):
 47        self.n_snapshots += 1
 48        self.snapshot_pos = (self.snapshot_pos[0] + 50,
 49                             self.snapshot_pos[1] + 50)
 50        title = "Open3D - Multi-Window Demo (Snapshot #" + str(
 51            self.n_snapshots) + ")"
 52        new_vis = o3d.visualization.O3DVisualizer(title)
 53        mat = o3d.visualization.rendering.MaterialRecord()
 54        mat.shader = "defaultUnlit"
 55        new_vis.add_geometry(CLOUD_NAME + " #" + str(self.n_snapshots),
 56                             self.cloud, mat)
 57        new_vis.reset_camera_to_default()
 58        bounds = self.cloud.get_axis_aligned_bounding_box()
 59        extent = bounds.get_extent()
 60        new_vis.setup_camera(60, bounds.get_center(),
 61                             bounds.get_center() + [0, 0, -3], [0, -1, 0])
 62        o3d.visualization.gui.Application.instance.add_window(new_vis)
 63        new_vis.os_frame = o3d.visualization.gui.Rect(self.snapshot_pos[0],
 64                                                      self.snapshot_pos[1],
 65                                                      new_vis.os_frame.width,
 66                                                      new_vis.os_frame.height)
 67
 68    def on_main_window_closing(self):
 69        self.is_done = True
 70        return True  # False would cancel the close
 71
 72    def update_thread(self):
 73        # This is NOT the UI thread, need to call post_to_main_thread() to update
 74        # the scene or any part of the UI.
 75        pcd_data = o3d.data.DemoICPPointClouds()
 76        self.cloud = o3d.io.read_point_cloud(pcd_data.paths[0])
 77        bounds = self.cloud.get_axis_aligned_bounding_box()
 78        extent = bounds.get_extent()
 79
 80        def add_first_cloud():
 81            mat = o3d.visualization.rendering.MaterialRecord()
 82            mat.shader = "defaultUnlit"
 83            self.main_vis.add_geometry(CLOUD_NAME, self.cloud, mat)
 84            self.main_vis.reset_camera_to_default()
 85            self.main_vis.setup_camera(60, bounds.get_center(),
 86                                       bounds.get_center() + [0, 0, -3],
 87                                       [0, -1, 0])
 88
 89        o3d.visualization.gui.Application.instance.post_to_main_thread(
 90            self.main_vis, add_first_cloud)
 91
 92        while not self.is_done:
 93            time.sleep(0.1)
 94
 95            # Perturb the cloud with a random walk to simulate an actual read
 96            pts = np.asarray(self.cloud.points)
 97            magnitude = 0.005 * extent
 98            displacement = magnitude * (np.random.random_sample(pts.shape) -
 99                                        0.5)
100            new_pts = pts + displacement
101            self.cloud.points = o3d.utility.Vector3dVector(new_pts)
102
103            def update_cloud():
104                # Note: if the number of points is less than or equal to the
105                #       number of points in the original object that was added,
106                #       using self.scene.update_geometry() will be faster.
107                #       Requires that the point cloud be a t.PointCloud.
108                self.main_vis.remove_geometry(CLOUD_NAME)
109                mat = o3d.visualization.rendering.MaterialRecord()
110                mat.shader = "defaultUnlit"
111                self.main_vis.add_geometry(CLOUD_NAME, self.cloud, mat)
112
113            if self.is_done:  # might have changed while sleeping
114                break
115            o3d.visualization.gui.Application.instance.post_to_main_thread(
116                self.main_vis, update_cloud)
117
118
119if __name__ == "__main__":
120    main()

non_blocking_visualization.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8# examples/python/visualization/non_blocking_visualization.py
 9
10import open3d as o3d
11import numpy as np
12
13
14def prepare_data():
15    pcd_data = o3d.data.DemoICPPointClouds()
16    source_raw = o3d.io.read_point_cloud(pcd_data.paths[0])
17    target_raw = o3d.io.read_point_cloud(pcd_data.paths[1])
18    source = source_raw.voxel_down_sample(voxel_size=0.02)
19    target = target_raw.voxel_down_sample(voxel_size=0.02)
20
21    trans = [[0.862, 0.011, -0.507, 0.0], [-0.139, 0.967, -0.215, 0.7],
22             [0.487, 0.255, 0.835, -1.4], [0.0, 0.0, 0.0, 1.0]]
23    source.transform(trans)
24    flip_transform = [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]
25    source.transform(flip_transform)
26    target.transform(flip_transform)
27    return source, target
28
29
30def demo_non_blocking_visualization():
31    o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug)
32
33    source, target = prepare_data()
34    vis = o3d.visualization.Visualizer()
35    vis.create_window()
36    vis.add_geometry(source)
37    vis.add_geometry(target)
38    threshold = 0.05
39    icp_iteration = 100
40    save_image = False
41
42    for i in range(icp_iteration):
43        reg_p2l = o3d.pipelines.registration.registration_icp(
44            source, target, threshold, np.identity(4),
45            o3d.pipelines.registration.TransformationEstimationPointToPlane(),
46            o3d.pipelines.registration.ICPConvergenceCriteria(max_iteration=1))
47        source.transform(reg_p2l.transformation)
48        vis.update_geometry(source)
49        vis.poll_events()
50        vis.update_renderer()
51        if save_image:
52            vis.capture_screen_image("temp_%04d.jpg" % i)
53    vis.destroy_window()
54
55    o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Info)
56
57
58if __name__ == '__main__':
59    demo_non_blocking_visualization()

non_english.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import open3d.visualization.gui as gui
  9import os.path
 10import platform
 11
 12basedir = os.path.dirname(os.path.realpath(__file__))
 13
 14# This is all-widgets.py with some modifications for non-English languages.
 15# Please see all-widgets.py for usage of the GUI widgets
 16
 17MODE_SERIF = "serif"
 18MODE_COMMON_HANYU = "common"
 19MODE_SERIF_AND_COMMON_HANYU = "serif+common"
 20MODE_COMMON_HANYU_EN = "hanyu_en+common"
 21MODE_ALL_HANYU = "all"
 22MODE_CUSTOM_CHARS = "custom"
 23
 24#mode = MODE_SERIF
 25#mode = MODE_COMMON_HANYU
 26mode = MODE_SERIF_AND_COMMON_HANYU
 27#mode = MODE_ALL_HANYU
 28#mode = MODE_CUSTOM_CHARS
 29
 30# Fonts can be names or paths
 31if platform.system() == "Darwin":
 32    serif = "Times New Roman"
 33    hanzi = "STHeiti Light"
 34    chess = "/System/Library/Fonts/Apple Symbols.ttf"
 35elif platform.system() == "Windows":
 36    # it is necessary to specify paths on Windows since it stores its fonts
 37    # with a cryptic name, so font name searches do not work on Windows
 38    serif = "c:/windows/fonts/times.ttf"  # Times New Roman
 39    hanzi = "c:/windows/fonts/msyh.ttc"  # YaHei UI
 40    chess = "c:/windows/fonts/seguisym.ttf"  # Segoe UI Symbol
 41else:
 42    # Assumes Ubuntu 18.04
 43    serif = "DejaVuSerif"
 44    hanzi = "NotoSansCJK"
 45    chess = "/usr/share/fonts/truetype/freefont/FreeSerif.ttf"
 46
 47
 48def main():
 49    gui.Application.instance.initialize()
 50
 51    # Font changes must be done after initialization but before creating
 52    # a window.
 53
 54    # MODE_SERIF changes the English font; Chinese will not be displayed
 55    font = None
 56    if mode == MODE_SERIF:
 57        font = gui.FontDescription(serif)
 58    # MODE_COMMON_HANYU uses the default English font and adds common Chinese
 59    elif mode == MODE_COMMON_HANYU:
 60        font = gui.FontDescription()
 61        font.add_typeface_for_language(hanzi, "zh")
 62    # MODE_SERIF_AND_COMMON_HANYU uses a serif English font and adds common
 63    # Chinese characters
 64    elif mode == MODE_SERIF_AND_COMMON_HANYU:
 65        font = gui.FontDescription(serif)
 66        font.add_typeface_for_language(hanzi, "zh")
 67    # MODE_COMMON_HANYU_EN the Chinese font for both English and the common
 68    # characters
 69    elif mode == MODE_COMMON_HANYU_EN:
 70        font = gui.FontDescription(hanzi)
 71        font.add_typeface_for_language(hanzi, "zh")
 72    # MODE_ALL_HANYU uses the default English font but includes all the Chinese
 73    # characters (which uses a substantial amount of memory)
 74    elif mode == MODE_ALL_HANYU:
 75        font = gui.FontDescription()
 76        font.add_typeface_for_language(hanzi, "zh_all")
 77    elif mode == MODE_CUSTOM_CHARS:
 78        range = [0x2654, 0x2655, 0x2656, 0x2657, 0x2658, 0x2659]
 79        font = gui.FontDescription()
 80        font.add_typeface_for_code_points(chess, range)
 81
 82    if font is not None:
 83        gui.Application.instance.set_font(gui.Application.DEFAULT_FONT_ID, font)
 84
 85    w = ExampleWindow()
 86    gui.Application.instance.run()
 87
 88
 89class ExampleWindow:
 90    MENU_CHECKABLE = 1
 91    MENU_DISABLED = 2
 92    MENU_QUIT = 3
 93
 94    def __init__(self):
 95        self.window = gui.Application.instance.create_window("Test", 400, 768)
 96        # self.window = gui.Application.instance.create_window("Test", 400, 768,
 97        #                                                        x=50, y=100)
 98        w = self.window  # for more concise code
 99
100        # Rather than specifying sizes in pixels, which may vary in size based
101        # on the monitor, especially on macOS which has 220 dpi monitors, use
102        # the em-size. This way sizings will be proportional to the font size,
103        # which will create a more visually consistent size across platforms.
104        em = w.theme.font_size
105
106        # Widgets are laid out in layouts: gui.Horiz, gui.Vert,
107        # gui.CollapsableVert, and gui.VGrid. By nesting the layouts we can
108        # achieve complex designs. Usually we use a vertical layout as the
109        # topmost widget, since widgets tend to be organized from top to bottom.
110        # Within that, we usually have a series of horizontal layouts for each
111        # row.
112        layout = gui.Vert(0, gui.Margins(0.5 * em, 0.5 * em, 0.5 * em,
113                                         0.5 * em))
114
115        # Create the menu. The menu is global (because the macOS menu is global),
116        # so only create it once.
117        if gui.Application.instance.menubar is None:
118            menubar = gui.Menu()
119            test_menu = gui.Menu()
120            test_menu.add_item("An option", ExampleWindow.MENU_CHECKABLE)
121            test_menu.set_checked(ExampleWindow.MENU_CHECKABLE, True)
122            test_menu.add_item("Unavailable feature",
123                               ExampleWindow.MENU_DISABLED)
124            test_menu.set_enabled(ExampleWindow.MENU_DISABLED, False)
125            test_menu.add_separator()
126            test_menu.add_item("Quit", ExampleWindow.MENU_QUIT)
127            # On macOS the first menu item is the application menu item and will
128            # always be the name of the application (probably "Python"),
129            # regardless of what you pass in here. The application menu is
130            # typically where About..., Preferences..., and Quit go.
131            menubar.add_menu("Test", test_menu)
132            gui.Application.instance.menubar = menubar
133
134        # Each window needs to know what to do with the menu items, so we need
135        # to tell the window how to handle menu items.
136        w.set_on_menu_item_activated(ExampleWindow.MENU_CHECKABLE,
137                                     self._on_menu_checkable)
138        w.set_on_menu_item_activated(ExampleWindow.MENU_QUIT,
139                                     self._on_menu_quit)
140
141        # Create a file-chooser widget. One part will be a text edit widget for
142        # the filename and clicking on the button will let the user choose using
143        # the file dialog.
144        self._fileedit = gui.TextEdit()
145        filedlgbutton = gui.Button("...")
146        filedlgbutton.horizontal_padding_em = 0.5
147        filedlgbutton.vertical_padding_em = 0
148        filedlgbutton.set_on_clicked(self._on_filedlg_button)
149
150        # (Create the horizontal widget for the row. This will make sure the
151        # text editor takes up as much space as it can.)
152        fileedit_layout = gui.Horiz()
153        fileedit_layout.add_child(gui.Label("Model file"))
154        fileedit_layout.add_child(self._fileedit)
155        fileedit_layout.add_fixed(0.25 * em)
156        fileedit_layout.add_child(filedlgbutton)
157        # add to the top-level (vertical) layout
158        layout.add_child(fileedit_layout)
159
160        # Create a collapsible vertical widget, which takes up enough vertical
161        # space for all its children when open, but only enough for text when
162        # closed. This is useful for property pages, so the user can hide sets
163        # of properties they rarely use. All layouts take a spacing parameter,
164        # which is the spacinging between items in the widget, and a margins
165        # parameter, which specifies the spacing of the left, top, right,
166        # bottom margins. (This acts like the 'padding' property in CSS.)
167        collapse = gui.CollapsableVert("Widgets", 0.33 * em,
168                                       gui.Margins(em, 0, 0, 0))
169        if mode == MODE_CUSTOM_CHARS:
170            self._label = gui.Label("♔♕♖♗♘♙")
171        elif mode == MODE_ALL_HANYU:
172            self._label = gui.Label("天地玄黃,宇宙洪荒。日月盈昃,辰宿列張。")
173        else:
174            self._label = gui.Label("锄禾日当午,汗滴禾下土。谁知盘中餐,粒粒皆辛苦。")
175        self._label.text_color = gui.Color(1.0, 0.5, 0.0)
176        collapse.add_child(self._label)
177
178        # Create a checkbox. Checking or unchecking would usually be used to set
179        # a binary property, but in this case it will show a simple message box,
180        # which illustrates how to create simple dialogs.
181        cb = gui.Checkbox("Enable some really cool effect")
182        cb.set_on_checked(self._on_cb)  # set the callback function
183        collapse.add_child(cb)
184
185        # Create a color editor. We will change the color of the orange label
186        # above when the color changes.
187        color = gui.ColorEdit()
188        color.color_value = self._label.text_color
189        color.set_on_value_changed(self._on_color)
190        collapse.add_child(color)
191
192        # This is a combobox, nothing fancy here, just set a simple function to
193        # handle the user selecting an item.
194        combo = gui.Combobox()
195        combo.add_item("Show point labels")
196        combo.add_item("Show point velocity")
197        combo.add_item("Show bounding boxes")
198        combo.set_on_selection_changed(self._on_combo)
199        collapse.add_child(combo)
200
201        # Add a simple image
202        logo = gui.ImageWidget(basedir + "/icon-32.png")
203        collapse.add_child(logo)
204
205        # Add a list of items
206        lv = gui.ListView()
207        lv.set_items(["Ground", "Trees", "Buildings" "Cars", "People"])
208        lv.selected_index = lv.selected_index + 2  # initially is -1, so now 1
209        lv.set_on_selection_changed(self._on_list)
210        collapse.add_child(lv)
211
212        # Add a tree view
213        tree = gui.TreeView()
214        tree.add_text_item(tree.get_root_item(), "Camera")
215        geo_id = tree.add_text_item(tree.get_root_item(), "Geometries")
216        mesh_id = tree.add_text_item(geo_id, "Mesh")
217        tree.add_text_item(mesh_id, "Triangles")
218        tree.add_text_item(mesh_id, "Albedo texture")
219        tree.add_text_item(mesh_id, "Normal map")
220        points_id = tree.add_text_item(geo_id, "Points")
221        tree.can_select_items_with_children = True
222        tree.set_on_selection_changed(self._on_tree)
223        # does not call on_selection_changed: user did not change selection
224        tree.selected_item = points_id
225        collapse.add_child(tree)
226
227        # Add two number editors, one for integers and one for floating point
228        # Number editor can clamp numbers to a range, although this is more
229        # useful for integers than for floating point.
230        intedit = gui.NumberEdit(gui.NumberEdit.INT)
231        intedit.int_value = 0
232        intedit.set_limits(1, 19)  # value coerced to 1
233        intedit.int_value = intedit.int_value + 2  # value should be 3
234        doubleedit = gui.NumberEdit(gui.NumberEdit.DOUBLE)
235        numlayout = gui.Horiz()
236        numlayout.add_child(gui.Label("int"))
237        numlayout.add_child(intedit)
238        numlayout.add_fixed(em)  # manual spacing (could set it in Horiz() ctor)
239        numlayout.add_child(gui.Label("double"))
240        numlayout.add_child(doubleedit)
241        collapse.add_child(numlayout)
242
243        # Create a progress bar. It ranges from 0.0 to 1.0.
244        self._progress = gui.ProgressBar()
245        self._progress.value = 0.25  # 25% complete
246        self._progress.value = self._progress.value + 0.08  # 0.25 + 0.08 = 33%
247        prog_layout = gui.Horiz(em)
248        prog_layout.add_child(gui.Label("Progress..."))
249        prog_layout.add_child(self._progress)
250        collapse.add_child(prog_layout)
251
252        # Create a slider. It acts very similar to NumberEdit except that the
253        # user moves a slider and cannot type the number.
254        slider = gui.Slider(gui.Slider.INT)
255        slider.set_limits(5, 13)
256        slider.set_on_value_changed(self._on_slider)
257        collapse.add_child(slider)
258
259        # Create a text editor. The placeholder text (if not empty) will be
260        # displayed when there is no text, as concise help, or visible tooltip.
261        tedit = gui.TextEdit()
262        tedit.placeholder_text = "Edit me some text here"
263
264        # on_text_changed fires whenever the user changes the text (but not if
265        # the text_value property is assigned to).
266        tedit.set_on_text_changed(self._on_text_changed)
267
268        # on_value_changed fires whenever the user signals that they are finished
269        # editing the text, either by pressing return or by clicking outside of
270        # the text editor, thus losing text focus.
271        tedit.set_on_value_changed(self._on_value_changed)
272        collapse.add_child(tedit)
273
274        # Create a widget for showing/editing a 3D vector
275        vedit = gui.VectorEdit()
276        vedit.vector_value = [1, 2, 3]
277        vedit.set_on_value_changed(self._on_vedit)
278        collapse.add_child(vedit)
279
280        # Create a VGrid layout. This layout specifies the number of columns
281        # (two, in this case), and will place the first child in the first
282        # column, the second in the second, the third in the first, the fourth
283        # in the second, etc.
284        # So:
285        #      2 cols             3 cols                  4 cols
286        #   |  1  |  2  |   |  1  |  2  |  3  |   |  1  |  2  |  3  |  4  |
287        #   |  3  |  4  |   |  4  |  5  |  6  |   |  5  |  6  |  7  |  8  |
288        #   |  5  |  6  |   |  7  |  8  |  9  |   |  9  | 10  | 11  | 12  |
289        #   |    ...    |   |       ...       |   |         ...           |
290        vgrid = gui.VGrid(2)
291        vgrid.add_child(gui.Label("Trees"))
292        vgrid.add_child(gui.Label("12 items"))
293        vgrid.add_child(gui.Label("People"))
294        vgrid.add_child(gui.Label("2 (93% certainty)"))
295        vgrid.add_child(gui.Label("Cars"))
296        vgrid.add_child(gui.Label("5 (87% certainty)"))
297        collapse.add_child(vgrid)
298
299        # Create a tab control. This is really a set of N layouts on top of each
300        # other, but with only one selected.
301        tabs = gui.TabControl()
302        tab1 = gui.Vert()
303        tab1.add_child(gui.Checkbox("Enable option 1"))
304        tab1.add_child(gui.Checkbox("Enable option 2"))
305        tab1.add_child(gui.Checkbox("Enable option 3"))
306        tabs.add_tab("Options", tab1)
307        tab2 = gui.Vert()
308        tab2.add_child(gui.Label("No plugins detected"))
309        tab2.add_stretch()
310        tabs.add_tab("Plugins", tab2)
311        collapse.add_child(tabs)
312
313        # Quit button. (Typically this is a menu item)
314        button_layout = gui.Horiz()
315        ok_button = gui.Button("Ok")
316        ok_button.set_on_clicked(self._on_ok)
317        button_layout.add_stretch()
318        button_layout.add_child(ok_button)
319
320        layout.add_child(collapse)
321        layout.add_child(button_layout)
322
323        # We're done, set the window's layout
324        w.add_child(layout)
325
326    def _on_filedlg_button(self):
327        filedlg = gui.FileDialog(gui.FileDialog.OPEN, "Select file",
328                                 self.window.theme)
329        filedlg.add_filter(".obj .ply .stl", "Triangle mesh (.obj, .ply, .stl)")
330        filedlg.add_filter("", "All files")
331        filedlg.set_on_cancel(self._on_filedlg_cancel)
332        filedlg.set_on_done(self._on_filedlg_done)
333        self.window.show_dialog(filedlg)
334
335    def _on_filedlg_cancel(self):
336        self.window.close_dialog()
337
338    def _on_filedlg_done(self, path):
339        self._fileedit.text_value = path
340        self.window.close_dialog()
341
342    def _on_cb(self, is_checked):
343        if is_checked:
344            text = "Sorry, effects are unimplemented"
345        else:
346            text = "Good choice"
347
348        self.show_message_dialog("There might be a problem...", text)
349
350    # This function is essentially the same as window.show_message_box(),
351    # so for something this simple just use that, but it illustrates making a
352    # dialog.
353    def show_message_dialog(self, title, message):
354        # A Dialog is just a widget, so you make its child a layout just like
355        # a Window.
356        dlg = gui.Dialog(title)
357
358        # Add the message text
359        em = self.window.theme.font_size
360        dlg_layout = gui.Vert(em, gui.Margins(em, em, em, em))
361        dlg_layout.add_child(gui.Label(message))
362
363        # Add the Ok button. We need to define a callback function to handle
364        # the click.
365        ok_button = gui.Button("Ok")
366        ok_button.set_on_clicked(self._on_dialog_ok)
367
368        # We want the Ok button to be an the right side, so we need to add
369        # a stretch item to the layout, otherwise the button will be the size
370        # of the entire row. A stretch item takes up as much space as it can,
371        # which forces the button to be its minimum size.
372        button_layout = gui.Horiz()
373        button_layout.add_stretch()
374        button_layout.add_child(ok_button)
375
376        # Add the button layout,
377        dlg_layout.add_child(button_layout)
378        # ... then add the layout as the child of the Dialog
379        dlg.add_child(dlg_layout)
380        # ... and now we can show the dialog
381        self.window.show_dialog(dlg)
382
383    def _on_dialog_ok(self):
384        self.window.close_dialog()
385
386    def _on_color(self, new_color):
387        self._label.text_color = new_color
388
389    def _on_combo(self, new_val, new_idx):
390        print(new_idx, new_val)
391
392    def _on_list(self, new_val, is_dbl_click):
393        print(new_val)
394
395    def _on_tree(self, new_item_id):
396        print(new_item_id)
397
398    def _on_slider(self, new_val):
399        self._progress.value = new_val / 20.0
400
401    def _on_text_changed(self, new_text):
402        print("edit:", new_text)
403
404    def _on_value_changed(self, new_text):
405        print("value:", new_text)
406
407    def _on_vedit(self, new_val):
408        print(new_val)
409
410    def _on_ok(self):
411        gui.Application.instance.quit()
412
413    def _on_menu_checkable(self):
414        gui.Application.instance.menubar.set_checked(
415            ExampleWindow.MENU_CHECKABLE,
416            not gui.Application.instance.menubar.is_checked(
417                ExampleWindow.MENU_CHECKABLE))
418
419    def _on_menu_quit(self):
420        gui.Application.instance.quit()
421
422
423# This class is essentially the same as window.show_message_box(),
424# so for something this simple just use that, but it illustrates making a
425# dialog.
426class MessageBox:
427
428    def __init__(self, title, message):
429        self._window = None
430
431        # A Dialog is just a widget, so you make its child a layout just like
432        # a Window.
433        dlg = gui.Dialog(title)
434
435        # Add the message text
436        em = self.window.theme.font_size
437        dlg_layout = gui.Vert(em, gui.Margins(em, em, em, em))
438        dlg_layout.add_child(gui.Label(message))
439
440        # Add the Ok button. We need to define a callback function to handle
441        # the click.
442        ok_button = gui.Button("Ok")
443        ok_button.set_on_clicked(self._on_ok)
444
445        # We want the Ok button to be an the right side, so we need to add
446        # a stretch item to the layout, otherwise the button will be the size
447        # of the entire row. A stretch item takes up as much space as it can,
448        # which forces the button to be its minimum size.
449        button_layout = gui.Horiz()
450        button_layout.add_stretch()
451        button_layout.add_child(ok_button)
452
453        # Add the button layout,
454        dlg_layout.add_child(button_layout)
455        # ... then add the layout as the child of the Dialog
456        dlg.add_child(dlg_layout)
457
458    def show(self, window):
459        self._window = window
460
461    def _on_ok(self):
462        self._window.close_dialog()
463
464
465if __name__ == "__main__":
466    main()

online_processing.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7"""Online 3D depth video processing pipeline.
  8
  9- Connects to a RGBD camera or RGBD video file (currently
 10  RealSense camera and bag file format are supported).
 11- Captures / reads color and depth frames. Allow recording from camera.
 12- Convert frames to point cloud, optionally with normals.
 13- Visualize point cloud video and results.
 14- Save point clouds and RGBD images for selected frames.
 15
 16For this example, Open3D must be built with -DBUILD_LIBREALSENSE=ON
 17"""
 18
 19import os
 20import json
 21import time
 22import logging as log
 23import argparse
 24import threading
 25from datetime import datetime
 26from concurrent.futures import ThreadPoolExecutor
 27import numpy as np
 28import open3d as o3d
 29import open3d.visualization.gui as gui
 30import open3d.visualization.rendering as rendering
 31
 32
 33# Camera and processing
 34class PipelineModel:
 35    """Controls IO (camera, video file, recording, saving frames). Methods run
 36    in worker threads."""
 37
 38    def __init__(self,
 39                 update_view,
 40                 camera_config_file=None,
 41                 rgbd_video=None,
 42                 device=None):
 43        """Initialize.
 44
 45        Args:
 46            update_view (callback): Callback to update display elements for a
 47                frame.
 48            camera_config_file (str): Camera configuration json file.
 49            rgbd_video (str): RS bag file containing the RGBD video. If this is
 50                provided, connected cameras are ignored.
 51            device (str): Compute device (e.g.: 'cpu:0' or 'cuda:0').
 52        """
 53        self.update_view = update_view
 54        if device:
 55            self.device = device.lower()
 56        else:
 57            self.device = 'cuda:0' if o3d.core.cuda.is_available() else 'cpu:0'
 58        self.o3d_device = o3d.core.Device(self.device)
 59
 60        self.video = None
 61        self.camera = None
 62        self.flag_capture = False
 63        self.cv_capture = threading.Condition()  # condition variable
 64        self.recording = False  # Are we currently recording
 65        self.flag_record = False  # Request to start/stop recording
 66        if rgbd_video:  # Video file
 67            self.video = o3d.t.io.RGBDVideoReader.create(rgbd_video)
 68            self.rgbd_metadata = self.video.metadata
 69            self.status_message = f"Video {rgbd_video} opened."
 70
 71        else:  # RGBD camera
 72            now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 73            filename = f"{now}.bag"
 74            self.camera = o3d.t.io.RealSenseSensor()
 75            if camera_config_file:
 76                with open(camera_config_file) as ccf:
 77                    self.camera.init_sensor(o3d.t.io.RealSenseSensorConfig(
 78                        json.load(ccf)),
 79                                            filename=filename)
 80            else:
 81                self.camera.init_sensor(filename=filename)
 82            self.camera.start_capture(start_record=False)
 83            self.rgbd_metadata = self.camera.get_metadata()
 84            self.status_message = f"Camera {self.rgbd_metadata.serial_number} opened."
 85
 86        log.info(self.rgbd_metadata)
 87
 88        # RGBD -> PCD
 89        self.extrinsics = o3d.core.Tensor.eye(4,
 90                                              dtype=o3d.core.Dtype.Float32,
 91                                              device=self.o3d_device)
 92        self.intrinsic_matrix = o3d.core.Tensor(
 93            self.rgbd_metadata.intrinsics.intrinsic_matrix,
 94            dtype=o3d.core.Dtype.Float32,
 95            device=self.o3d_device)
 96        self.depth_max = 3.0  # m
 97        self.pcd_stride = 2  # downsample point cloud, may increase frame rate
 98        self.flag_normals = False
 99        self.flag_save_rgbd = False
100        self.flag_save_pcd = False
101
102        self.pcd_frame = None
103        self.rgbd_frame = None
104        self.executor = ThreadPoolExecutor(max_workers=3,
105                                           thread_name_prefix='Capture-Save')
106        self.flag_exit = False
107
108    @property
109    def max_points(self):
110        """Max points in one frame for the camera or RGBD video resolution."""
111        return self.rgbd_metadata.width * self.rgbd_metadata.height
112
113    @property
114    def vfov(self):
115        """Camera or RGBD video vertical field of view."""
116        return np.rad2deg(2 * np.arctan(self.intrinsic_matrix[1, 2].item() /
117                                        self.intrinsic_matrix[1, 1].item()))
118
119    def run(self):
120        """Run pipeline."""
121        n_pts = 0
122        frame_id = 0
123        t1 = time.perf_counter()
124        if self.video:
125            self.rgbd_frame = self.video.next_frame()
126        else:
127            self.rgbd_frame = self.camera.capture_frame(
128                wait=True, align_depth_to_color=True)
129
130        pcd_errors = 0
131        while (not self.flag_exit and
132               (self.video is None or  # Camera
133                (self.video and not self.video.is_eof()))):  # Video
134            if self.video:
135                future_rgbd_frame = self.executor.submit(self.video.next_frame)
136            else:
137                future_rgbd_frame = self.executor.submit(
138                    self.camera.capture_frame,
139                    wait=True,
140                    align_depth_to_color=True)
141
142            if self.flag_save_pcd:
143                self.save_pcd()
144                self.flag_save_pcd = False
145            try:
146                self.rgbd_frame = self.rgbd_frame.to(self.o3d_device)
147                self.pcd_frame = o3d.t.geometry.PointCloud.create_from_rgbd_image(
148                    self.rgbd_frame, self.intrinsic_matrix, self.extrinsics,
149                    self.rgbd_metadata.depth_scale, self.depth_max,
150                    self.pcd_stride, self.flag_normals)
151                depth_in_color = self.rgbd_frame.depth.colorize_depth(
152                    self.rgbd_metadata.depth_scale, 0, self.depth_max)
153            except RuntimeError:
154                pcd_errors += 1
155
156            if self.pcd_frame.is_empty():
157                log.warning(f"No valid depth data in frame {frame_id})")
158                continue
159
160            n_pts += self.pcd_frame.point.positions.shape[0]
161            if frame_id % 60 == 0 and frame_id > 0:
162                t0, t1 = t1, time.perf_counter()
163                log.debug(f"\nframe_id = {frame_id}, \t {(t1-t0)*1000./60:0.2f}"
164                          f"ms/frame \t {(t1-t0)*1e9/n_pts} ms/Mp\t")
165                n_pts = 0
166            frame_elements = {
167                'color': self.rgbd_frame.color.cpu(),
168                'depth': depth_in_color.cpu(),
169                'pcd': self.pcd_frame.cpu(),
170                'status_message': self.status_message
171            }
172            self.update_view(frame_elements)
173
174            if self.flag_save_rgbd:
175                self.save_rgbd()
176                self.flag_save_rgbd = False
177            self.rgbd_frame = future_rgbd_frame.result()
178            with self.cv_capture:  # Wait for capture to be enabled
179                self.cv_capture.wait_for(
180                    predicate=lambda: self.flag_capture or self.flag_exit)
181            self.toggle_record()
182            frame_id += 1
183
184        if self.camera:
185            self.camera.stop_capture()
186        else:
187            self.video.close()
188        self.executor.shutdown()
189        log.debug(f"create_from_depth_image() errors = {pcd_errors}")
190
191    def toggle_record(self):
192        if self.camera is not None:
193            if self.flag_record and not self.recording:
194                self.camera.resume_record()
195                self.recording = True
196            elif not self.flag_record and self.recording:
197                self.camera.pause_record()
198                self.recording = False
199
200    def save_pcd(self):
201        """Save current point cloud."""
202        now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
203        filename = f"{self.rgbd_metadata.serial_number}_pcd_{now}.ply"
204        # Convert colors to uint8 for compatibility
205        self.pcd_frame.point.colors = (self.pcd_frame.point.colors * 255).to(
206            o3d.core.Dtype.UInt8)
207        self.executor.submit(o3d.t.io.write_point_cloud,
208                             filename,
209                             self.pcd_frame,
210                             write_ascii=False,
211                             compressed=True,
212                             print_progress=False)
213        self.status_message = f"Saving point cloud to {filename}."
214
215    def save_rgbd(self):
216        """Save current RGBD image pair."""
217        now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
218        filename = f"{self.rgbd_metadata.serial_number}_color_{now}.jpg"
219        self.executor.submit(o3d.t.io.write_image, filename,
220                             self.rgbd_frame.color)
221        filename = f"{self.rgbd_metadata.serial_number}_depth_{now}.png"
222        self.executor.submit(o3d.t.io.write_image, filename,
223                             self.rgbd_frame.depth)
224        self.status_message = (
225            f"Saving RGBD images to {filename[:-3]}.{{jpg,png}}.")
226
227
228class PipelineView:
229    """Controls display and user interface. All methods must run in the main thread."""
230
231    def __init__(self, vfov=60, max_pcd_vertices=1 << 20, **callbacks):
232        """Initialize.
233
234        Args:
235            vfov (float): Vertical field of view for the 3D scene.
236            max_pcd_vertices (int): Maximum point clud verties for which memory
237                is allocated.
238            callbacks (dict of kwargs): Callbacks provided by the controller
239                for various operations.
240        """
241
242        self.vfov = vfov
243        self.max_pcd_vertices = max_pcd_vertices
244
245        gui.Application.instance.initialize()
246        self.window = gui.Application.instance.create_window(
247            "Open3D || Online RGBD Video Processing", 1280, 960)
248        # Called on window layout (eg: resize)
249        self.window.set_on_layout(self.on_layout)
250        self.window.set_on_close(callbacks['on_window_close'])
251
252        self.pcd_material = o3d.visualization.rendering.MaterialRecord()
253        self.pcd_material.shader = "defaultLit"
254        # Set n_pixels displayed for each 3D point, accounting for HiDPI scaling
255        self.pcd_material.point_size = int(4 * self.window.scaling)
256
257        # 3D scene
258        self.pcdview = gui.SceneWidget()
259        self.window.add_child(self.pcdview)
260        self.pcdview.enable_scene_caching(
261            True)  # makes UI _much_ more responsive
262        self.pcdview.scene = rendering.Open3DScene(self.window.renderer)
263        self.pcdview.scene.set_background([1, 1, 1, 1])  # White background
264        self.pcdview.scene.set_lighting(
265            rendering.Open3DScene.LightingProfile.SOFT_SHADOWS, [0, -6, 0])
266        # Point cloud bounds, depends on the sensor range
267        self.pcd_bounds = o3d.geometry.AxisAlignedBoundingBox([-3, -3, 0],
268                                                              [3, 3, 6])
269        self.camera_view()  # Initially look from the camera
270        em = self.window.theme.font_size
271
272        # Options panel
273        self.panel = gui.Vert(em, gui.Margins(em, em, em, em))
274        self.panel.preferred_width = int(360 * self.window.scaling)
275        self.window.add_child(self.panel)
276        toggles = gui.Horiz(em)
277        self.panel.add_child(toggles)
278
279        toggle_capture = gui.ToggleSwitch("Capture / Play")
280        toggle_capture.is_on = False
281        toggle_capture.set_on_clicked(
282            callbacks['on_toggle_capture'])  # callback
283        toggles.add_child(toggle_capture)
284
285        self.flag_normals = False
286        self.toggle_normals = gui.ToggleSwitch("Colors / Normals")
287        self.toggle_normals.is_on = False
288        self.toggle_normals.set_on_clicked(
289            callbacks['on_toggle_normals'])  # callback
290        toggles.add_child(self.toggle_normals)
291
292        view_buttons = gui.Horiz(em)
293        self.panel.add_child(view_buttons)
294        view_buttons.add_stretch()  # for centering
295        camera_view = gui.Button("Camera view")
296        camera_view.set_on_clicked(self.camera_view)  # callback
297        view_buttons.add_child(camera_view)
298        birds_eye_view = gui.Button("Bird's eye view")
299        birds_eye_view.set_on_clicked(self.birds_eye_view)  # callback
300        view_buttons.add_child(birds_eye_view)
301        view_buttons.add_stretch()  # for centering
302
303        save_toggle = gui.Horiz(em)
304        self.panel.add_child(save_toggle)
305        save_toggle.add_child(gui.Label("Record / Save"))
306        self.toggle_record = None
307        if callbacks['on_toggle_record'] is not None:
308            save_toggle.add_fixed(1.5 * em)
309            self.toggle_record = gui.ToggleSwitch("Video")
310            self.toggle_record.is_on = False
311            self.toggle_record.set_on_clicked(callbacks['on_toggle_record'])
312            save_toggle.add_child(self.toggle_record)
313
314        save_buttons = gui.Horiz(em)
315        self.panel.add_child(save_buttons)
316        save_buttons.add_stretch()  # for centering
317        save_pcd = gui.Button("Save Point cloud")
318        save_pcd.set_on_clicked(callbacks['on_save_pcd'])
319        save_buttons.add_child(save_pcd)
320        save_rgbd = gui.Button("Save RGBD frame")
321        save_rgbd.set_on_clicked(callbacks['on_save_rgbd'])
322        save_buttons.add_child(save_rgbd)
323        save_buttons.add_stretch()  # for centering
324
325        self.video_size = (int(240 * self.window.scaling),
326                           int(320 * self.window.scaling), 3)
327        self.show_color = gui.CollapsableVert("Color image")
328        self.show_color.set_is_open(False)
329        self.panel.add_child(self.show_color)
330        self.color_video = gui.ImageWidget(
331            o3d.geometry.Image(np.zeros(self.video_size, dtype=np.uint8)))
332        self.show_color.add_child(self.color_video)
333        self.show_depth = gui.CollapsableVert("Depth image")
334        self.show_depth.set_is_open(False)
335        self.panel.add_child(self.show_depth)
336        self.depth_video = gui.ImageWidget(
337            o3d.geometry.Image(np.zeros(self.video_size, dtype=np.uint8)))
338        self.show_depth.add_child(self.depth_video)
339
340        self.status_message = gui.Label("")
341        self.panel.add_child(self.status_message)
342
343        self.flag_exit = False
344        self.flag_gui_init = False
345
346    def update(self, frame_elements):
347        """Update visualization with point cloud and images. Must run in main
348        thread since this makes GUI calls.
349
350        Args:
351            frame_elements: dict {element_type: geometry element}.
352                Dictionary of element types to geometry elements to be updated
353                in the GUI:
354                    'pcd': point cloud,
355                    'color': rgb image (3 channel, uint8),
356                    'depth': depth image (uint8),
357                    'status_message': message
358        """
359        if not self.flag_gui_init:
360            # Set dummy point cloud to allocate graphics memory
361            dummy_pcd = o3d.t.geometry.PointCloud({
362                'positions':
363                    o3d.core.Tensor.zeros((self.max_pcd_vertices, 3),
364                                          o3d.core.Dtype.Float32),
365                'colors':
366                    o3d.core.Tensor.zeros((self.max_pcd_vertices, 3),
367                                          o3d.core.Dtype.Float32),
368                'normals':
369                    o3d.core.Tensor.zeros((self.max_pcd_vertices, 3),
370                                          o3d.core.Dtype.Float32)
371            })
372            if self.pcdview.scene.has_geometry('pcd'):
373                self.pcdview.scene.remove_geometry('pcd')
374
375            self.pcd_material.shader = "normals" if self.flag_normals else "defaultLit"
376            self.pcdview.scene.add_geometry('pcd', dummy_pcd, self.pcd_material)
377            self.flag_gui_init = True
378
379        # TODO(ssheorey) Switch to update_geometry() after #3452 is fixed
380        if os.name == 'nt':
381            self.pcdview.scene.remove_geometry('pcd')
382            self.pcdview.scene.add_geometry('pcd', frame_elements['pcd'],
383                                            self.pcd_material)
384        else:
385            update_flags = (rendering.Scene.UPDATE_POINTS_FLAG |
386                            rendering.Scene.UPDATE_COLORS_FLAG |
387                            (rendering.Scene.UPDATE_NORMALS_FLAG
388                             if self.flag_normals else 0))
389            self.pcdview.scene.scene.update_geometry('pcd',
390                                                     frame_elements['pcd'],
391                                                     update_flags)
392
393        # Update color and depth images
394        # TODO(ssheorey) Remove CPU transfer after we have CUDA -> OpenGL bridge
395        if self.show_color.get_is_open() and 'color' in frame_elements:
396            sampling_ratio = self.video_size[1] / frame_elements['color'].columns
397            self.color_video.update_image(
398                frame_elements['color'].resize(sampling_ratio).cpu())
399        if self.show_depth.get_is_open() and 'depth' in frame_elements:
400            sampling_ratio = self.video_size[1] / frame_elements['depth'].columns
401            self.depth_video.update_image(
402                frame_elements['depth'].resize(sampling_ratio).cpu())
403
404        if 'status_message' in frame_elements:
405            self.status_message.text = frame_elements["status_message"]
406
407        self.pcdview.force_redraw()
408
409    def camera_view(self):
410        """Callback to reset point cloud view to the camera"""
411        self.pcdview.setup_camera(self.vfov, self.pcd_bounds, [0, 0, 0])
412        # Look at [0, 0, 1] from camera placed at [0, 0, 0] with Y axis
413        # pointing at [0, -1, 0]
414        self.pcdview.scene.camera.look_at([0, 0, 1], [0, 0, 0], [0, -1, 0])
415
416    def birds_eye_view(self):
417        """Callback to reset point cloud view to birds eye (overhead) view"""
418        self.pcdview.setup_camera(self.vfov, self.pcd_bounds, [0, 0, 0])
419        self.pcdview.scene.camera.look_at([0, 0, 1.5], [0, 3, 1.5], [0, -1, 0])
420
421    def on_layout(self, layout_context):
422        # The on_layout callback should set the frame (position + size) of every
423        # child correctly. After the callback is done the window will layout
424        # the grandchildren.
425        """Callback on window initialize / resize"""
426        frame = self.window.content_rect
427        self.pcdview.frame = frame
428        panel_size = self.panel.calc_preferred_size(layout_context,
429                                                    self.panel.Constraints())
430        self.panel.frame = gui.Rect(frame.get_right() - panel_size.width,
431                                    frame.y, panel_size.width,
432                                    panel_size.height)
433
434
435class PipelineController:
436    """Entry point for the app. Controls the PipelineModel object for IO and
437    processing  and the PipelineView object for display and UI. All methods
438    operate on the main thread.
439    """
440
441    def __init__(self, camera_config_file=None, rgbd_video=None, device=None):
442        """Initialize.
443
444        Args:
445            camera_config_file (str): Camera configuration json file.
446            rgbd_video (str): RS bag file containing the RGBD video. If this is
447                provided, connected cameras are ignored.
448            device (str): Compute device (e.g.: 'cpu:0' or 'cuda:0').
449        """
450        self.pipeline_model = PipelineModel(self.update_view,
451                                            camera_config_file, rgbd_video,
452                                            device)
453
454        self.pipeline_view = PipelineView(
455            1.25 * self.pipeline_model.vfov,
456            self.pipeline_model.max_points,
457            on_window_close=self.on_window_close,
458            on_toggle_capture=self.on_toggle_capture,
459            on_save_pcd=self.on_save_pcd,
460            on_save_rgbd=self.on_save_rgbd,
461            on_toggle_record=self.on_toggle_record
462            if rgbd_video is None else None,
463            on_toggle_normals=self.on_toggle_normals)
464
465        threading.Thread(name='PipelineModel',
466                         target=self.pipeline_model.run).start()
467        gui.Application.instance.run()
468
469    def update_view(self, frame_elements):
470        """Updates view with new data. May be called from any thread.
471
472        Args:
473            frame_elements (dict): Display elements (point cloud and images)
474                from the new frame to be shown.
475        """
476        gui.Application.instance.post_to_main_thread(
477            self.pipeline_view.window,
478            lambda: self.pipeline_view.update(frame_elements))
479
480    def on_toggle_capture(self, is_enabled):
481        """Callback to toggle capture."""
482        self.pipeline_model.flag_capture = is_enabled
483        if not is_enabled:
484            self.on_toggle_record(False)
485            if self.pipeline_view.toggle_record is not None:
486                self.pipeline_view.toggle_record.is_on = False
487        else:
488            with self.pipeline_model.cv_capture:
489                self.pipeline_model.cv_capture.notify()
490
491    def on_toggle_record(self, is_enabled):
492        """Callback to toggle recording RGBD video."""
493        self.pipeline_model.flag_record = is_enabled
494
495    def on_toggle_normals(self, is_enabled):
496        """Callback to toggle display of normals"""
497        self.pipeline_model.flag_normals = is_enabled
498        self.pipeline_view.flag_normals = is_enabled
499        self.pipeline_view.flag_gui_init = False
500
501    def on_window_close(self):
502        """Callback when the user closes the application window."""
503        self.pipeline_model.flag_exit = True
504        with self.pipeline_model.cv_capture:
505            self.pipeline_model.cv_capture.notify_all()
506        return True  # OK to close window
507
508    def on_save_pcd(self):
509        """Callback to save current point cloud."""
510        self.pipeline_model.flag_save_pcd = True
511
512    def on_save_rgbd(self):
513        """Callback to save current RGBD image pair."""
514        self.pipeline_model.flag_save_rgbd = True
515
516
517if __name__ == "__main__":
518
519    log.basicConfig(level=log.INFO)
520    parser = argparse.ArgumentParser(
521        description=__doc__,
522        formatter_class=argparse.RawDescriptionHelpFormatter)
523    parser.add_argument('--camera-config',
524                        help='RGBD camera configuration JSON file')
525    parser.add_argument('--rgbd-video', help='RGBD video file (RealSense bag)')
526    parser.add_argument('--device',
527                        help='Device to run computations. e.g. cpu:0 or cuda:0 '
528                        'Default is CUDA GPU if available, else CPU.')
529
530    args = parser.parse_args()
531    if args.camera_config and args.rgbd_video:
532        log.critical(
533            "Please provide only one of --camera-config and --rgbd-video arguments"
534        )
535    else:
536        PipelineController(args.camera_config, args.rgbd_video, args.device)

remove_geometry.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import open3d as o3d
 9import numpy as np
10import time
11import copy
12
13
14def visualize_non_blocking(vis, pcds):
15    for pcd in pcds:
16        vis.update_geometry(pcd)
17    vis.poll_events()
18    vis.update_renderer()
19
20
21pcd_data = o3d.data.PCDPointCloud()
22pcd_orig = o3d.io.read_point_cloud(pcd_data.path)
23flip_transform = [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]
24pcd_orig.transform(flip_transform)
25n_pcd = 5
26pcds = []
27for i in range(n_pcd):
28    pcds.append(copy.deepcopy(pcd_orig))
29    trans = np.identity(4)
30    trans[:3, 3] = [3 * i, 0, 0]
31    pcds[i].transform(trans)
32
33vis = o3d.visualization.Visualizer()
34vis.create_window()
35start_time = time.time()
36added = [False] * n_pcd
37
38curr_sec = int(time.time() - start_time)
39prev_sec = curr_sec - 1
40
41while True:
42    curr_sec = int(time.time() - start_time)
43    if curr_sec - prev_sec == 1:
44        prev_sec = curr_sec
45
46        for i in range(n_pcd):
47            if curr_sec % (n_pcd * 2) == i and not added[i]:
48                vis.add_geometry(pcds[i])
49                added[i] = True
50                print("Adding %d" % i)
51            if curr_sec % (n_pcd * 2) == (i + n_pcd) and added[i]:
52                vis.remove_geometry(pcds[i])
53                added[i] = False
54                print("Removing %d" % i)
55
56    visualize_non_blocking(vis, pcds)

render_to_image.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import open3d as o3d
 9import open3d.visualization.rendering as rendering
10
11
12def main():
13    render = rendering.OffscreenRenderer(640, 480)
14
15    yellow = rendering.MaterialRecord()
16    yellow.base_color = [1.0, 0.75, 0.0, 1.0]
17    yellow.shader = "defaultLit"
18
19    green = rendering.MaterialRecord()
20    green.base_color = [0.0, 0.5, 0.0, 1.0]
21    green.shader = "defaultLit"
22
23    grey = rendering.MaterialRecord()
24    grey.base_color = [0.7, 0.7, 0.7, 1.0]
25    grey.shader = "defaultLit"
26
27    white = rendering.MaterialRecord()
28    white.base_color = [1.0, 1.0, 1.0, 1.0]
29    white.shader = "defaultLit"
30
31    cyl = o3d.geometry.TriangleMesh.create_cylinder(.05, 3)
32    cyl.compute_vertex_normals()
33    cyl.translate([-2, 0, 1.5])
34    sphere = o3d.geometry.TriangleMesh.create_sphere(.2)
35    sphere.compute_vertex_normals()
36    sphere.translate([-2, 0, 3])
37
38    box = o3d.geometry.TriangleMesh.create_box(2, 2, 1)
39    box.compute_vertex_normals()
40    box.translate([-1, -1, 0])
41    solid = o3d.geometry.TriangleMesh.create_icosahedron(0.5)
42    solid.compute_triangle_normals()
43    solid.compute_vertex_normals()
44    solid.translate([0, 0, 1.75])
45
46    render.scene.add_geometry("cyl", cyl, green)
47    render.scene.add_geometry("sphere", sphere, yellow)
48    render.scene.add_geometry("box", box, grey)
49    render.scene.add_geometry("solid", solid, white)
50    render.setup_camera(60.0, [0, 0, 0], [0, 10, 0], [0, 0, 1])
51    render.scene.scene.set_sun_light([0.707, 0.0, -.707], [1.0, 1.0, 1.0],
52                                     75000)
53    render.scene.scene.enable_sun_light(True)
54    render.scene.show_axes(True)
55
56    img = render.render_to_image()
57    print("Saving image at test.png")
58    o3d.io.write_image("test.png", img, 9)
59
60    render.setup_camera(60.0, [0, 0, 0], [-10, 0, 0], [0, 0, 1])
61    img = render.render_to_image()
62    print("Saving image at test2.png")
63    o3d.io.write_image("test2.png", img, 9)
64
65
66if __name__ == "__main__":
67    main()

tensorboard_pytorch.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7import copy
  8import os
  9import sys
 10import numpy as np
 11import open3d as o3d
 12# pylint: disable-next=unused-import
 13from open3d.visualization.tensorboard_plugin.util import to_dict_batch
 14from torch.utils.tensorboard import SummaryWriter
 15
 16BASE_LOGDIR = "demo_logs/pytorch/"
 17MODEL_DIR = os.path.join(
 18    os.path.dirname(os.path.dirname(os.path.dirname(
 19        os.path.realpath(__file__)))), "test_data", "monkey")
 20
 21
 22def small_scale(run_name="small_scale"):
 23    """Basic demo with cube and cylinder with normals and colors.
 24    """
 25    logdir = os.path.join(BASE_LOGDIR, run_name)
 26    writer = SummaryWriter(logdir)
 27
 28    cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4, create_uv_map=True)
 29    cube.compute_vertex_normals()
 30    cylinder = o3d.geometry.TriangleMesh.create_cylinder(radius=1.0,
 31                                                         height=2.0,
 32                                                         resolution=20,
 33                                                         split=4,
 34                                                         create_uv_map=True)
 35    cylinder.compute_vertex_normals()
 36    colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
 37    for step in range(3):
 38        cube.paint_uniform_color(colors[step])
 39        writer.add_3d('cube', to_dict_batch([cube]), step=step)
 40        cylinder.paint_uniform_color(colors[step])
 41        writer.add_3d('cylinder', to_dict_batch([cylinder]), step=step)
 42
 43
 44def property_reference(run_name="property_reference"):
 45    """Produces identical visualization to small_scale, but does not store
 46    repeated properties of ``vertex_positions`` and ``vertex_normals``.
 47    """
 48    logdir = os.path.join(BASE_LOGDIR, run_name)
 49    writer = SummaryWriter(logdir)
 50
 51    cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4, create_uv_map=True)
 52    cube.compute_vertex_normals()
 53    cylinder = o3d.geometry.TriangleMesh.create_cylinder(radius=1.0,
 54                                                         height=2.0,
 55                                                         resolution=20,
 56                                                         split=4,
 57                                                         create_uv_map=True)
 58    cylinder.compute_vertex_normals()
 59    colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
 60    for step in range(3):
 61        cube.paint_uniform_color(colors[step])
 62        cube_summary = to_dict_batch([cube])
 63        if step > 0:
 64            cube_summary['vertex_positions'] = 0
 65            cube_summary['vertex_normals'] = 0
 66        writer.add_3d('cube', cube_summary, step=step)
 67        cylinder.paint_uniform_color(colors[step])
 68        cylinder_summary = to_dict_batch([cylinder])
 69        if step > 0:
 70            cylinder_summary['vertex_positions'] = 0
 71            cylinder_summary['vertex_normals'] = 0
 72        writer.add_3d('cylinder', cylinder_summary, step=step)
 73
 74
 75def large_scale(n_steps=16,
 76                batch_size=1,
 77                base_resolution=200,
 78                run_name="large_scale"):
 79    """Generate a large scale summary. Geometry resolution increases linearly
 80    with step. Each element in a batch is painted a different color.
 81    """
 82    logdir = os.path.join(BASE_LOGDIR, run_name)
 83    writer = SummaryWriter(logdir)
 84    colors = []
 85    for k in range(batch_size):
 86        t = k * np.pi / batch_size
 87        colors.append(((1 + np.sin(t)) / 2, (1 + np.cos(t)) / 2, t / np.pi))
 88    for step in range(n_steps):
 89        resolution = base_resolution * (step + 1)
 90        cylinder_list = []
 91        mobius_list = []
 92        cylinder = o3d.geometry.TriangleMesh.create_cylinder(
 93            radius=1.0, height=2.0, resolution=resolution, split=4)
 94        cylinder.compute_vertex_normals()
 95        mobius = o3d.geometry.TriangleMesh.create_mobius(
 96            length_split=int(3.5 * resolution),
 97            width_split=int(0.75 * resolution),
 98            twists=1,
 99            raidus=1,
100            flatness=1,
101            width=1,
102            scale=1)
103        mobius.compute_vertex_normals()
104        for b in range(batch_size):
105            cylinder_list.append(copy.deepcopy(cylinder))
106            cylinder_list[b].paint_uniform_color(colors[b])
107            mobius_list.append(copy.deepcopy(mobius))
108            mobius_list[b].paint_uniform_color(colors[b])
109        writer.add_3d('cylinder',
110                      to_dict_batch(cylinder_list),
111                      step=step,
112                      max_outputs=batch_size)
113        writer.add_3d('mobius',
114                      to_dict_batch(mobius_list),
115                      step=step,
116                      max_outputs=batch_size)
117
118
119def with_material(model_dir=MODEL_DIR):
120    """Read an obj model from a directory and write as a TensorBoard summary.
121    """
122    model_name = os.path.basename(model_dir)
123    logdir = os.path.join(BASE_LOGDIR, model_name)
124    model_path = os.path.join(model_dir, model_name + ".obj")
125    model = o3d.t.geometry.TriangleMesh.from_legacy(
126        o3d.io.read_triangle_mesh(model_path))
127    summary_3d = {
128        "vertex_positions": model.vertex.positions,
129        "vertex_normals": model.vertex.normals,
130        "triangle_texture_uvs": model.triangle["texture_uvs"],
131        "triangle_indices": model.triangle.indices,
132        "material_name": "defaultLit"
133    }
134    names_to_o3dprop = {"ao": "ambient_occlusion"}
135
136    for texture in ("albedo", "normal", "ao", "metallic", "roughness"):
137        texture_file = os.path.join(model_dir, texture + ".png")
138        if os.path.exists(texture_file):
139            texture = names_to_o3dprop.get(texture, texture)
140            summary_3d.update({
141                ("material_texture_map_" + texture):
142                    o3d.t.io.read_image(texture_file)
143            })
144            if texture == "metallic":
145                summary_3d.update(material_scalar_metallic=1.0)
146
147    writer = SummaryWriter(logdir)
148    writer.add_3d(model_name, summary_3d, step=0)
149
150
151def demo_scene():
152    """Write the demo_scene.py example showing rich PBR materials as a summary
153    """
154    import demo_scene
155    geoms = demo_scene.create_scene()
156    writer = SummaryWriter(os.path.join(BASE_LOGDIR, 'demo_scene'))
157    for geom_data in geoms:
158        geom = geom_data["geometry"]
159        summary_3d = {}
160        for key, tensor in geom.vertex.items():
161            summary_3d["vertex_" + key] = tensor
162        for key, tensor in geom.triangle.items():
163            summary_3d["triangle_" + key] = tensor
164        if geom.has_valid_material():
165            summary_3d["material_name"] = geom.material.material_name
166            for key, value in geom.material.scalar_properties.items():
167                summary_3d["material_scalar_" + key] = value
168            for key, value in geom.material.vector_properties.items():
169                summary_3d["material_vector_" + key] = value
170            for key, value in geom.material.texture_maps.items():
171                summary_3d["material_texture_map_" + key] = value
172        writer.add_3d(geom_data["name"], summary_3d, step=0)
173
174
175if __name__ == "__main__":
176
177    examples = ('small_scale', 'large_scale', 'property_reference',
178                'with_material', 'demo_scene')
179    selected = tuple(eg for eg in sys.argv[1:] if eg in examples)
180    if len(selected) == 0:
181        print(f'Usage: python {__file__} EXAMPLE...')
182        print(f'  where EXAMPLE are from {examples}')
183        selected = ('property_reference', 'with_material')
184
185    for eg in selected:
186        locals()[eg]()
187
188    print(f"Run 'tensorboard --logdir {BASE_LOGDIR}' to visualize the 3D "
189          "summary.")

tensorboard_tensorflow.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7import copy
  8import os
  9import sys
 10import numpy as np
 11import open3d as o3d
 12from open3d.visualization.tensorboard_plugin import summary
 13from open3d.visualization.tensorboard_plugin.util import to_dict_batch
 14import tensorflow as tf
 15
 16BASE_LOGDIR = "demo_logs/tf/"
 17MODEL_DIR = os.path.join(
 18    os.path.dirname(os.path.dirname(os.path.dirname(
 19        os.path.realpath(__file__)))), "test_data", "monkey")
 20
 21
 22def small_scale(run_name="small_scale"):
 23    """Basic demo with cube and cylinder with normals and colors.
 24    """
 25    logdir = os.path.join(BASE_LOGDIR, run_name)
 26    writer = tf.summary.create_file_writer(logdir)
 27
 28    cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4, create_uv_map=True)
 29    cube.compute_vertex_normals()
 30    cylinder = o3d.geometry.TriangleMesh.create_cylinder(radius=1.0,
 31                                                         height=2.0,
 32                                                         resolution=20,
 33                                                         split=4,
 34                                                         create_uv_map=True)
 35    cylinder.compute_vertex_normals()
 36    colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
 37    with writer.as_default():
 38        for step in range(3):
 39            cube.paint_uniform_color(colors[step])
 40            summary.add_3d('cube',
 41                           to_dict_batch([cube]),
 42                           step=step,
 43                           logdir=logdir)
 44            cylinder.paint_uniform_color(colors[step])
 45            summary.add_3d('cylinder',
 46                           to_dict_batch([cylinder]),
 47                           step=step,
 48                           logdir=logdir)
 49
 50
 51def property_reference(run_name="property_reference"):
 52    """Produces identical visualization to small_scale, but does not store
 53    repeated properties of ``vertex_positions`` and ``vertex_normals``.
 54    """
 55    logdir = os.path.join(BASE_LOGDIR, run_name)
 56    writer = tf.summary.create_file_writer(logdir)
 57
 58    cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4, create_uv_map=True)
 59    cube.compute_vertex_normals()
 60    cylinder = o3d.geometry.TriangleMesh.create_cylinder(radius=1.0,
 61                                                         height=2.0,
 62                                                         resolution=20,
 63                                                         split=4,
 64                                                         create_uv_map=True)
 65    cylinder.compute_vertex_normals()
 66    colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
 67    with writer.as_default():
 68        for step in range(3):
 69            cube.paint_uniform_color(colors[step])
 70            cube_summary = to_dict_batch([cube])
 71            if step > 0:
 72                cube_summary['vertex_positions'] = 0
 73                cube_summary['vertex_normals'] = 0
 74            summary.add_3d('cube', cube_summary, step=step, logdir=logdir)
 75            cylinder.paint_uniform_color(colors[step])
 76            cylinder_summary = to_dict_batch([cylinder])
 77            if step > 0:
 78                cylinder_summary['vertex_positions'] = 0
 79                cylinder_summary['vertex_normals'] = 0
 80            summary.add_3d('cylinder',
 81                           cylinder_summary,
 82                           step=step,
 83                           logdir=logdir)
 84
 85
 86def large_scale(n_steps=16,
 87                batch_size=1,
 88                base_resolution=200,
 89                run_name="large_scale"):
 90    """Generate a large scale summary. Geometry resolution increases linearly
 91    with step. Each element in a batch is painted a different color.
 92    """
 93    logdir = os.path.join(BASE_LOGDIR, run_name)
 94    writer = tf.summary.create_file_writer(logdir)
 95    colors = []
 96    for k in range(batch_size):
 97        t = k * np.pi / batch_size
 98        colors.append(((1 + np.sin(t)) / 2, (1 + np.cos(t)) / 2, t / np.pi))
 99    with writer.as_default():
100        for step in range(n_steps):
101            resolution = base_resolution * (step + 1)
102            cylinder_list = []
103            mobius_list = []
104            cylinder = o3d.geometry.TriangleMesh.create_cylinder(
105                radius=1.0, height=2.0, resolution=resolution, split=4)
106            cylinder.compute_vertex_normals()
107            mobius = o3d.geometry.TriangleMesh.create_mobius(
108                length_split=int(3.5 * resolution),
109                width_split=int(0.75 * resolution),
110                twists=1,
111                raidus=1,
112                flatness=1,
113                width=1,
114                scale=1)
115            mobius.compute_vertex_normals()
116            for b in range(batch_size):
117                cylinder_list.append(copy.deepcopy(cylinder))
118                cylinder_list[b].paint_uniform_color(colors[b])
119                mobius_list.append(copy.deepcopy(mobius))
120                mobius_list[b].paint_uniform_color(colors[b])
121            summary.add_3d('cylinder',
122                           to_dict_batch(cylinder_list),
123                           step=step,
124                           logdir=logdir,
125                           max_outputs=batch_size)
126            summary.add_3d('mobius',
127                           to_dict_batch(mobius_list),
128                           step=step,
129                           logdir=logdir,
130                           max_outputs=batch_size)
131
132
133def with_material(model_dir=MODEL_DIR):
134    """Read an obj model from a directory and write as a TensorBoard summary.
135    """
136    model_name = os.path.basename(model_dir)
137    logdir = os.path.join(BASE_LOGDIR, model_name)
138    model_path = os.path.join(model_dir, model_name + ".obj")
139    model = o3d.t.geometry.TriangleMesh.from_legacy(
140        o3d.io.read_triangle_mesh(model_path))
141    summary_3d = {
142        "vertex_positions": model.vertex.positions,
143        "vertex_normals": model.vertex.normals,
144        "triangle_texture_uvs": model.triangle["texture_uvs"],
145        "triangle_indices": model.triangle.indices,
146        "material_name": "defaultLit"
147    }
148    names_to_o3dprop = {"ao": "ambient_occlusion"}
149
150    for texture in ("albedo", "normal", "ao", "metallic", "roughness"):
151        texture_file = os.path.join(model_dir, texture + ".png")
152        if os.path.exists(texture_file):
153            texture = names_to_o3dprop.get(texture, texture)
154            summary_3d.update({
155                ("material_texture_map_" + texture):
156                    o3d.t.io.read_image(texture_file)
157            })
158            if texture == "metallic":
159                summary_3d.update(material_scalar_metallic=1.0)
160
161    writer = tf.summary.create_file_writer(logdir)
162    with writer.as_default():
163        summary.add_3d(model_name, summary_3d, step=0, logdir=logdir)
164
165
166def demo_scene():
167    """Write the demo_scene.py example showing rich PBR materials as a summary.
168    """
169    import demo_scene
170    geoms = demo_scene.create_scene()
171    logdir = os.path.join(BASE_LOGDIR, 'demo_scene')
172    writer = tf.summary.create_file_writer(logdir)
173    for geom_data in geoms:
174        geom = geom_data["geometry"]
175        summary_3d = {}
176        for key, tensor in geom.vertex.items():
177            summary_3d["vertex_" + key] = tensor
178        for key, tensor in geom.triangle.items():
179            summary_3d["triangle_" + key] = tensor
180        if geom.has_valid_material():
181            summary_3d["material_name"] = geom.material.material_name
182            for key, value in geom.material.scalar_properties.items():
183                summary_3d["material_scalar_" + key] = value
184            for key, value in geom.material.vector_properties.items():
185                summary_3d["material_vector_" + key] = value
186            for key, value in geom.material.texture_maps.items():
187                summary_3d["material_texture_map_" + key] = value
188        with writer.as_default():
189            summary.add_3d(geom_data["name"], summary_3d, step=0, logdir=logdir)
190
191
192if __name__ == "__main__":
193
194    examples = ('small_scale', 'large_scale', 'property_reference',
195                'with_material', 'demo_scene')
196    selected = tuple(eg for eg in sys.argv[1:] if eg in examples)
197    if len(selected) == 0:
198        print(f'Usage: python {__file__} EXAMPLE...')
199        print(f'  where EXAMPLE are from {examples}')
200        selected = ('property_reference', 'with_material')
201
202    for eg in selected:
203        locals()[eg]()
204
205    print(f"Run 'tensorboard --logdir {BASE_LOGDIR}' to visualize the 3D "
206          "summary.")

text3d.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import numpy as np
 9import open3d as o3d
10import open3d.visualization.gui as gui
11import open3d.visualization.rendering as rendering
12
13
14def make_point_cloud(npts, center, radius):
15    pts = np.random.uniform(-radius, radius, size=[npts, 3]) + center
16    cloud = o3d.geometry.PointCloud()
17    cloud.points = o3d.utility.Vector3dVector(pts)
18    colors = np.random.uniform(0.0, 1.0, size=[npts, 3])
19    cloud.colors = o3d.utility.Vector3dVector(colors)
20    return cloud
21
22
23def high_level():
24    app = gui.Application.instance
25    app.initialize()
26
27    points = make_point_cloud(100, (0, 0, 0), 1.0)
28
29    vis = o3d.visualization.O3DVisualizer("Open3D - 3D Text", 1024, 768)
30    vis.show_settings = True
31    vis.add_geometry("Points", points)
32    for idx in range(0, len(points.points)):
33        vis.add_3d_label(points.points[idx], "{}".format(idx))
34    vis.reset_camera_to_default()
35
36    app.add_window(vis)
37    app.run()
38
39
40def low_level():
41    app = gui.Application.instance
42    app.initialize()
43
44    points = make_point_cloud(100, (0, 0, 0), 1.0)
45
46    w = app.create_window("Open3D - 3D Text", 1024, 768)
47    widget3d = gui.SceneWidget()
48    widget3d.scene = rendering.Open3DScene(w.renderer)
49    mat = rendering.MaterialRecord()
50    mat.shader = "defaultUnlit"
51    mat.point_size = 5 * w.scaling
52    widget3d.scene.add_geometry("Points", points, mat)
53    for idx in range(0, len(points.points)):
54        l = widget3d.add_3d_label(points.points[idx], "{}".format(idx))
55        l.color = gui.Color(points.colors[idx][0], points.colors[idx][1],
56                            points.colors[idx][2])
57        l.scale = np.random.uniform(0.5, 3.0)
58    bbox = widget3d.scene.bounding_box
59    widget3d.setup_camera(60.0, bbox, bbox.get_center())
60    w.add_child(widget3d)
61
62    app.run()
63
64
65if __name__ == "__main__":
66    high_level()
67    low_level()

textured_mesh.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import sys
 9import os
10import open3d as o3d
11
12
13def main():
14    if len(sys.argv) < 2:
15        print("""Usage: textured-mesh.py [model directory]
16    This example will load [model directory].obj plus any of albedo, normal,
17    ao, metallic and roughness textures present. The textures should be named
18    albedo.png, normal.png, ao.png, metallic.png and roughness.png
19    respectively.""")
20        sys.exit()
21
22    model_dir = os.path.normpath(os.path.realpath(sys.argv[1]))
23    model_name = os.path.join(model_dir, os.path.basename(model_dir) + ".obj")
24    mesh = o3d.t.geometry.TriangleMesh.from_legacy(
25        o3d.io.read_triangle_mesh(model_name))
26    material = mesh.material
27    material.material_name = "defaultLit"
28
29    names_to_o3dprop = {"ao": "ambient_occlusion"}
30    for texture in ("albedo", "normal", "ao", "metallic", "roughness"):
31        texture_file = os.path.join(model_dir, texture + ".png")
32        if os.path.exists(texture_file):
33            texture = names_to_o3dprop.get(texture, texture)
34            material.texture_maps[texture] = o3d.t.io.read_image(texture_file)
35    if "metallic" in material.texture_maps:
36        material.scalar_properties["metallic"] = 1.0
37
38    o3d.visualization.draw(mesh, title=model_name)
39
40
41if __name__ == "__main__":
42    main()

textured_model.py#

 1# ----------------------------------------------------------------------------
 2# -                        Open3D: www.open3d.org                            -
 3# ----------------------------------------------------------------------------
 4# Copyright (c) 2018-2023 www.open3d.org
 5# SPDX-License-Identifier: MIT
 6# ----------------------------------------------------------------------------
 7
 8import sys
 9import os
10import open3d as o3d
11
12
13def main():
14    if len(sys.argv) < 2:
15        print("""Usage: texture-model.py [model directory]
16    This example will load [model directory].obj plus any of albedo, normal,
17    ao, metallic and roughness textures present.""")
18        sys.exit()
19
20    model_dir = sys.argv[1]
21    model_name = os.path.join(model_dir, os.path.basename(model_dir) + ".obj")
22    model = o3d.io.read_triangle_mesh(model_name)
23    material = o3d.visualization.rendering.MaterialRecord()
24    material.shader = "defaultLit"
25
26    albedo_name = os.path.join(model_dir, "albedo.png")
27    normal_name = os.path.join(model_dir, "normal.png")
28    ao_name = os.path.join(model_dir, "ao.png")
29    metallic_name = os.path.join(model_dir, "metallic.png")
30    roughness_name = os.path.join(model_dir, "roughness.png")
31    if os.path.exists(albedo_name):
32        material.albedo_img = o3d.io.read_image(albedo_name)
33    if os.path.exists(normal_name):
34        material.normal_img = o3d.io.read_image(normal_name)
35    if os.path.exists(ao_name):
36        material.ao_img = o3d.io.read_image(ao_name)
37    if os.path.exists(metallic_name):
38        material.base_metallic = 1.0
39        material.metallic_img = o3d.io.read_image(metallic_name)
40    if os.path.exists(roughness_name):
41        material.roughness_img = o3d.io.read_image(roughness_name)
42
43    o3d.visualization.draw([{
44        "name": "cube",
45        "geometry": model,
46        "material": material
47    }])
48
49
50if __name__ == "__main__":
51    main()

to_mitsuba.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import open3d as o3d
  9import mitsuba as mi
 10
 11
 12def render_mesh(mesh, mesh_center):
 13    scene = mi.load_dict({
 14        'type': 'scene',
 15        'integrator': {
 16            'type': 'path'
 17        },
 18        'light': {
 19            'type': 'constant',
 20            'radiance': {
 21                'type': 'rgb',
 22                'value': 1.0
 23            }
 24            # NOTE: For better results comment out the constant emitter above
 25            # and uncomment out the lines below changing the filename to an HDRI
 26            # envmap you have.
 27            # 'type': 'envmap',
 28            # 'filename': '/home/renes/Downloads/solitude_interior_4k.exr'
 29        },
 30        'sensor': {
 31            'type':
 32                'perspective',
 33            'focal_length':
 34                '50mm',
 35            'to_world':
 36                mi.ScalarTransform4f.look_at(origin=[0, 0, 5],
 37                                             target=mesh_center,
 38                                             up=[0, 1, 0]),
 39            'thefilm': {
 40                'type': 'hdrfilm',
 41                'width': 1024,
 42                'height': 768,
 43            },
 44            'thesampler': {
 45                'type': 'multijitter',
 46                'sample_count': 64,
 47            },
 48        },
 49        'themesh': mesh,
 50    })
 51
 52    img = mi.render(scene, spp=256)
 53    return img
 54
 55
 56# Default to LLVM variant which should be available on all
 57# platforms. If you have a system with a CUDA device then comment out LLVM
 58# variant and uncomment cuda variant
 59mi.set_variant('llvm_ad_rgb')
 60# mi.set_variant('cuda_ad_rgb')
 61
 62# Load mesh and maps using Open3D
 63dataset = o3d.data.MonkeyModel()
 64mesh = o3d.t.io.read_triangle_mesh(dataset.path)
 65mesh_center = mesh.get_axis_aligned_bounding_box().get_center()
 66mesh.material.set_default_properties()
 67mesh.material.material_name = 'defaultLit'
 68mesh.material.scalar_properties['metallic'] = 1.0
 69mesh.material.texture_maps['albedo'] = o3d.t.io.read_image(
 70    dataset.path_map['albedo'])
 71mesh.material.texture_maps['roughness'] = o3d.t.io.read_image(
 72    dataset.path_map['roughness'])
 73mesh.material.texture_maps['metallic'] = o3d.t.io.read_image(
 74    dataset.path_map['metallic'])
 75
 76print('Render mesh with material converted to Mitsuba principled BSDF')
 77mi_mesh = mesh.to_mitsuba('monkey')
 78img = render_mesh(mi_mesh, mesh_center.numpy())
 79mi.Bitmap(img).write('test.exr')
 80
 81print('Render mesh with normal-mapped prnincipled BSDF')
 82mesh.material.texture_maps['normal'] = o3d.t.io.read_image(
 83    dataset.path_map['normal'])
 84mi_mesh = mesh.to_mitsuba('monkey')
 85img = render_mesh(mi_mesh, mesh_center.numpy())
 86mi.Bitmap(img).write('test2.exr')
 87
 88print('Rendering mesh with Mitsuba smooth plastic BSDF')
 89bsdf_smooth_plastic = mi.load_dict({
 90    'type': 'plastic',
 91    'diffuse_reflectance': {
 92        'type': 'rgb',
 93        'value': [0.1, 0.27, 0.36]
 94    },
 95    'int_ior': 1.9
 96})
 97mi_mesh = mesh.to_mitsuba('monkey', bsdf=bsdf_smooth_plastic)
 98img = render_mesh(mi_mesh, mesh_center.numpy())
 99mi.Bitmap(img).write('test3.exr')
100
101# Render with Open3D
102o3d.visualization.draw(mesh)

video.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import numpy as np
  9import open3d as o3d
 10import open3d.visualization.gui as gui
 11import open3d.visualization.rendering as rendering
 12import time
 13import threading
 14
 15
 16def rescale_greyscale(img):
 17    data = np.asarray(img)
 18    assert (len(data.shape) == 2)  # requires 1 channel image
 19    dataFloat = data.astype(np.float64)
 20    max_val = dataFloat.max()
 21    # We don't currently support 16-bit images, so convert to 8-bit
 22    dataFloat *= 255.0 / max_val
 23    data8 = dataFloat.astype(np.uint8)
 24    return o3d.geometry.Image(data8)
 25
 26
 27class VideoWindow:
 28
 29    def __init__(self):
 30        self.rgb_images = []
 31        rgbd_data = o3d.data.SampleRedwoodRGBDImages()
 32        for path in rgbd_data.color_paths:
 33            img = o3d.io.read_image(path)
 34            self.rgb_images.append(img)
 35        self.depth_images = []
 36        for path in rgbd_data.depth_paths:
 37            img = o3d.io.read_image(path)
 38            # The images are pretty dark, so rescale them so that it is
 39            # obvious that this is a depth image, for the sake of the example
 40            img = rescale_greyscale(img)
 41            self.depth_images.append(img)
 42        assert (len(self.rgb_images) == len(self.depth_images))
 43
 44        self.window = gui.Application.instance.create_window(
 45            "Open3D - Video Example", 1000, 500)
 46        self.window.set_on_layout(self._on_layout)
 47        self.window.set_on_close(self._on_close)
 48
 49        self.widget3d = gui.SceneWidget()
 50        self.widget3d.scene = rendering.Open3DScene(self.window.renderer)
 51        self.window.add_child(self.widget3d)
 52
 53        lit = rendering.MaterialRecord()
 54        lit.shader = "defaultLit"
 55        tet = o3d.geometry.TriangleMesh.create_tetrahedron()
 56        tet.compute_vertex_normals()
 57        tet.paint_uniform_color([0.5, 0.75, 1.0])
 58        self.widget3d.scene.add_geometry("tetrahedron", tet, lit)
 59        bounds = self.widget3d.scene.bounding_box
 60        self.widget3d.setup_camera(60.0, bounds, bounds.get_center())
 61        self.widget3d.scene.show_axes(True)
 62
 63        em = self.window.theme.font_size
 64        margin = 0.5 * em
 65        self.panel = gui.Vert(0.5 * em, gui.Margins(margin))
 66        self.panel.add_child(gui.Label("Color image"))
 67        self.rgb_widget = gui.ImageWidget(self.rgb_images[0])
 68        self.panel.add_child(self.rgb_widget)
 69        self.panel.add_child(gui.Label("Depth image (normalized)"))
 70        self.depth_widget = gui.ImageWidget(self.depth_images[0])
 71        self.panel.add_child(self.depth_widget)
 72        self.window.add_child(self.panel)
 73
 74        self.is_done = False
 75        threading.Thread(target=self._update_thread).start()
 76
 77    def _on_layout(self, layout_context):
 78        contentRect = self.window.content_rect
 79        panel_width = 15 * layout_context.theme.font_size  # 15 ems wide
 80        self.widget3d.frame = gui.Rect(contentRect.x, contentRect.y,
 81                                       contentRect.width - panel_width,
 82                                       contentRect.height)
 83        self.panel.frame = gui.Rect(self.widget3d.frame.get_right(),
 84                                    contentRect.y, panel_width,
 85                                    contentRect.height)
 86
 87    def _on_close(self):
 88        self.is_done = True
 89        return True  # False would cancel the close
 90
 91    def _update_thread(self):
 92        # This is NOT the UI thread, need to call post_to_main_thread() to update
 93        # the scene or any part of the UI.
 94        idx = 0
 95        while not self.is_done:
 96            time.sleep(0.100)
 97
 98            # Get the next frame, for instance, reading a frame from the camera.
 99            rgb_frame = self.rgb_images[idx]
100            depth_frame = self.depth_images[idx]
101            idx += 1
102            if idx >= len(self.rgb_images):
103                idx = 0
104
105            # Update the images. This must be done on the UI thread.
106            def update():
107                self.rgb_widget.update_image(rgb_frame)
108                self.depth_widget.update_image(depth_frame)
109                self.widget3d.scene.set_background([1, 1, 1, 1], rgb_frame)
110
111            if not self.is_done:
112                gui.Application.instance.post_to_main_thread(
113                    self.window, update)
114
115
116def main():
117    app = o3d.visualization.gui.Application.instance
118    app.initialize()
119
120    win = VideoWindow()
121
122    app.run()
123
124
125if __name__ == "__main__":
126    main()

vis_gui.py#

  1# ----------------------------------------------------------------------------
  2# -                        Open3D: www.open3d.org                            -
  3# ----------------------------------------------------------------------------
  4# Copyright (c) 2018-2023 www.open3d.org
  5# SPDX-License-Identifier: MIT
  6# ----------------------------------------------------------------------------
  7
  8import glob
  9import numpy as np
 10import open3d as o3d
 11import open3d.visualization.gui as gui
 12import open3d.visualization.rendering as rendering
 13import os
 14import platform
 15import sys
 16
 17isMacOS = (platform.system() == "Darwin")
 18
 19
 20class Settings:
 21    UNLIT = "defaultUnlit"
 22    LIT = "defaultLit"
 23    NORMALS = "normals"
 24    DEPTH = "depth"
 25
 26    DEFAULT_PROFILE_NAME = "Bright day with sun at +Y [default]"
 27    POINT_CLOUD_PROFILE_NAME = "Cloudy day (no direct sun)"
 28    CUSTOM_PROFILE_NAME = "Custom"
 29    LIGHTING_PROFILES = {
 30        DEFAULT_PROFILE_NAME: {
 31            "ibl_intensity": 45000,
 32            "sun_intensity": 45000,
 33            "sun_dir": [0.577, -0.577, -0.577],
 34            # "ibl_rotation":
 35            "use_ibl": True,
 36            "use_sun": True,
 37        },
 38        "Bright day with sun at -Y": {
 39            "ibl_intensity": 45000,
 40            "sun_intensity": 45000,
 41            "sun_dir": [0.577, 0.577, 0.577],
 42            # "ibl_rotation":
 43            "use_ibl": True,
 44            "use_sun": True,
 45        },
 46        "Bright day with sun at +Z": {
 47            "ibl_intensity": 45000,
 48            "sun_intensity": 45000,
 49            "sun_dir": [0.577, 0.577, -0.577],
 50            # "ibl_rotation":
 51            "use_ibl": True,
 52            "use_sun": True,
 53        },
 54        "Less Bright day with sun at +Y": {
 55            "ibl_intensity": 35000,
 56            "sun_intensity": 50000,
 57            "sun_dir": [0.577, -0.577, -0.577],
 58            # "ibl_rotation":
 59            "use_ibl": True,
 60            "use_sun": True,
 61        },
 62        "Less Bright day with sun at -Y": {
 63            "ibl_intensity": 35000,
 64            "sun_intensity": 50000,
 65            "sun_dir": [0.577, 0.577, 0.577],
 66            # "ibl_rotation":
 67            "use_ibl": True,
 68            "use_sun": True,
 69        },
 70        "Less Bright day with sun at +Z": {
 71            "ibl_intensity": 35000,
 72            "sun_intensity": 50000,
 73            "sun_dir": [0.577, 0.577, -0.577],
 74            # "ibl_rotation":
 75            "use_ibl": True,
 76            "use_sun": True,
 77        },
 78        POINT_CLOUD_PROFILE_NAME: {
 79            "ibl_intensity": 60000,
 80            "sun_intensity": 50000,
 81            "use_ibl": True,
 82            "use_sun": False,
 83            # "ibl_rotation":
 84        },
 85    }
 86
 87    DEFAULT_MATERIAL_NAME = "Polished ceramic [default]"
 88    PREFAB = {
 89        DEFAULT_MATERIAL_NAME: {
 90            "metallic": 0.0,
 91            "roughness": 0.7,
 92            "reflectance": 0.5,
 93            "clearcoat": 0.2,
 94            "clearcoat_roughness": 0.2,
 95            "anisotropy": 0.0
 96        },
 97        "Metal (rougher)": {
 98            "metallic": 1.0,
 99            "roughness": 0.5,
100            "reflectance": 0.9,
101            "clearcoat": 0.0,
102            "clearcoat_roughness": 0.0,
103            "anisotropy": 0.0
104        },
105        "Metal (smoother)": {
106            "metallic": 1.0,
107            "roughness": 0.3,
108            "reflectance": 0.9,
109            "clearcoat": 0.0,
110            "clearcoat_roughness": 0.0,
111            "anisotropy": 0.0
112        },
113        "Plastic": {
114            "metallic": 0.0,
115            "roughness": 0.5,
116            "reflectance": 0.5,
117            "clearcoat": 0.5,
118            "clearcoat_roughness": 0.2,
119            "anisotropy": 0.0
120        },
121        "Glazed ceramic": {
122            "metallic": 0.0,
123            "roughness": 0.5,
124            "reflectance": 0.9,
125            "clearcoat": 1.0,
126            "clearcoat_roughness": 0.1,
127            "anisotropy": 0.0
128        },
129        "Clay": {
130            "metallic": 0.0,
131            "roughness": 1.0,
132            "reflectance": 0.5,
133            "clearcoat": 0.1,
134            "clearcoat_roughness": 0.287,
135            "anisotropy": 0.0
136        },
137    }
138
139    def __init__(self):
140        self.mouse_model = gui.SceneWidget.Controls.ROTATE_CAMERA
141        self.bg_color = gui.Color(1, 1, 1)
142        self.show_skybox = False
143        self.show_axes = False
144        self.use_ibl = True
145        self.use_sun = True
146        self.new_ibl_name = None  # clear to None after loading
147        self.ibl_intensity = 45000
148        self.sun_intensity = 45000
149        self.sun_dir = [0.577, -0.577, -0.577]
150        self.sun_color = gui.Color(1, 1, 1)
151
152        self.apply_material = True  # clear to False after processing
153        self._materials = {
154            Settings.LIT: rendering.MaterialRecord(),
155            Settings.UNLIT: rendering.MaterialRecord(),
156            Settings.NORMALS: rendering.MaterialRecord(),
157            Settings.DEPTH: rendering.MaterialRecord()
158        }
159        self._materials[Settings.LIT].base_color = [0.9, 0.9, 0.9, 1.0]
160        self._materials[Settings.LIT].shader = Settings.LIT
161        self._materials[Settings.UNLIT].base_color = [0.9, 0.9, 0.9, 1.0]
162        self._materials[Settings.UNLIT].shader = Settings.UNLIT
163        self._materials[Settings.NORMALS].shader = Settings.NORMALS
164        self._materials[Settings.DEPTH].shader = Settings.DEPTH
165
166        # Conveniently, assigning from self._materials[...] assigns a reference,
167        # not a copy, so if we change the property of a material, then switch
168        # to another one, then come back, the old setting will still be there.
169        self.material = self._materials[Settings.LIT]
170
171    def set_material(self, name):
172        self.material = self._materials[name]
173        self.apply_material = True
174
175    def apply_material_prefab(self, name):
176        assert (self.material.shader == Settings.LIT)
177        prefab = Settings.PREFAB[name]
178        for key, val in prefab.items():
179            setattr(self.material, "base_" + key, val)
180
181    def apply_lighting_profile(self, name):
182        profile = Settings.LIGHTING_PROFILES[name]
183        for key, val in profile.items():
184            setattr(self, key, val)
185
186
187class AppWindow:
188    MENU_OPEN = 1
189    MENU_EXPORT = 2
190    MENU_QUIT = 3
191    MENU_SHOW_SETTINGS = 11
192    MENU_ABOUT = 21
193
194    DEFAULT_IBL = "default"
195
196    MATERIAL_NAMES = ["Lit", "Unlit", "Normals", "Depth"]
197    MATERIAL_SHADERS = [
198        Settings.LIT, Settings.UNLIT, Settings.NORMALS, Settings.DEPTH
199    ]
200
201    def __init__(self, width, height):
202        self.settings = Settings()
203        resource_path = gui.Application.instance.resource_path
204        self.settings.new_ibl_name = resource_path + "/" + AppWindow.DEFAULT_IBL
205
206        self.window = gui.Application.instance.create_window(
207            "Open3D", width, height)
208        w = self.window  # to make the code more concise
209
210        # 3D widget
211        self._scene = gui.SceneWidget()
212        self._scene.scene = rendering.Open3DScene(w.renderer)
213        self._scene.set_on_sun_direction_changed(self._on_sun_dir)
214
215        # ---- Settings panel ----
216        # Rather than specifying sizes in pixels, which may vary in size based
217        # on the monitor, especially on macOS which has 220 dpi monitors, use
218        # the em-size. This way sizings will be proportional to the font size,
219        # which will create a more visually consistent size across platforms.
220        em = w.theme.font_size
221        separation_height = int(round(0.5 * em))
222
223        # Widgets are laid out in layouts: gui.Horiz, gui.Vert,
224        # gui.CollapsableVert, and gui.VGrid. By nesting the layouts we can
225        # achieve complex designs. Usually we use a vertical layout as the
226        # topmost widget, since widgets tend to be organized from top to bottom.
227        # Within that, we usually have a series of horizontal layouts for each
228        # row. All layouts take a spacing parameter, which is the spacing
229        # between items in the widget, and a margins parameter, which specifies
230        # the spacing of the left, top, right, bottom margins. (This acts like
231        # the 'padding' property in CSS.)
232        self._settings_panel = gui.Vert(
233            0, gui.Margins(0.25 * em, 0.25 * em, 0.25 * em, 0.25 * em))
234
235        # Create a collapsible vertical widget, which takes up enough vertical
236        # space for all its children when open, but only enough for text when
237        # closed. This is useful for property pages, so the user can hide sets
238        # of properties they rarely use.
239        view_ctrls = gui.CollapsableVert("View controls", 0.25 * em,
240                                         gui.Margins(em, 0, 0, 0))
241
242        self._arcball_button = gui.Button("Arcball")
243        self._arcball_button.horizontal_padding_em = 0.5
244        self._arcball_button.vertical_padding_em = 0
245        self._arcball_button.set_on_clicked(self._set_mouse_mode_rotate)
246        self._fly_button = gui.Button("Fly")
247        self._fly_button.horizontal_padding_em = 0.5
248        self._fly_button.vertical_padding_em = 0
249        self._fly_button.set_on_clicked(self._set_mouse_mode_fly)
250        self._model_button = gui.Button("Model")
251        self._model_button.horizontal_padding_em = 0.5
252        self._model_button.vertical_padding_em = 0
253        self._model_button.set_on_clicked(self._set_mouse_mode_model)
254        self._sun_button = gui.Button("Sun")
255        self._sun_button.horizontal_padding_em = 0.5
256        self._sun_button.vertical_padding_em = 0
257        self._sun_button.set_on_clicked(self._set_mouse_mode_sun)
258        self._ibl_button = gui.Button("Environment")
259        self._ibl_button.horizontal_padding_em = 0.5
260        self._ibl_button.vertical_padding_em = 0
261        self._ibl_button.set_on_clicked(self._set_mouse_mode_ibl)
262        view_ctrls.add_child(gui.Label("Mouse controls"))
263        # We want two rows of buttons, so make two horizontal layouts. We also
264        # want the buttons centered, which we can do be putting a stretch item
265        # as the first and last item. Stretch items take up as much space as
266        # possible, and since there are two, they will each take half the extra
267        # space, thus centering the buttons.
268        h = gui.Horiz(0.25 * em)  # row 1
269        h.add_stretch()
270        h.add_child(self._arcball_button)
271        h.add_child(self._fly_button)
272        h.add_child(self._model_button)
273        h.add_stretch()
274        view_ctrls.add_child(h)
275        h = gui.Horiz(0.25 * em)  # row 2
276        h.add_stretch()
277        h.add_child(self._sun_button)
278        h.add_child(self._ibl_button)
279        h.add_stretch()
280        view_ctrls.add_child(h)
281
282        self._show_skybox = gui.Checkbox("Show skymap")
283        self._show_skybox.set_on_checked(self._on_show_skybox)
284        view_ctrls.add_fixed(separation_height)
285        view_ctrls.add_child(self._show_skybox)
286
287        self._bg_color = gui.ColorEdit()
288        self._bg_color.set_on_value_changed(self._on_bg_color)
289
290        grid = gui.VGrid(2, 0.25 * em)
291        grid.add_child(gui.Label("BG Color"))
292        grid.add_child(self._bg_color)
293        view_ctrls.add_child(grid)
294
295        self._show_axes = gui.Checkbox("Show axes")
296        self._show_axes.set_on_checked(self._on_show_axes)
297        view_ctrls.add_fixed(separation_height)
298        view_ctrls.add_child(self._show_axes)
299
300        self._profiles = gui.Combobox()
301        for name in sorted(Settings.LIGHTING_PROFILES.keys()):
302            self._profiles.add_item(name)
303        self._profiles.add_item(Settings.CUSTOM_PROFILE_NAME)
304        self._profiles.set_on_selection_changed(self._on_lighting_profile)
305        view_ctrls.add_fixed(separation_height)
306        view_ctrls.add_child(gui.Label("Lighting profiles"))
307        view_ctrls.add_child(self._profiles)
308        self._settings_panel.add_fixed(separation_height)
309        self._settings_panel.add_child(view_ctrls)
310
311        advanced = gui.CollapsableVert("Advanced lighting", 0,
312                                       gui.Margins(em, 0, 0, 0))
313        advanced.set_is_open(False)
314
315        self._use_ibl = gui.Checkbox("HDR map")
316        self._use_ibl.set_on_checked(self._on_use_ibl)
317        self._use_sun = gui.Checkbox("Sun")
318        self._use_sun.set_on_checked(self._on_use_sun)
319        advanced.add_child(gui.Label("Light sources"))
320        h = gui.Horiz(em)
321        h.add_child(self._use_ibl)
322        h.add_child(self._use_sun)
323        advanced.add_child(h)
324
325        self._ibl_map = gui.Combobox()
326        for ibl in glob.glob(gui.Application.instance.resource_path +
327                             "/*_ibl.ktx"):
328
329            self._ibl_map.add_item(os.path.basename(ibl[:-8]))
330        self._ibl_map.selected_text = AppWindow.DEFAULT_IBL
331        self._ibl_map.set_on_selection_changed(self._on_new_ibl)
332        self._ibl_intensity = gui.Slider(gui.Slider.INT)
333        self._ibl_intensity.set_limits(0, 200000)
334        self._ibl_intensity.set_on_value_changed(self._on_ibl_intensity)
335        grid = gui.VGrid(2, 0.25 * em)
336        grid.add_child(gui.Label("HDR map"))
337        grid.add_child(self._ibl_map)
338        grid.add_child(gui.Label("Intensity"))
339        grid.add_child(self._ibl_intensity)
340        advanced.add_fixed(separation_height)
341        advanced.add_child(gui.Label("Environment"))
342        advanced.add_child(grid)
343
344        self._sun_intensity = gui.Slider(gui.Slider.INT)
345        self._sun_intensity.set_limits(0, 200000)
346        self._sun_intensity.set_on_value_changed(self._on_sun_intensity)
347        self._sun_dir = gui.VectorEdit()
348        self._sun_dir.set_on_value_changed(self._on_sun_dir)
349        self._sun_color = gui.ColorEdit()
350        self._sun_color.set_on_value_changed(self._on_sun_color)
351        grid = gui.VGrid(2, 0.25 * em)
352        grid.add_child(gui.Label("Intensity"))
353        grid.add_child(self._sun_intensity)
354        grid.add_child(gui.Label("Direction"))
355        grid.add_child(self._sun_dir)
356        grid.add_child(gui.Label("Color"))
357        grid.add_child(self._sun_color)
358        advanced.add_fixed(separation_height)
359        advanced.add_child(gui.Label("Sun (Directional light)"))
360        advanced.add_child(grid)
361
362        self._settings_panel.add_fixed(separation_height)
363        self._settings_panel.add_child(advanced)
364
365        material_settings = gui.CollapsableVert("Material settings", 0,
366                                                gui.Margins(em, 0, 0, 0))
367
368        self._shader = gui.Combobox()
369        self._shader.add_item(AppWindow.MATERIAL_NAMES[0])
370        self._shader.add_item(AppWindow.MATERIAL_NAMES[1])
371        self._shader.add_item(AppWindow.MATERIAL_NAMES[2])
372        self._shader.add_item(AppWindow.MATERIAL_NAMES[3])
373        self._shader.set_on_selection_changed(self._on_shader)
374        self._material_prefab = gui.Combobox()
375        for prefab_name in sorted(Settings.PREFAB.keys()):
376            self._material_prefab.add_item(prefab_name)
377        self._material_prefab.selected_text = Settings.DEFAULT_MATERIAL_NAME
378        self._material_prefab.set_on_selection_changed(self._on_material_prefab)
379        self._material_color = gui.ColorEdit()
380        self._material_color.set_on_value_changed(self._on_material_color)
381        self._point_size = gui.Slider(gui.Slider.INT)
382        self._point_size.set_limits(1, 10)
383        self._point_size.set_on_value_changed(self._on_point_size)
384
385        grid = gui.VGrid(2, 0.25 * em)
386        grid.add_child(gui.Label("Type"))
387        grid.add_child(self._shader)
388        grid.add_child(gui.Label("Material"))
389        grid.add_child(self._material_prefab)
390        grid.add_child(gui.Label("Color"))
391        grid.add_child(self._material_color)
392        grid.add_child(gui.Label("Point size"))
393        grid.add_child(self._point_size)
394        material_settings.add_child(grid)
395
396        self._settings_panel.add_fixed(separation_height)
397        self._settings_panel.add_child(material_settings)
398        # ----
399
400        # Normally our user interface can be children of all one layout (usually
401        # a vertical layout), which is then the only child of the window. In our
402        # case we want the scene to take up all the space and the settings panel
403        # to go above it. We can do this custom layout by providing an on_layout
404        # callback. The on_layout callback should set the frame
405        # (position + size) of every child correctly. After the callback is
406        # done the window will layout the grandchildren.
407        w.set_on_layout(self._on_layout)
408        w.add_child(self._scene)
409        w.add_child(self._settings_panel)
410
411        # ---- Menu ----
412        # The menu is global (because the macOS menu is global), so only create
413        # it once, no matter how many windows are created
414        if gui.Application.instance.menubar is None:
415            if isMacOS:
416                app_menu = gui.Menu()
417                app_menu.add_item("About", AppWindow.MENU_ABOUT)
418                app_menu.add_separator()
419                app_menu.add_item("Quit", AppWindow.MENU_QUIT)
420            file_menu = gui.Menu()
421            file_menu.add_item("Open...", AppWindow.MENU_OPEN)
422            file_menu.add_item("Export Current Image...", AppWindow.MENU_EXPORT)
423            if not isMacOS:
424                file_menu.add_separator()
425                file_menu.add_item("Quit", AppWindow.MENU_QUIT)
426            settings_menu = gui.Menu()
427            settings_menu.add_item("Lighting & Materials",
428                                   AppWindow.MENU_SHOW_SETTINGS)
429            settings_menu.set_checked(AppWindow.MENU_SHOW_SETTINGS, True)
430            help_menu = gui.Menu()
431            help_menu.add_item("About", AppWindow.MENU_ABOUT)
432
433            menu = gui.Menu()
434            if isMacOS:
435                # macOS will name the first menu item for the running application
436                # (in our case, probably "Python"), regardless of what we call
437                # it. This is the application menu, and it is where the
438                # About..., Preferences..., and Quit menu items typically go.
439                menu.add_menu("Example", app_menu)
440                menu.add_menu("File", file_menu)
441                menu.add_menu("Settings", settings_menu)
442                # Don't include help menu unless it has something more than
443                # About...
444            else:
445                menu.add_menu("File", file_menu)
446                menu.add_menu("Settings", settings_menu)
447                menu.add_menu("Help", help_menu)
448            gui.Application.instance.menubar = menu
449
450        # The menubar is global, but we need to connect the menu items to the
451        # window, so that the window can call the appropriate function when the
452        # menu item is activated.
453        w.set_on_menu_item_activated(AppWindow.MENU_OPEN, self._on_menu_open)
454        w.set_on_menu_item_activated(AppWindow.MENU_EXPORT,
455                                     self._on_menu_export)
456        w.set_on_menu_item_activated(AppWindow.MENU_QUIT, self._on_menu_quit)
457        w.set_on_menu_item_activated(AppWindow.MENU_SHOW_SETTINGS,
458                                     self._on_menu_toggle_settings_panel)
459        w.set_on_menu_item_activated(AppWindow.MENU_ABOUT, self._on_menu_about)
460        # ----
461
462        self._apply_settings()
463
464    def _apply_settings(self):
465        bg_color = [
466            self.settings.bg_color.red, self.settings.bg_color.green,
467            self.settings.bg_color.blue, self.settings.bg_color.alpha
468        ]
469        self._scene.scene.set_background(bg_color)
470        self._scene.scene.show_skybox(self.settings.show_skybox)
471        self._scene.scene.show_axes(self.settings.show_axes)
472        if self.settings.new_ibl_name is not None:
473            self._scene.scene.scene.set_indirect_light(
474                self.settings.new_ibl_name)
475            # Clear new_ibl_name, so we don't keep reloading this image every
476            # time the settings are applied.
477            self.settings.new_ibl_name = None
478        self._scene.scene.scene.enable_indirect_light(self.settings.use_ibl)
479        self._scene.scene.scene.set_indirect_light_intensity(
480            self.settings.ibl_intensity)
481        sun_color = [
482            self.settings.sun_color.red, self.settings.sun_color.green,
483            self.settings.sun_color.blue
484        ]
485        self._scene.scene.scene.set_sun_light(self.settings.sun_dir, sun_color,
486                                              self.settings.sun_intensity)
487        self._scene.scene.scene.enable_sun_light(self.settings.use_sun)
488
489        if self.settings.apply_material:
490            self._scene.scene.update_material(self.settings.material)
491            self.settings.apply_material = False
492
493        self._bg_color.color_value = self.settings.bg_color
494        self._show_skybox.checked = self.settings.show_skybox
495        self._show_axes.checked = self.settings.show_axes
496        self._use_ibl.checked = self.settings.use_ibl
497        self._use_sun.checked = self.settings.use_sun
498        self._ibl_intensity.int_value = self.settings.ibl_intensity
499        self._sun_intensity.int_value = self.settings.sun_intensity
500        self._sun_dir.vector_value = self.settings.sun_dir
501        self._sun_color.color_value = self.settings.sun_color
502        self._material_prefab.enabled = (
503            self.settings.material.shader == Settings.LIT)
504        c = gui.Color(self.settings.material.base_color[0],
505                      self.settings.material.base_color[1],
506                      self.settings.material.base_color[2],
507                      self.settings.material.base_color[3])
508        self._material_color.color_value = c
509        self._point_size.double_value = self.settings.material.point_size
510
511    def _on_layout(self, layout_context):
512        # The on_layout callback should set the frame (position + size) of every
513        # child correctly. After the callback is done the window will layout
514        # the grandchildren.
515        r = self.window.content_rect
516        self._scene.frame = r
517        width = 17 * layout_context.theme.font_size
518        height = min(
519            r.height,
520            self._settings_panel.calc_preferred_size(
521                layout_context, gui.Widget.Constraints()).height)
522        self._settings_panel.frame = gui.Rect(r.get_right() - width, r.y, width,
523                                              height)
524
525    def _set_mouse_mode_rotate(self):
526        self._scene.set_view_controls(gui.SceneWidget.Controls.ROTATE_CAMERA)
527
528    def _set_mouse_mode_fly(self):
529        self._scene.set_view_controls(gui.SceneWidget.Controls.FLY)
530
531    def _set_mouse_mode_sun(self):
532        self._scene.set_view_controls(gui.SceneWidget.Controls.ROTATE_SUN)
533
534    def _set_mouse_mode_ibl(self):
535        self._scene.set_view_controls(gui.SceneWidget.Controls.ROTATE_IBL)
536
537    def _set_mouse_mode_model(self):
538        self._scene.set_view_controls(gui.SceneWidget.Controls.ROTATE_MODEL)
539
540    def _on_bg_color(self, new_color):
541        self.settings.bg_color = new_color
542        self._apply_settings()
543
544    def _on_show_skybox(self, show):
545        self.settings.show_skybox = show
546        self._apply_settings()
547
548    def _on_show_axes(self, show):
549        self.settings.show_axes = show
550        self._apply_settings()
551
552    def _on_use_ibl(self, use):
553        self.settings.use_ibl = use
554        self._profiles.selected_text = Settings.CUSTOM_PROFILE_NAME
555        self._apply_settings()
556
557    def _on_use_sun(self, use):
558        self.settings.use_sun = use
559        self._profiles.selected_text = Settings.CUSTOM_PROFILE_NAME
560        self._apply_settings()
561
562    def _on_lighting_profile(self, name, index):
563        if name != Settings.CUSTOM_PROFILE_NAME:
564            self.settings.apply_lighting_profile(name)
565            self._apply_settings()
566
567    def _on_new_ibl(self, name, index):
568        self.settings.new_ibl_name = gui.Application.instance.resource_path + "/" + name
569        self._profiles.selected_text = Settings.CUSTOM_PROFILE_NAME
570        self._apply_settings()
571
572    def _on_ibl_intensity(self, intensity):
573        self.settings.ibl_intensity = int(intensity)
574        self._profiles.selected_text = Settings.CUSTOM_PROFILE_NAME
575        self._apply_settings()
576
577    def _on_sun_intensity(self, intensity):
578        self.settings.sun_intensity = int(intensity)
579        self._profiles.selected_text = Settings.CUSTOM_PROFILE_NAME
580        self._apply_settings()
581
582    def _on_sun_dir(self, sun_dir):
583        self.settings.sun_dir = sun_dir
584        self._profiles.selected_text = Settings.CUSTOM_PROFILE_NAME
585        self._apply_settings()
586
587    def _on_sun_color(self, color):
588        self.settings.sun_color = color
589        self._apply_settings()
590
591    def _on_shader(self, name, index):
592        self.settings.set_material(AppWindow.MATERIAL_SHADERS[index])
593        self._apply_settings()
594
595    def _on_material_prefab(self, name, index):
596        self.settings.apply_material_prefab(name)
597        self.settings.apply_material = True
598        self._apply_settings()
599
600    def _on_material_color(self, color):
601        self.settings.material.base_color = [
602            color.red, color.green, color.blue, color.alpha
603        ]
604        self.settings.apply_material = True
605        self._apply_settings()
606
607    def _on_point_size(self, size):
608        self.settings.material.point_size = int(size)
609        self.settings.apply_material = True
610        self._apply_settings()
611
612    def _on_menu_open(self):
613        dlg = gui.FileDialog(gui.FileDialog.OPEN, "Choose file to load",
614                             self.window.theme)
615        dlg.add_filter(
616            ".ply .stl .fbx .obj .off .gltf .glb",
617            "Triangle mesh files (.ply, .stl, .fbx, .obj, .off, "
618            ".gltf, .glb)")
619        dlg.add_filter(
620            ".xyz .xyzn .xyzrgb .ply .pcd .pts",
621            "Point cloud files (.xyz, .xyzn, .xyzrgb, .ply, "
622            ".pcd, .pts)")
623        dlg.add_filter(".ply", "Polygon files (.ply)")
624        dlg.add_filter(".stl", "Stereolithography files (.stl)")
625        dlg.add_filter(".fbx", "Autodesk Filmbox files (.fbx)")
626        dlg.add_filter(".obj", "Wavefront OBJ files (.obj)")
627        dlg.add_filter(".off", "Object file format (.off)")
628        dlg.add_filter(".gltf", "OpenGL transfer files (.gltf)")
629        dlg.add_filter(".glb", "OpenGL binary transfer files (.glb)")
630        dlg.add_filter(".xyz", "ASCII point cloud files (.xyz)")
631        dlg.add_filter(".xyzn", "ASCII point cloud with normals (.xyzn)")
632        dlg.add_filter(".xyzrgb",
633                       "ASCII point cloud files with colors (.xyzrgb)")
634        dlg