Loading [MathJax]/extensions/TeX/AMSsymbols.js
Open3D (C++ API)  0.14.1
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
VoxelBlockGridImpl.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // The MIT License (MIT)
5 //
6 // Copyright (c) 2018-2021 www.open3d.org
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a copy
9 // of this software and associated documentation files (the "Software"), to deal
10 // in the Software without restriction, including without limitation the rights
11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 // copies of the Software, and to permit persons to whom the Software is
13 // furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 // IN THE SOFTWARE.
25 // ----------------------------------------------------------------------------
26 
27 #include <atomic>
28 #include <cmath>
29 
30 #include "open3d/core/Dispatch.h"
31 #include "open3d/core/Dtype.h"
33 #include "open3d/core/SizeVector.h"
34 #include "open3d/core/Tensor.h"
40 #include "open3d/utility/Logging.h"
41 #include "open3d/utility/Timer.h"
42 
43 namespace open3d {
44 namespace t {
45 namespace geometry {
46 namespace kernel {
47 namespace voxel_grid {
48 
49 using index_t = int;
51 
52 #if defined(__CUDACC__)
53 void GetVoxelCoordinatesAndFlattenedIndicesCUDA
54 #else
56 #endif
57  (const core::Tensor& buf_indices,
58  const core::Tensor& block_keys,
59  core::Tensor& voxel_coords,
60  core::Tensor& flattened_indices,
61  index_t resolution,
62  float voxel_size) {
63  core::Device device = buf_indices.GetDevice();
64 
65  const index_t* buf_indices_ptr = buf_indices.GetDataPtr<index_t>();
66  const index_t* block_key_ptr = block_keys.GetDataPtr<index_t>();
67 
68  float* voxel_coords_ptr = voxel_coords.GetDataPtr<float>();
69  int64_t* flattened_indices_ptr = flattened_indices.GetDataPtr<int64_t>();
70 
71  index_t n = flattened_indices.GetLength();
72  ArrayIndexer voxel_indexer({resolution, resolution, resolution});
73  index_t resolution3 = resolution * resolution * resolution;
74 
75  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t workload_idx) {
76  index_t block_idx = buf_indices_ptr[workload_idx / resolution3];
77  index_t voxel_idx = workload_idx % resolution3;
78 
79  index_t block_key_offset = block_idx * 3;
80  index_t xb = block_key_ptr[block_key_offset + 0];
81  index_t yb = block_key_ptr[block_key_offset + 1];
82  index_t zb = block_key_ptr[block_key_offset + 2];
83 
84  index_t xv, yv, zv;
85  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
86 
87  float x = (xb * resolution + xv) * voxel_size;
88  float y = (yb * resolution + yv) * voxel_size;
89  float z = (zb * resolution + zv) * voxel_size;
90 
91  flattened_indices_ptr[workload_idx] =
92  block_idx * resolution3 + voxel_idx;
93 
94  index_t voxel_coords_offset = workload_idx * 3;
95  voxel_coords_ptr[voxel_coords_offset + 0] = x;
96  voxel_coords_ptr[voxel_coords_offset + 1] = y;
97  voxel_coords_ptr[voxel_coords_offset + 2] = z;
98  });
99 }
100 
101 inline OPEN3D_DEVICE index_t
103  index_t yo,
104  index_t zo,
105  index_t curr_block_idx,
106  index_t resolution,
107  const ArrayIndexer& nb_block_masks_indexer,
108  const ArrayIndexer& nb_block_indices_indexer) {
109  index_t xn = (xo + resolution) % resolution;
110  index_t yn = (yo + resolution) % resolution;
111  index_t zn = (zo + resolution) % resolution;
112 
113  index_t dxb = Sign(xo - xn);
114  index_t dyb = Sign(yo - yn);
115  index_t dzb = Sign(zo - zn);
116 
117  index_t nb_idx = (dxb + 1) + (dyb + 1) * 3 + (dzb + 1) * 9;
118 
119  bool block_mask_i =
120  *nb_block_masks_indexer.GetDataPtr<bool>(curr_block_idx, nb_idx);
121  if (!block_mask_i) return -1;
122 
123  index_t block_idx_i = *nb_block_indices_indexer.GetDataPtr<index_t>(
124  curr_block_idx, nb_idx);
125 
126  return (((block_idx_i * resolution) + zn) * resolution + yn) * resolution +
127  xn;
128 }
129 
130 template <typename tsdf_t>
132  const tsdf_t* tsdf_base_ptr,
133  index_t xo,
134  index_t yo,
135  index_t zo,
136  index_t curr_block_idx,
137  float* n,
138  index_t resolution,
139  const ArrayIndexer& nb_block_masks_indexer,
140  const ArrayIndexer& nb_block_indices_indexer) {
141  auto GetLinearIdx = [&] OPEN3D_DEVICE(index_t xo, index_t yo,
142  index_t zo) -> index_t {
143  return DeviceGetLinearIdx(xo, yo, zo, curr_block_idx, resolution,
144  nb_block_masks_indexer,
145  nb_block_indices_indexer);
146  };
147  index_t vxp = GetLinearIdx(xo + 1, yo, zo);
148  index_t vxn = GetLinearIdx(xo - 1, yo, zo);
149  index_t vyp = GetLinearIdx(xo, yo + 1, zo);
150  index_t vyn = GetLinearIdx(xo, yo - 1, zo);
151  index_t vzp = GetLinearIdx(xo, yo, zo + 1);
152  index_t vzn = GetLinearIdx(xo, yo, zo - 1);
153  if (vxp >= 0 && vxn >= 0) n[0] = tsdf_base_ptr[vxp] - tsdf_base_ptr[vxn];
154  if (vyp >= 0 && vyn >= 0) n[1] = tsdf_base_ptr[vyp] - tsdf_base_ptr[vyn];
155  if (vzp >= 0 && vzn >= 0) n[2] = tsdf_base_ptr[vzp] - tsdf_base_ptr[vzn];
156 };
157 
158 template <typename input_depth_t,
159  typename input_color_t,
160  typename tsdf_t,
161  typename weight_t,
162  typename color_t>
163 #if defined(__CUDACC__)
164 void IntegrateCUDA
165 #else
166 void IntegrateCPU
167 #endif
168  (const core::Tensor& depth,
169  const core::Tensor& color,
170  const core::Tensor& indices,
171  const core::Tensor& block_keys,
172  TensorMap& block_value_map,
173  const core::Tensor& intrinsics,
174  const core::Tensor& extrinsics,
175  index_t resolution,
176  float voxel_size,
177  float sdf_trunc,
178  float depth_scale,
179  float depth_max) {
180  // Parameters
181  index_t resolution2 = resolution * resolution;
182  index_t resolution3 = resolution2 * resolution;
183 
184  TransformIndexer transform_indexer(intrinsics, extrinsics, voxel_size);
185 
186  ArrayIndexer voxel_indexer({resolution, resolution, resolution});
187 
188  ArrayIndexer block_keys_indexer(block_keys, 1);
189  ArrayIndexer depth_indexer(depth, 2);
190  core::Device device = block_keys.GetDevice();
191 
192  const index_t* indices_ptr = indices.GetDataPtr<index_t>();
193 
194  if (!block_value_map.Contains("tsdf") ||
195  !block_value_map.Contains("weight")) {
197  "TSDF and/or weight not allocated in blocks, please implement "
198  "customized integration.");
199  }
200  tsdf_t* tsdf_base_ptr = block_value_map.at("tsdf").GetDataPtr<tsdf_t>();
201  weight_t* weight_base_ptr =
202  block_value_map.at("weight").GetDataPtr<weight_t>();
203 
204  bool integrate_color =
205  block_value_map.Contains("color") && color.NumElements() > 0;
206  color_t* color_base_ptr = nullptr;
207  ArrayIndexer color_indexer;
208 
209  float color_multiplier = 1.0;
210  if (integrate_color) {
211  color_base_ptr = block_value_map.at("color").GetDataPtr<color_t>();
212  color_indexer = ArrayIndexer(color, 2);
213 
214  // Float32: [0, 1] -> [0, 255]
215  if (color.GetDtype() == core::Float32) {
216  color_multiplier = 255.0;
217  }
218  }
219 
220  index_t n = indices.GetLength() * resolution3;
221  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t workload_idx) {
222  // Natural index (0, N) -> (block_idx, voxel_idx)
223  index_t block_idx = indices_ptr[workload_idx / resolution3];
224  index_t voxel_idx = workload_idx % resolution3;
225 
227  // block_idx -> (x_block, y_block, z_block)
228  index_t* block_key_ptr =
229  block_keys_indexer.GetDataPtr<index_t>(block_idx);
230  index_t xb = block_key_ptr[0];
231  index_t yb = block_key_ptr[1];
232  index_t zb = block_key_ptr[2];
233 
234  // voxel_idx -> (x_voxel, y_voxel, z_voxel)
235  index_t xv, yv, zv;
236  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
237 
238  // coordinate in world (in voxel)
239  index_t x = xb * resolution + xv;
240  index_t y = yb * resolution + yv;
241  index_t z = zb * resolution + zv;
242 
243  // coordinate in camera (in voxel -> in meter)
244  float xc, yc, zc, u, v;
245  transform_indexer.RigidTransform(static_cast<float>(x),
246  static_cast<float>(y),
247  static_cast<float>(z), &xc, &yc, &zc);
248 
249  // coordinate in image (in pixel)
250  transform_indexer.Project(xc, yc, zc, &u, &v);
251  if (!depth_indexer.InBoundary(u, v)) {
252  return;
253  }
254 
255  index_t ui = static_cast<index_t>(u);
256  index_t vi = static_cast<index_t>(v);
257 
258  // Associate image workload and compute SDF and
259  // TSDF.
260  float depth =
261  *depth_indexer.GetDataPtr<input_depth_t>(ui, vi) / depth_scale;
262 
263  float sdf = depth - zc;
264  if (depth <= 0 || depth > depth_max || zc <= 0 || sdf < -sdf_trunc) {
265  return;
266  }
267  sdf = sdf < sdf_trunc ? sdf : sdf_trunc;
268  sdf /= sdf_trunc;
269 
270  index_t linear_idx = block_idx * resolution3 + voxel_idx;
271 
272  tsdf_t* tsdf_ptr = tsdf_base_ptr + linear_idx;
273  weight_t* weight_ptr = weight_base_ptr + linear_idx;
274 
275  float inv_wsum = 1.0f / (*weight_ptr + 1);
276  float weight = *weight_ptr;
277  *tsdf_ptr = (weight * (*tsdf_ptr) + sdf) * inv_wsum;
278 
279  if (integrate_color) {
280  color_t* color_ptr = color_base_ptr + 3 * linear_idx;
281  input_color_t* input_color_ptr =
282  color_indexer.GetDataPtr<input_color_t>(ui, vi);
283 
284  for (index_t i = 0; i < 3; ++i) {
285  color_ptr[i] = (weight * color_ptr[i] +
286  input_color_ptr[i] * color_multiplier) *
287  inv_wsum;
288  }
289  }
290  *weight_ptr = weight + 1;
291  });
292 
293 #if defined(__CUDACC__)
295 #endif
296 }
297 
298 struct MiniVecCache {
303 
305  return (xin == x && yin == y && zin == z) ? block_idx : -1;
306  }
307 
308  inline void OPEN3D_DEVICE Update(index_t xin,
309  index_t yin,
310  index_t zin,
311  index_t block_idx_in) {
312  x = xin;
313  y = yin;
314  z = zin;
315  block_idx = block_idx_in;
316  }
317 };
318 
319 template <typename tsdf_t, typename weight_t, typename color_t>
320 #if defined(__CUDACC__)
321 void RayCastCUDA
322 #else
323 void RayCastCPU
324 #endif
325  (std::shared_ptr<core::HashMap>& hashmap,
326  const TensorMap& block_value_map,
327  const core::Tensor& range,
328  TensorMap& renderings_map,
329  const core::Tensor& intrinsics,
330  const core::Tensor& extrinsics,
331  index_t h,
332  index_t w,
333  index_t block_resolution,
334  float voxel_size,
335  float sdf_trunc,
336  float depth_scale,
337  float depth_min,
338  float depth_max,
339  float weight_threshold) {
340  using Key = utility::MiniVec<index_t, 3>;
343 
344  auto device_hashmap = hashmap->GetDeviceHashBackend();
345 #if defined(__CUDACC__)
346  auto cuda_hashmap =
347  std::dynamic_pointer_cast<core::StdGPUHashBackend<Key, Hash, Eq>>(
348  device_hashmap);
349  if (cuda_hashmap == nullptr) {
351  "Unsupported backend: CUDA raycasting only supports STDGPU.");
352  }
353  auto hashmap_impl = cuda_hashmap->GetImpl();
354 #else
355  auto cpu_hashmap =
356  std::dynamic_pointer_cast<core::TBBHashBackend<Key, Hash, Eq>>(
357  device_hashmap);
358  if (cpu_hashmap == nullptr) {
360  "Unsupported backend: CPU raycasting only supports TBB.");
361  }
362  auto hashmap_impl = *cpu_hashmap->GetImpl();
363 #endif
364 
365  core::Device device = hashmap->GetDevice();
366 
367  ArrayIndexer range_indexer(range, 2);
368 
369  // Geometry
370  ArrayIndexer depth_indexer;
371  ArrayIndexer vertex_indexer;
372  ArrayIndexer normal_indexer;
373 
374  // Diff rendering
375  ArrayIndexer index_indexer;
376  ArrayIndexer mask_indexer;
377  ArrayIndexer interp_ratio_indexer;
378  ArrayIndexer interp_ratio_dx_indexer;
379  ArrayIndexer interp_ratio_dy_indexer;
380  ArrayIndexer interp_ratio_dz_indexer;
381 
382  // Color
383  ArrayIndexer color_indexer;
384 
385  if (!block_value_map.Contains("tsdf") ||
386  !block_value_map.Contains("weight")) {
388  "TSDF and/or weight not allocated in blocks, please implement "
389  "customized integration.");
390  }
391  const tsdf_t* tsdf_base_ptr =
392  block_value_map.at("tsdf").GetDataPtr<tsdf_t>();
393  const weight_t* weight_base_ptr =
394  block_value_map.at("weight").GetDataPtr<weight_t>();
395 
396  // Geometry
397  if (renderings_map.Contains("depth")) {
398  depth_indexer = ArrayIndexer(renderings_map.at("depth"), 2);
399  }
400  if (renderings_map.Contains("vertex")) {
401  vertex_indexer = ArrayIndexer(renderings_map.at("vertex"), 2);
402  }
403  if (renderings_map.Contains("normal")) {
404  normal_indexer = ArrayIndexer(renderings_map.at("normal"), 2);
405  }
406 
407  // Diff rendering
408  if (renderings_map.Contains("index")) {
409  index_indexer = ArrayIndexer(renderings_map.at("index"), 2);
410  }
411  if (renderings_map.Contains("mask")) {
412  mask_indexer = ArrayIndexer(renderings_map.at("mask"), 2);
413  }
414  if (renderings_map.Contains("interp_ratio")) {
415  interp_ratio_indexer =
416  ArrayIndexer(renderings_map.at("interp_ratio"), 2);
417  }
418  if (renderings_map.Contains("interp_ratio_dx")) {
419  interp_ratio_dx_indexer =
420  ArrayIndexer(renderings_map.at("interp_ratio_dx"), 2);
421  }
422  if (renderings_map.Contains("interp_ratio_dy")) {
423  interp_ratio_dy_indexer =
424  ArrayIndexer(renderings_map.at("interp_ratio_dy"), 2);
425  }
426  if (renderings_map.Contains("interp_ratio_dz")) {
427  interp_ratio_dz_indexer =
428  ArrayIndexer(renderings_map.at("interp_ratio_dz"), 2);
429  }
430 
431  // Color
432  bool render_color = false;
433  if (block_value_map.Contains("color") && renderings_map.Contains("color")) {
434  render_color = true;
435  color_indexer = ArrayIndexer(renderings_map.at("color"), 2);
436  }
437  const color_t* color_base_ptr =
438  render_color ? block_value_map.at("color").GetDataPtr<color_t>()
439  : nullptr;
440 
441  bool visit_neighbors = render_color || normal_indexer.GetDataPtr() ||
442  mask_indexer.GetDataPtr() ||
443  index_indexer.GetDataPtr() ||
444  interp_ratio_indexer.GetDataPtr() ||
445  interp_ratio_dx_indexer.GetDataPtr() ||
446  interp_ratio_dy_indexer.GetDataPtr() ||
447  interp_ratio_dz_indexer.GetDataPtr();
448 
449  TransformIndexer c2w_transform_indexer(
450  intrinsics, t::geometry::InverseTransformation(extrinsics));
451  TransformIndexer w2c_transform_indexer(intrinsics, extrinsics);
452 
453  index_t rows = h;
454  index_t cols = w;
455  index_t n = rows * cols;
456 
457  float block_size = voxel_size * block_resolution;
458  index_t resolution2 = block_resolution * block_resolution;
459  index_t resolution3 = resolution2 * block_resolution;
460 
461 #ifndef __CUDACC__
462  using std::max;
463  using std::sqrt;
464 #endif
465 
466  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t workload_idx) {
467  auto GetLinearIdxAtP = [&] OPEN3D_DEVICE(
468  index_t x_b, index_t y_b, index_t z_b,
469  index_t x_v, index_t y_v, index_t z_v,
470  core::buf_index_t block_buf_idx,
471  MiniVecCache & cache) -> index_t {
472  index_t x_vn = (x_v + block_resolution) % block_resolution;
473  index_t y_vn = (y_v + block_resolution) % block_resolution;
474  index_t z_vn = (z_v + block_resolution) % block_resolution;
475 
476  index_t dx_b = Sign(x_v - x_vn);
477  index_t dy_b = Sign(y_v - y_vn);
478  index_t dz_b = Sign(z_v - z_vn);
479 
480  if (dx_b == 0 && dy_b == 0 && dz_b == 0) {
481  return block_buf_idx * resolution3 + z_v * resolution2 +
482  y_v * block_resolution + x_v;
483  } else {
484  Key key(x_b + dx_b, y_b + dy_b, z_b + dz_b);
485 
486  index_t block_buf_idx = cache.Check(key[0], key[1], key[2]);
487  if (block_buf_idx < 0) {
488  auto iter = hashmap_impl.find(key);
489  if (iter == hashmap_impl.end()) return -1;
490  block_buf_idx = iter->second;
491  cache.Update(key[0], key[1], key[2], block_buf_idx);
492  }
493 
494  return block_buf_idx * resolution3 + z_vn * resolution2 +
495  y_vn * block_resolution + x_vn;
496  }
497  };
498 
499  auto GetLinearIdxAtT = [&] OPEN3D_DEVICE(
500  float x_o, float y_o, float z_o,
501  float x_d, float y_d, float z_d, float t,
502  MiniVecCache& cache) -> index_t {
503  float x_g = x_o + t * x_d;
504  float y_g = y_o + t * y_d;
505  float z_g = z_o + t * z_d;
506 
507  // MiniVec coordinate and look up
508  index_t x_b = static_cast<index_t>(floorf(x_g / block_size));
509  index_t y_b = static_cast<index_t>(floorf(y_g / block_size));
510  index_t z_b = static_cast<index_t>(floorf(z_g / block_size));
511 
512  Key key(x_b, y_b, z_b);
513  index_t block_buf_idx = cache.Check(x_b, y_b, z_b);
514  if (block_buf_idx < 0) {
515  auto iter = hashmap_impl.find(key);
516  if (iter == hashmap_impl.end()) return -1;
517  block_buf_idx = iter->second;
518  cache.Update(x_b, y_b, z_b, block_buf_idx);
519  }
520 
521  // Voxel coordinate and look up
522  index_t x_v = index_t((x_g - x_b * block_size) / voxel_size);
523  index_t y_v = index_t((y_g - y_b * block_size) / voxel_size);
524  index_t z_v = index_t((z_g - z_b * block_size) / voxel_size);
525 
526  return block_buf_idx * resolution3 + z_v * resolution2 +
527  y_v * block_resolution + x_v;
528  };
529 
530  index_t y = workload_idx / cols;
531  index_t x = workload_idx % cols;
532 
533  const float* range = range_indexer.GetDataPtr<float>(x / 8, y / 8);
534 
535  float* depth_ptr = nullptr;
536  float* vertex_ptr = nullptr;
537  float* color_ptr = nullptr;
538  float* normal_ptr = nullptr;
539 
540  int64_t* index_ptr = nullptr;
541  bool* mask_ptr = nullptr;
542  float* interp_ratio_ptr = nullptr;
543  float* interp_ratio_dx_ptr = nullptr;
544  float* interp_ratio_dy_ptr = nullptr;
545  float* interp_ratio_dz_ptr = nullptr;
546 
547  if (vertex_indexer.GetDataPtr()) {
548  vertex_ptr = vertex_indexer.GetDataPtr<float>(x, y);
549  vertex_ptr[0] = 0;
550  vertex_ptr[1] = 0;
551  vertex_ptr[2] = 0;
552  }
553  if (depth_indexer.GetDataPtr()) {
554  depth_ptr = depth_indexer.GetDataPtr<float>(x, y);
555  depth_ptr[0] = 0;
556  }
557  if (normal_indexer.GetDataPtr()) {
558  normal_ptr = normal_indexer.GetDataPtr<float>(x, y);
559  normal_ptr[0] = 0;
560  normal_ptr[1] = 0;
561  normal_ptr[2] = 0;
562  }
563 
564  if (mask_indexer.GetDataPtr()) {
565  mask_ptr = mask_indexer.GetDataPtr<bool>(x, y);
566 #ifdef __CUDACC__
567 #pragma unroll
568 #endif
569  for (int i = 0; i < 8; ++i) {
570  mask_ptr[i] = false;
571  }
572  }
573  if (index_indexer.GetDataPtr()) {
574  index_ptr = index_indexer.GetDataPtr<int64_t>(x, y);
575 #ifdef __CUDACC__
576 #pragma unroll
577 #endif
578  for (int i = 0; i < 8; ++i) {
579  index_ptr[i] = 0;
580  }
581  }
582  if (interp_ratio_indexer.GetDataPtr()) {
583  interp_ratio_ptr = interp_ratio_indexer.GetDataPtr<float>(x, y);
584 #ifdef __CUDACC__
585 #pragma unroll
586 #endif
587  for (int i = 0; i < 8; ++i) {
588  interp_ratio_ptr[i] = 0;
589  }
590  }
591  if (interp_ratio_dx_indexer.GetDataPtr()) {
592  interp_ratio_dx_ptr =
593  interp_ratio_dx_indexer.GetDataPtr<float>(x, y);
594 #ifdef __CUDACC__
595 #pragma unroll
596 #endif
597  for (int i = 0; i < 8; ++i) {
598  interp_ratio_dx_ptr[i] = 0;
599  }
600  }
601  if (interp_ratio_dy_indexer.GetDataPtr()) {
602  interp_ratio_dy_ptr =
603  interp_ratio_dy_indexer.GetDataPtr<float>(x, y);
604 #ifdef __CUDACC__
605 #pragma unroll
606 #endif
607  for (int i = 0; i < 8; ++i) {
608  interp_ratio_dy_ptr[i] = 0;
609  }
610  }
611  if (interp_ratio_dz_indexer.GetDataPtr()) {
612  interp_ratio_dz_ptr =
613  interp_ratio_dz_indexer.GetDataPtr<float>(x, y);
614 #ifdef __CUDACC__
615 #pragma unroll
616 #endif
617  for (int i = 0; i < 8; ++i) {
618  interp_ratio_dz_ptr[i] = 0;
619  }
620  }
621 
622  if (color_indexer.GetDataPtr()) {
623  color_ptr = color_indexer.GetDataPtr<float>(x, y);
624  color_ptr[0] = 0;
625  color_ptr[1] = 0;
626  color_ptr[2] = 0;
627  }
628 
629  float t = range[0];
630  const float t_max = range[1];
631  if (t >= t_max) return;
632 
633  // Coordinates in camera and global
634  float x_c = 0, y_c = 0, z_c = 0;
635  float x_g = 0, y_g = 0, z_g = 0;
636  float x_o = 0, y_o = 0, z_o = 0;
637 
638  // Iterative ray intersection check
639  float t_prev = t;
640 
641  float tsdf_prev = -1.0f;
642  float tsdf = 1.0;
643  float w = 0.0;
644 
645  // Camera origin
646  c2w_transform_indexer.RigidTransform(0, 0, 0, &x_o, &y_o, &z_o);
647 
648  // Direction
649  c2w_transform_indexer.Unproject(static_cast<float>(x),
650  static_cast<float>(y), 1.0f, &x_c, &y_c,
651  &z_c);
652  c2w_transform_indexer.RigidTransform(x_c, y_c, z_c, &x_g, &y_g, &z_g);
653  float x_d = (x_g - x_o);
654  float y_d = (y_g - y_o);
655  float z_d = (z_g - z_o);
656 
657  MiniVecCache cache{0, 0, 0, -1};
658  bool surface_found = false;
659  while (t < t_max) {
660  index_t linear_idx =
661  GetLinearIdxAtT(x_o, y_o, z_o, x_d, y_d, z_d, t, cache);
662 
663  if (linear_idx < 0) {
664  t_prev = t;
665  t += block_size;
666  } else {
667  tsdf_prev = tsdf;
668  tsdf = tsdf_base_ptr[linear_idx];
669  w = weight_base_ptr[linear_idx];
670  if (tsdf_prev > 0 && w >= weight_threshold && tsdf <= 0) {
671  surface_found = true;
672  break;
673  }
674  t_prev = t;
675  float delta = tsdf * sdf_trunc;
676  t += delta < voxel_size ? voxel_size : delta;
677  }
678  }
679 
680  if (surface_found) {
681  float t_intersect =
682  (t * tsdf_prev - t_prev * tsdf) / (tsdf_prev - tsdf);
683  x_g = x_o + t_intersect * x_d;
684  y_g = y_o + t_intersect * y_d;
685  z_g = z_o + t_intersect * z_d;
686 
687  // Trivial vertex assignment
688  if (depth_ptr) {
689  *depth_ptr = t_intersect * depth_scale;
690  }
691  if (vertex_ptr) {
692  w2c_transform_indexer.RigidTransform(
693  x_g, y_g, z_g, vertex_ptr + 0, vertex_ptr + 1,
694  vertex_ptr + 2);
695  }
696  if (!visit_neighbors) return;
697 
698  // Trilinear interpolation
699  // TODO(wei): simplify the flow by splitting the
700  // functions given what is enabled
701  index_t x_b = static_cast<index_t>(floorf(x_g / block_size));
702  index_t y_b = static_cast<index_t>(floorf(y_g / block_size));
703  index_t z_b = static_cast<index_t>(floorf(z_g / block_size));
704  float x_v = (x_g - float(x_b) * block_size) / voxel_size;
705  float y_v = (y_g - float(y_b) * block_size) / voxel_size;
706  float z_v = (z_g - float(z_b) * block_size) / voxel_size;
707 
708  Key key(x_b, y_b, z_b);
709 
710  index_t block_buf_idx = cache.Check(x_b, y_b, z_b);
711  if (block_buf_idx < 0) {
712  auto iter = hashmap_impl.find(key);
713  if (iter == hashmap_impl.end()) return;
714  block_buf_idx = iter->second;
715  cache.Update(x_b, y_b, z_b, block_buf_idx);
716  }
717 
718  index_t x_v_floor = static_cast<index_t>(floorf(x_v));
719  index_t y_v_floor = static_cast<index_t>(floorf(y_v));
720  index_t z_v_floor = static_cast<index_t>(floorf(z_v));
721 
722  float ratio_x = x_v - float(x_v_floor);
723  float ratio_y = y_v - float(y_v_floor);
724  float ratio_z = z_v - float(z_v_floor);
725 
726  float sum_r = 0.0;
727  for (index_t k = 0; k < 8; ++k) {
728  index_t dx_v = (k & 1) > 0 ? 1 : 0;
729  index_t dy_v = (k & 2) > 0 ? 1 : 0;
730  index_t dz_v = (k & 4) > 0 ? 1 : 0;
731 
732  index_t linear_idx_k = GetLinearIdxAtP(
733  x_b, y_b, z_b, x_v_floor + dx_v, y_v_floor + dy_v,
734  z_v_floor + dz_v, block_buf_idx, cache);
735 
736  if (linear_idx_k >= 0 && weight_base_ptr[linear_idx_k] > 0) {
737  float rx = dx_v * (ratio_x) + (1 - dx_v) * (1 - ratio_x);
738  float ry = dy_v * (ratio_y) + (1 - dy_v) * (1 - ratio_y);
739  float rz = dz_v * (ratio_z) + (1 - dz_v) * (1 - ratio_z);
740  float r = rx * ry * rz;
741 
742  if (interp_ratio_ptr) {
743  interp_ratio_ptr[k] = r;
744  }
745  if (mask_ptr) {
746  mask_ptr[k] = true;
747  }
748  if (index_ptr) {
749  index_ptr[k] = linear_idx_k;
750  }
751 
752  float tsdf_k = tsdf_base_ptr[linear_idx_k];
753  float interp_ratio_dx = ry * rz * (2 * dx_v - 1);
754  float interp_ratio_dy = rx * rz * (2 * dy_v - 1);
755  float interp_ratio_dz = rx * ry * (2 * dz_v - 1);
756 
757  if (interp_ratio_dx_ptr) {
758  interp_ratio_dx_ptr[k] = interp_ratio_dx;
759  }
760  if (interp_ratio_dy_ptr) {
761  interp_ratio_dy_ptr[k] = interp_ratio_dy;
762  }
763  if (interp_ratio_dz_ptr) {
764  interp_ratio_dz_ptr[k] = interp_ratio_dz;
765  }
766 
767  if (normal_ptr) {
768  normal_ptr[0] += interp_ratio_dx * tsdf_k;
769  normal_ptr[1] += interp_ratio_dy * tsdf_k;
770  normal_ptr[2] += interp_ratio_dz * tsdf_k;
771  }
772 
773  if (color_ptr) {
774  index_t color_linear_idx = linear_idx_k * 3;
775  color_ptr[0] +=
776  r * color_base_ptr[color_linear_idx + 0];
777  color_ptr[1] +=
778  r * color_base_ptr[color_linear_idx + 1];
779  color_ptr[2] +=
780  r * color_base_ptr[color_linear_idx + 2];
781  }
782 
783  sum_r += r;
784  }
785  } // loop over 8 neighbors
786 
787  if (sum_r > 0) {
788  sum_r *= 255.0;
789  if (color_ptr) {
790  color_ptr[0] /= sum_r;
791  color_ptr[1] /= sum_r;
792  color_ptr[2] /= sum_r;
793  }
794 
795  if (normal_ptr) {
796  float norm = sqrt(normal_ptr[0] * normal_ptr[0] +
797  normal_ptr[1] * normal_ptr[1] +
798  normal_ptr[2] * normal_ptr[2]);
799  w2c_transform_indexer.Rotate(
800  -normal_ptr[0] / norm, -normal_ptr[1] / norm,
801  -normal_ptr[2] / norm, normal_ptr + 0,
802  normal_ptr + 1, normal_ptr + 2);
803  }
804  }
805  } // surface-found
806  });
807 
808 #if defined(__CUDACC__)
810 #endif
811 }
812 
813 template <typename tsdf_t, typename weight_t, typename color_t>
814 #if defined(__CUDACC__)
815 void ExtractPointCloudCUDA
816 #else
818 #endif
819  (const core::Tensor& indices,
820  const core::Tensor& nb_indices,
821  const core::Tensor& nb_masks,
822  const core::Tensor& block_keys,
823  const TensorMap& block_value_map,
825  core::Tensor& normals,
826  core::Tensor& colors,
827  index_t resolution,
828  float voxel_size,
829  float weight_threshold,
830  int& valid_size) {
831  core::Device device = block_keys.GetDevice();
832 
833  // Parameters
834  index_t resolution2 = resolution * resolution;
835  index_t resolution3 = resolution2 * resolution;
836 
837  // Shape / transform indexers, no data involved
838  ArrayIndexer voxel_indexer({resolution, resolution, resolution});
839 
840  // Real data indexer
841  ArrayIndexer block_keys_indexer(block_keys, 1);
842  ArrayIndexer nb_block_masks_indexer(nb_masks, 2);
843  ArrayIndexer nb_block_indices_indexer(nb_indices, 2);
844 
845  // Plain arrays that does not require indexers
846  const index_t* indices_ptr = indices.GetDataPtr<index_t>();
847 
848  if (!block_value_map.Contains("tsdf") ||
849  !block_value_map.Contains("weight")) {
851  "TSDF and/or weight not allocated in blocks, please implement "
852  "customized integration.");
853  }
854  const tsdf_t* tsdf_base_ptr =
855  block_value_map.at("tsdf").GetDataPtr<tsdf_t>();
856  const weight_t* weight_base_ptr =
857  block_value_map.at("weight").GetDataPtr<weight_t>();
858  const color_t* color_base_ptr = nullptr;
859  if (block_value_map.Contains("color")) {
860  color_base_ptr = block_value_map.at("color").GetDataPtr<color_t>();
861  }
862 
863  index_t n_blocks = indices.GetLength();
864  index_t n = n_blocks * resolution3;
865 
866  // Output
867 #if defined(__CUDACC__)
868  core::Tensor count(std::vector<index_t>{0}, {1}, core::Int32,
869  block_keys.GetDevice());
870  index_t* count_ptr = count.GetDataPtr<index_t>();
871 #else
872  std::atomic<index_t> count_atomic(0);
873  std::atomic<index_t>* count_ptr = &count_atomic;
874 #endif
875 
876  if (valid_size < 0) {
878  "No estimated max point cloud size provided, using a 2-pass "
879  "estimation. Surface extraction could be slow.");
880  // This pass determines valid number of points.
881 
882  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t workload_idx) {
883  auto GetLinearIdx = [&] OPEN3D_DEVICE(
884  index_t xo, index_t yo, index_t zo,
885  index_t curr_block_idx) -> index_t {
886  return DeviceGetLinearIdx(xo, yo, zo, curr_block_idx,
887  resolution, nb_block_masks_indexer,
888  nb_block_indices_indexer);
889  };
890 
891  // Natural index (0, N) -> (block_idx,
892  // voxel_idx)
893  index_t workload_block_idx = workload_idx / resolution3;
894  index_t block_idx = indices_ptr[workload_block_idx];
895  index_t voxel_idx = workload_idx % resolution3;
896 
897  // voxel_idx -> (x_voxel, y_voxel, z_voxel)
898  index_t xv, yv, zv;
899  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
900 
901  index_t linear_idx = block_idx * resolution3 + voxel_idx;
902  float tsdf_o = tsdf_base_ptr[linear_idx];
903  float weight_o = weight_base_ptr[linear_idx];
904  if (weight_o <= weight_threshold) return;
905 
906  // Enumerate x-y-z directions
907  for (index_t i = 0; i < 3; ++i) {
908  index_t linear_idx_i =
909  GetLinearIdx(xv + (i == 0), yv + (i == 1),
910  zv + (i == 2), workload_block_idx);
911  if (linear_idx_i < 0) continue;
912 
913  float tsdf_i = tsdf_base_ptr[linear_idx_i];
914  float weight_i = weight_base_ptr[linear_idx_i];
915  if (weight_i > weight_threshold && tsdf_i * tsdf_o < 0) {
916  OPEN3D_ATOMIC_ADD(count_ptr, 1);
917  }
918  }
919  });
920 
921 #if defined(__CUDACC__)
922  valid_size = count[0].Item<index_t>();
923  count[0] = 0;
924 #else
925  valid_size = (*count_ptr).load();
926  (*count_ptr) = 0;
927 #endif
928  }
929 
930  if (points.GetLength() == 0) {
931  points = core::Tensor({valid_size, 3}, core::Float32, device);
932  }
933  ArrayIndexer point_indexer(points, 1);
934 
935  // Normals
936  ArrayIndexer normal_indexer;
937  normals = core::Tensor({valid_size, 3}, core::Float32, device);
938  normal_indexer = ArrayIndexer(normals, 1);
939 
940  // This pass extracts exact surface points.
941 
942  // Colors
943  ArrayIndexer color_indexer;
944  if (color_base_ptr) {
945  colors = core::Tensor({valid_size, 3}, core::Float32, device);
946  color_indexer = ArrayIndexer(colors, 1);
947  }
948 
949  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t workload_idx) {
950  auto GetLinearIdx = [&] OPEN3D_DEVICE(
951  index_t xo, index_t yo, index_t zo,
952  index_t curr_block_idx) -> index_t {
953  return DeviceGetLinearIdx(xo, yo, zo, curr_block_idx, resolution,
954  nb_block_masks_indexer,
955  nb_block_indices_indexer);
956  };
957 
958  auto GetNormal = [&] OPEN3D_DEVICE(index_t xo, index_t yo, index_t zo,
959  index_t curr_block_idx, float* n) {
960  return DeviceGetNormal<tsdf_t>(
961  tsdf_base_ptr, xo, yo, zo, curr_block_idx, n, resolution,
962  nb_block_masks_indexer, nb_block_indices_indexer);
963  };
964 
965  // Natural index (0, N) -> (block_idx, voxel_idx)
966  index_t workload_block_idx = workload_idx / resolution3;
967  index_t block_idx = indices_ptr[workload_block_idx];
968  index_t voxel_idx = workload_idx % resolution3;
969 
971  // block_idx -> (x_block, y_block, z_block)
972  index_t* block_key_ptr =
973  block_keys_indexer.GetDataPtr<index_t>(block_idx);
974  index_t xb = block_key_ptr[0];
975  index_t yb = block_key_ptr[1];
976  index_t zb = block_key_ptr[2];
977 
978  // voxel_idx -> (x_voxel, y_voxel, z_voxel)
979  index_t xv, yv, zv;
980  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
981 
982  index_t linear_idx = block_idx * resolution3 + voxel_idx;
983  float tsdf_o = tsdf_base_ptr[linear_idx];
984  float weight_o = weight_base_ptr[linear_idx];
985  if (weight_o <= weight_threshold) return;
986 
987  float no[3] = {0}, ne[3] = {0};
988 
989  // Get normal at origin
990  GetNormal(xv, yv, zv, workload_block_idx, no);
991 
992  index_t x = xb * resolution + xv;
993  index_t y = yb * resolution + yv;
994  index_t z = zb * resolution + zv;
995 
996  // Enumerate x-y-z axis
997  for (index_t i = 0; i < 3; ++i) {
998  index_t linear_idx_i =
999  GetLinearIdx(xv + (i == 0), yv + (i == 1), zv + (i == 2),
1000  workload_block_idx);
1001  if (linear_idx_i < 0) continue;
1002 
1003  float tsdf_i = tsdf_base_ptr[linear_idx_i];
1004  float weight_i = weight_base_ptr[linear_idx_i];
1005  if (weight_i > weight_threshold && tsdf_i * tsdf_o < 0) {
1006  float ratio = (0 - tsdf_o) / (tsdf_i - tsdf_o);
1007 
1008  index_t idx = OPEN3D_ATOMIC_ADD(count_ptr, 1);
1009  if (idx >= valid_size) {
1010  printf("Point cloud size larger than "
1011  "estimated, please increase the "
1012  "estimation!\n");
1013  return;
1014  }
1015 
1016  float* point_ptr = point_indexer.GetDataPtr<float>(idx);
1017  point_ptr[0] = voxel_size * (x + ratio * int(i == 0));
1018  point_ptr[1] = voxel_size * (y + ratio * int(i == 1));
1019  point_ptr[2] = voxel_size * (z + ratio * int(i == 2));
1020 
1021  // Get normal at edge and interpolate
1022  float* normal_ptr = normal_indexer.GetDataPtr<float>(idx);
1023  GetNormal(xv + (i == 0), yv + (i == 1), zv + (i == 2),
1024  workload_block_idx, ne);
1025  float nx = (1 - ratio) * no[0] + ratio * ne[0];
1026  float ny = (1 - ratio) * no[1] + ratio * ne[1];
1027  float nz = (1 - ratio) * no[2] + ratio * ne[2];
1028  float norm = static_cast<float>(
1029  sqrt(nx * nx + ny * ny + nz * nz) + 1e-5);
1030  normal_ptr[0] = nx / norm;
1031  normal_ptr[1] = ny / norm;
1032  normal_ptr[2] = nz / norm;
1033 
1034  if (color_base_ptr) {
1035  float* color_ptr = color_indexer.GetDataPtr<float>(idx);
1036  const color_t* color_o_ptr =
1037  color_base_ptr + 3 * linear_idx;
1038  float r_o = color_o_ptr[0];
1039  float g_o = color_o_ptr[1];
1040  float b_o = color_o_ptr[2];
1041 
1042  const color_t* color_i_ptr =
1043  color_base_ptr + 3 * linear_idx_i;
1044  float r_i = color_i_ptr[0];
1045  float g_i = color_i_ptr[1];
1046  float b_i = color_i_ptr[2];
1047 
1048  color_ptr[0] = ((1 - ratio) * r_o + ratio * r_i) / 255.0f;
1049  color_ptr[1] = ((1 - ratio) * g_o + ratio * g_i) / 255.0f;
1050  color_ptr[2] = ((1 - ratio) * b_o + ratio * b_i) / 255.0f;
1051  }
1052  }
1053  }
1054  });
1055 
1056 #if defined(__CUDACC__)
1057  index_t total_count = count.Item<index_t>();
1058 #else
1059  index_t total_count = (*count_ptr).load();
1060 #endif
1061 
1062  utility::LogDebug("{} vertices extracted", total_count);
1063  valid_size = total_count;
1064 
1065 #if defined(BUILD_CUDA_MODULE) && defined(__CUDACC__)
1067 #endif
1068 }
1069 
1070 template <typename tsdf_t, typename weight_t, typename color_t>
1071 #if defined(__CUDACC__)
1072 void ExtractTriangleMeshCUDA
1073 #else
1075 #endif
1076  (const core::Tensor& block_indices,
1077  const core::Tensor& inv_block_indices,
1078  const core::Tensor& nb_block_indices,
1079  const core::Tensor& nb_block_masks,
1080  const core::Tensor& block_keys,
1081  const TensorMap& block_value_map,
1082  core::Tensor& vertices,
1083  core::Tensor& triangles,
1084  core::Tensor& vertex_normals,
1085  core::Tensor& vertex_colors,
1086  index_t block_resolution,
1087  float voxel_size,
1088  float weight_threshold,
1089  index_t& vertex_count) {
1090  core::Device device = block_indices.GetDevice();
1091 
1092  index_t resolution = block_resolution;
1093  index_t resolution3 = resolution * resolution * resolution;
1094 
1095  // Shape / transform indexers, no data involved
1096  ArrayIndexer voxel_indexer({resolution, resolution, resolution});
1097  index_t n_blocks = static_cast<index_t>(block_indices.GetLength());
1098 
1099  // TODO(wei): profile performance by replacing the table to a hashmap.
1100  // Voxel-wise mesh info. 4 channels correspond to:
1101  // 3 edges' corresponding vertex index + 1 table index.
1102  core::Tensor mesh_structure;
1103  try {
1104  mesh_structure = core::Tensor::Zeros(
1105  {n_blocks, resolution, resolution, resolution, 4}, core::Int32,
1106  device);
1107  } catch (const std::runtime_error&) {
1109  "[MeshExtractionKernel] Unable to allocate assistance mesh "
1110  "structure for Marching "
1111  "Cubes with {} active voxel blocks. Please consider using a "
1112  "larger voxel size (currently {}) for TSDF "
1113  "integration, or using tsdf_volume.cpu() to perform mesh "
1114  "extraction on CPU.",
1115  n_blocks, voxel_size);
1116  }
1117 
1118  // Real data indexer
1119  ArrayIndexer mesh_structure_indexer(mesh_structure, 4);
1120  ArrayIndexer nb_block_masks_indexer(nb_block_masks, 2);
1121  ArrayIndexer nb_block_indices_indexer(nb_block_indices, 2);
1122 
1123  // Plain arrays that does not require indexers
1124  const index_t* indices_ptr = block_indices.GetDataPtr<index_t>();
1125  const index_t* inv_indices_ptr = inv_block_indices.GetDataPtr<index_t>();
1126 
1127  if (!block_value_map.Contains("tsdf") ||
1128  !block_value_map.Contains("weight")) {
1130  "TSDF and/or weight not allocated in blocks, please implement "
1131  "customized integration.");
1132  }
1133  const tsdf_t* tsdf_base_ptr =
1134  block_value_map.at("tsdf").GetDataPtr<tsdf_t>();
1135  const weight_t* weight_base_ptr =
1136  block_value_map.at("weight").GetDataPtr<weight_t>();
1137  const color_t* color_base_ptr = nullptr;
1138  if (block_value_map.Contains("color")) {
1139  color_base_ptr = block_value_map.at("color").GetDataPtr<color_t>();
1140  }
1141 
1142  index_t n = n_blocks * resolution3;
1143  // Pass 0: analyze mesh structure, set up one-on-one correspondences
1144  // from edges to vertices.
1145 
1146  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t widx) {
1147  auto GetLinearIdx = [&] OPEN3D_DEVICE(
1148  index_t xo, index_t yo, index_t zo,
1149  index_t curr_block_idx) -> index_t {
1150  return DeviceGetLinearIdx(xo, yo, zo, curr_block_idx,
1151  static_cast<index_t>(resolution),
1152  nb_block_masks_indexer,
1153  nb_block_indices_indexer);
1154  };
1155 
1156  // Natural index (0, N) -> (block_idx, voxel_idx)
1157  index_t workload_block_idx = widx / resolution3;
1158  index_t voxel_idx = widx % resolution3;
1159 
1160  // voxel_idx -> (x_voxel, y_voxel, z_voxel)
1161  index_t xv, yv, zv;
1162  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
1163 
1164  // Check per-vertex sign in the cube to determine cube
1165  // type
1166  index_t table_idx = 0;
1167  for (index_t i = 0; i < 8; ++i) {
1168  index_t linear_idx_i =
1169  GetLinearIdx(xv + vtx_shifts[i][0], yv + vtx_shifts[i][1],
1170  zv + vtx_shifts[i][2], workload_block_idx);
1171  if (linear_idx_i < 0) return;
1172 
1173  float tsdf_i = tsdf_base_ptr[linear_idx_i];
1174  float weight_i = weight_base_ptr[linear_idx_i];
1175  if (weight_i <= weight_threshold) return;
1176 
1177  table_idx |= ((tsdf_i < 0) ? (1 << i) : 0);
1178  }
1179 
1180  index_t* mesh_struct_ptr = mesh_structure_indexer.GetDataPtr<index_t>(
1181  xv, yv, zv, workload_block_idx);
1182  mesh_struct_ptr[3] = table_idx;
1183 
1184  if (table_idx == 0 || table_idx == 255) return;
1185 
1186  // Check per-edge sign determine the cube type
1187  index_t edges_with_vertices = edge_table[table_idx];
1188  for (index_t i = 0; i < 12; ++i) {
1189  if (edges_with_vertices & (1 << i)) {
1190  index_t xv_i = xv + edge_shifts[i][0];
1191  index_t yv_i = yv + edge_shifts[i][1];
1192  index_t zv_i = zv + edge_shifts[i][2];
1193  index_t edge_i = edge_shifts[i][3];
1194 
1195  index_t dxb = xv_i / resolution;
1196  index_t dyb = yv_i / resolution;
1197  index_t dzb = zv_i / resolution;
1198 
1199  index_t nb_idx = (dxb + 1) + (dyb + 1) * 3 + (dzb + 1) * 9;
1200 
1201  index_t block_idx_i =
1202  *nb_block_indices_indexer.GetDataPtr<index_t>(
1203  workload_block_idx, nb_idx);
1204  index_t* mesh_ptr_i =
1205  mesh_structure_indexer.GetDataPtr<index_t>(
1206  xv_i - dxb * resolution,
1207  yv_i - dyb * resolution,
1208  zv_i - dzb * resolution,
1209  inv_indices_ptr[block_idx_i]);
1210 
1211  // Non-atomic write, but we are safe
1212  mesh_ptr_i[edge_i] = -1;
1213  }
1214  }
1215  });
1216 
1217  // Pass 1: determine valid number of vertices (if not preset)
1218 #if defined(__CUDACC__)
1219  core::Tensor count(std::vector<index_t>{0}, {}, core::Int32, device);
1220 
1221  index_t* count_ptr = count.GetDataPtr<index_t>();
1222 #else
1223  std::atomic<index_t> count_atomic(0);
1224  std::atomic<index_t>* count_ptr = &count_atomic;
1225 #endif
1226 
1227  if (vertex_count < 0) {
1228  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t widx) {
1229  // Natural index (0, N) -> (block_idx, voxel_idx)
1230  index_t workload_block_idx = widx / resolution3;
1231  index_t voxel_idx = widx % resolution3;
1232 
1233  // voxel_idx -> (x_voxel, y_voxel, z_voxel)
1234  index_t xv, yv, zv;
1235  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
1236 
1237  // Obtain voxel's mesh struct ptr
1238  index_t* mesh_struct_ptr =
1239  mesh_structure_indexer.GetDataPtr<index_t>(
1240  xv, yv, zv, workload_block_idx);
1241 
1242  // Early quit -- no allocated vertex to compute
1243  if (mesh_struct_ptr[0] != -1 && mesh_struct_ptr[1] != -1 &&
1244  mesh_struct_ptr[2] != -1) {
1245  return;
1246  }
1247 
1248  // Enumerate 3 edges in the voxel
1249  for (index_t e = 0; e < 3; ++e) {
1250  index_t vertex_idx = mesh_struct_ptr[e];
1251  if (vertex_idx != -1) continue;
1252 
1253  OPEN3D_ATOMIC_ADD(count_ptr, 1);
1254  }
1255  });
1256 
1257 #if defined(__CUDACC__)
1258  vertex_count = count.Item<index_t>();
1259 #else
1260  vertex_count = (*count_ptr).load();
1261 #endif
1262  }
1263 
1264  utility::LogDebug("Total vertex count = {}", vertex_count);
1265  vertices = core::Tensor({vertex_count, 3}, core::Float32, device);
1266 
1267  vertex_normals = core::Tensor({vertex_count, 3}, core::Float32, device);
1268  ArrayIndexer normal_indexer = ArrayIndexer(vertex_normals, 1);
1269 
1270  ArrayIndexer color_indexer;
1271  if (color_base_ptr) {
1272  vertex_colors = core::Tensor({vertex_count, 3}, core::Float32, device);
1273  color_indexer = ArrayIndexer(vertex_colors, 1);
1274  }
1275 
1276  ArrayIndexer block_keys_indexer(block_keys, 1);
1277  ArrayIndexer vertex_indexer(vertices, 1);
1278 
1279 #if defined(__CUDACC__)
1280  count = core::Tensor(std::vector<index_t>{0}, {}, core::Int32, device);
1281  count_ptr = count.GetDataPtr<index_t>();
1282 #else
1283  (*count_ptr) = 0;
1284 #endif
1285 
1286  // Pass 2: extract vertices.
1287 
1288  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t widx) {
1289  auto GetLinearIdx = [&] OPEN3D_DEVICE(
1290  index_t xo, index_t yo, index_t zo,
1291  index_t curr_block_idx) -> index_t {
1292  return DeviceGetLinearIdx(xo, yo, zo, curr_block_idx, resolution,
1293  nb_block_masks_indexer,
1294  nb_block_indices_indexer);
1295  };
1296 
1297  auto GetNormal = [&] OPEN3D_DEVICE(index_t xo, index_t yo, index_t zo,
1298  index_t curr_block_idx, float* n) {
1299  return DeviceGetNormal<tsdf_t>(
1300  tsdf_base_ptr, xo, yo, zo, curr_block_idx, n, resolution,
1301  nb_block_masks_indexer, nb_block_indices_indexer);
1302  };
1303 
1304  // Natural index (0, N) -> (block_idx, voxel_idx)
1305  index_t workload_block_idx = widx / resolution3;
1306  index_t block_idx = indices_ptr[workload_block_idx];
1307  index_t voxel_idx = widx % resolution3;
1308 
1309  // block_idx -> (x_block, y_block, z_block)
1310  index_t* block_key_ptr =
1311  block_keys_indexer.GetDataPtr<index_t>(block_idx);
1312  index_t xb = block_key_ptr[0];
1313  index_t yb = block_key_ptr[1];
1314  index_t zb = block_key_ptr[2];
1315 
1316  // voxel_idx -> (x_voxel, y_voxel, z_voxel)
1317  index_t xv, yv, zv;
1318  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
1319 
1320  // global coordinate (in voxels)
1321  index_t x = xb * resolution + xv;
1322  index_t y = yb * resolution + yv;
1323  index_t z = zb * resolution + zv;
1324 
1325  // Obtain voxel's mesh struct ptr
1326  index_t* mesh_struct_ptr = mesh_structure_indexer.GetDataPtr<index_t>(
1327  xv, yv, zv, workload_block_idx);
1328 
1329  // Early quit -- no allocated vertex to compute
1330  if (mesh_struct_ptr[0] != -1 && mesh_struct_ptr[1] != -1 &&
1331  mesh_struct_ptr[2] != -1) {
1332  return;
1333  }
1334 
1335  // Obtain voxel ptr
1336  index_t linear_idx = resolution3 * block_idx + voxel_idx;
1337  float tsdf_o = tsdf_base_ptr[linear_idx];
1338 
1339  float no[3] = {0}, ne[3] = {0};
1340 
1341  // Get normal at origin
1342  GetNormal(xv, yv, zv, workload_block_idx, no);
1343 
1344  // Enumerate 3 edges in the voxel
1345  for (index_t e = 0; e < 3; ++e) {
1346  index_t vertex_idx = mesh_struct_ptr[e];
1347  if (vertex_idx != -1) continue;
1348 
1349  index_t linear_idx_e =
1350  GetLinearIdx(xv + (e == 0), yv + (e == 1), zv + (e == 2),
1351  workload_block_idx);
1352  OPEN3D_ASSERT(linear_idx_e > 0 &&
1353  "Internal error: GetVoxelAt returns nullptr.");
1354  float tsdf_e = tsdf_base_ptr[linear_idx_e];
1355  float ratio = (0 - tsdf_o) / (tsdf_e - tsdf_o);
1356 
1357  index_t idx = OPEN3D_ATOMIC_ADD(count_ptr, 1);
1358  mesh_struct_ptr[e] = idx;
1359 
1360  float ratio_x = ratio * index_t(e == 0);
1361  float ratio_y = ratio * index_t(e == 1);
1362  float ratio_z = ratio * index_t(e == 2);
1363 
1364  float* vertex_ptr = vertex_indexer.GetDataPtr<float>(idx);
1365  vertex_ptr[0] = voxel_size * (x + ratio_x);
1366  vertex_ptr[1] = voxel_size * (y + ratio_y);
1367  vertex_ptr[2] = voxel_size * (z + ratio_z);
1368 
1369  // Get normal at edge and interpolate
1370  float* normal_ptr = normal_indexer.GetDataPtr<float>(idx);
1371  GetNormal(xv + (e == 0), yv + (e == 1), zv + (e == 2),
1372  workload_block_idx, ne);
1373  float nx = (1 - ratio) * no[0] + ratio * ne[0];
1374  float ny = (1 - ratio) * no[1] + ratio * ne[1];
1375  float nz = (1 - ratio) * no[2] + ratio * ne[2];
1376  float norm = static_cast<float>(sqrt(nx * nx + ny * ny + nz * nz) +
1377  1e-5);
1378  normal_ptr[0] = nx / norm;
1379  normal_ptr[1] = ny / norm;
1380  normal_ptr[2] = nz / norm;
1381 
1382  if (color_base_ptr) {
1383  float* color_ptr = color_indexer.GetDataPtr<float>(idx);
1384  float r_o = color_base_ptr[linear_idx * 3 + 0];
1385  float g_o = color_base_ptr[linear_idx * 3 + 1];
1386  float b_o = color_base_ptr[linear_idx * 3 + 2];
1387 
1388  float r_e = color_base_ptr[linear_idx_e * 3 + 0];
1389  float g_e = color_base_ptr[linear_idx_e * 3 + 1];
1390  float b_e = color_base_ptr[linear_idx_e * 3 + 2];
1391 
1392  color_ptr[0] = ((1 - ratio) * r_o + ratio * r_e) / 255.0f;
1393  color_ptr[1] = ((1 - ratio) * g_o + ratio * g_e) / 255.0f;
1394  color_ptr[2] = ((1 - ratio) * b_o + ratio * b_e) / 255.0f;
1395  }
1396  }
1397  });
1398 
1399  // Pass 3: connect vertices and form triangles.
1400  index_t triangle_count = vertex_count * 3;
1401  triangles = core::Tensor({triangle_count, 3}, core::Int32, device);
1402  ArrayIndexer triangle_indexer(triangles, 1);
1403 
1404 #if defined(__CUDACC__)
1405  count = core::Tensor(std::vector<index_t>{0}, {}, core::Int32, device);
1406  count_ptr = count.GetDataPtr<index_t>();
1407 #else
1408  (*count_ptr) = 0;
1409 #endif
1410  core::ParallelFor(device, n, [=] OPEN3D_DEVICE(index_t widx) {
1411  // Natural index (0, N) -> (block_idx, voxel_idx)
1412  index_t workload_block_idx = widx / resolution3;
1413  index_t voxel_idx = widx % resolution3;
1414 
1415  // voxel_idx -> (x_voxel, y_voxel, z_voxel)
1416  index_t xv, yv, zv;
1417  voxel_indexer.WorkloadToCoord(voxel_idx, &xv, &yv, &zv);
1418 
1419  // Obtain voxel's mesh struct ptr
1420  index_t* mesh_struct_ptr = mesh_structure_indexer.GetDataPtr<index_t>(
1421  xv, yv, zv, workload_block_idx);
1422 
1423  index_t table_idx = mesh_struct_ptr[3];
1424  if (tri_count[table_idx] == 0) return;
1425 
1426  for (index_t tri = 0; tri < 16; tri += 3) {
1427  if (tri_table[table_idx][tri] == -1) return;
1428 
1429  index_t tri_idx = OPEN3D_ATOMIC_ADD(count_ptr, 1);
1430 
1431  for (index_t vertex = 0; vertex < 3; ++vertex) {
1432  index_t edge = tri_table[table_idx][tri + vertex];
1433 
1434  index_t xv_i = xv + edge_shifts[edge][0];
1435  index_t yv_i = yv + edge_shifts[edge][1];
1436  index_t zv_i = zv + edge_shifts[edge][2];
1437  index_t edge_i = edge_shifts[edge][3];
1438 
1439  index_t dxb = xv_i / resolution;
1440  index_t dyb = yv_i / resolution;
1441  index_t dzb = zv_i / resolution;
1442 
1443  index_t nb_idx = (dxb + 1) + (dyb + 1) * 3 + (dzb + 1) * 9;
1444 
1445  index_t block_idx_i =
1446  *nb_block_indices_indexer.GetDataPtr<index_t>(
1447  workload_block_idx, nb_idx);
1448  index_t* mesh_struct_ptr_i =
1449  mesh_structure_indexer.GetDataPtr<index_t>(
1450  xv_i - dxb * resolution,
1451  yv_i - dyb * resolution,
1452  zv_i - dzb * resolution,
1453  inv_indices_ptr[block_idx_i]);
1454 
1455  index_t* triangle_ptr =
1456  triangle_indexer.GetDataPtr<index_t>(tri_idx);
1457  triangle_ptr[2 - vertex] = mesh_struct_ptr_i[edge_i];
1458  }
1459  }
1460  });
1461 
1462 #if defined(__CUDACC__)
1463  triangle_count = count.Item<index_t>();
1464 #else
1465  triangle_count = (*count_ptr).load();
1466 #endif
1467  utility::LogDebug("Total triangle count = {}", triangle_count);
1468  triangles = triangles.Slice(0, 0, triangle_count);
1469 }
1470 
1471 } // namespace voxel_grid
1472 } // namespace kernel
1473 } // namespace geometry
1474 } // namespace t
1475 } // namespace open3d
void RayCastCPU(std::shared_ptr< core::HashMap > &hashmap, const TensorMap &block_value_map, const core::Tensor &range_map, TensorMap &renderings_map, const core::Tensor &intrinsics, const core::Tensor &extrinsics, index_t h, index_t w, index_t block_resolution, float voxel_size, float sdf_trunc, float depth_scale, float depth_min, float depth_max, float weight_threshold)
Definition: VoxelBlockGridImpl.h:325
OPEN3D_HOST_DEVICE void Rotate(float x_in, float y_in, float z_in, float *x_out, float *y_out, float *z_out) const
Transform a 3D coordinate in camera coordinate to world coordinate.
Definition: GeometryIndexer.h:100
Definition: StdGPUHashBackend.h:134
Definition: GeometryIndexer.h:180
void IntegrateCPU(const core::Tensor &depth, const core::Tensor &color, const core::Tensor &block_indices, const core::Tensor &block_keys, TensorMap &block_value_map, const core::Tensor &intrinsics, const core::Tensor &extrinsics, index_t resolution, float voxel_size, float sdf_trunc, float depth_scale, float depth_max)
Definition: VoxelBlockGridImpl.h:168
TArrayIndexer< index_t > ArrayIndexer
Definition: VoxelBlockGridImpl.h:50
OPEN3D_DEVICE void DeviceGetNormal(const tsdf_t *tsdf_base_ptr, index_t xo, index_t yo, index_t zo, index_t curr_block_idx, float *n, index_t resolution, const ArrayIndexer &nb_block_masks_indexer, const ArrayIndexer &nb_block_indices_indexer)
Definition: VoxelBlockGridImpl.h:131
OPEN3D_HOST_DEVICE void Project(float x_in, float y_in, float z_in, float *u_out, float *v_out) const
Project a 3D coordinate in camera coordinate to a 2D uv coordinate.
Definition: GeometryIndexer.h:119
OPEN3D_HOST_DEVICE void * GetDataPtr() const
Definition: GeometryIndexer.h:335
OPEN3D_HOST_DEVICE int Sign(int x)
Definition: GeometryMacros.h:96
void ExtractTriangleMeshCPU(const core::Tensor &block_indices, const core::Tensor &inv_block_indices, const core::Tensor &nb_block_indices, const core::Tensor &nb_block_masks, const core::Tensor &block_keys, const TensorMap &block_value_map, core::Tensor &vertices, core::Tensor &triangles, core::Tensor &vertex_normals, core::Tensor &vertex_colors, index_t block_resolution, float voxel_size, float weight_threshold, index_t &vertex_count)
Definition: VoxelBlockGridImpl.h:1076
Definition: Dispatch.h:129
Helper class for converting coordinates/indices between 3D/3D, 3D/2D, 2D/3D.
Definition: GeometryIndexer.h:44
void OPEN3D_DEVICE Update(index_t xin, index_t yin, index_t zin, index_t block_idx_in)
Definition: VoxelBlockGridImpl.h:308
void ParallelFor(const Device &device, int64_t n, const func_t &func)
Definition: ParallelFor.h:122
const Dtype Float32
Definition: Dtype.cpp:61
int points
Definition: FilePCD.cpp:73
Device GetDevice() const
Definition: Tensor.cpp:1365
uint32_t buf_index_t
Definition: HashBackendBuffer.h:63
OPEN3D_DEVICE index_t DeviceGetLinearIdx(index_t xo, index_t yo, index_t zo, index_t curr_block_idx, index_t resolution, const ArrayIndexer &nb_block_masks_indexer, const ArrayIndexer &nb_block_indices_indexer)
Definition: VoxelBlockGridImpl.h:102
#define OPEN3D_DEVICE
Definition: CUDAUtils.h:64
int count
Definition: FilePCD.cpp:61
OPEN3D_HOST_DEVICE void Unproject(float u_in, float v_in, float d_in, float *x_out, float *y_out, float *z_out) const
Unproject a 2D uv coordinate with depth to 3D in camera coordinate.
Definition: GeometryIndexer.h:130
void ExtractPointCloudCPU(const core::Tensor &block_indices, const core::Tensor &nb_block_indices, const core::Tensor &nb_block_masks, const core::Tensor &block_keys, const TensorMap &block_value_map, core::Tensor &points, core::Tensor &normals, core::Tensor &colors, index_t block_resolution, float voxel_size, float weight_threshold, index_t &valid_size)
Definition: VoxelBlockGridImpl.h:819
#define OPEN3D_ATOMIC_ADD(X, Y)
Definition: GeometryMacros.h:58
OPEN3D_HOST_DEVICE void RigidTransform(float x_in, float y_in, float z_in, float *x_out, float *y_out, float *z_out) const
Transform a 3D coordinate in camera coordinate to world coordinate.
Definition: GeometryIndexer.h:81
void Synchronize()
Definition: CUDAUtils.cpp:78
Definition: Dispatch.h:113
index_t OPEN3D_DEVICE Check(index_t xin, index_t yin, index_t zin)
Definition: VoxelBlockGridImpl.h:304
#define LogDebug(...)
Definition: Logging.h:103
math::float4 color
Definition: LineSetBuffers.cpp:64
core::Tensor InverseTransformation(const core::Tensor &T)
TODO(wei): find a proper place for such functionalities.
Definition: Utility.h:96
const Dtype Int32
Definition: Dtype.cpp:65
Definition: Device.h:39
void GetVoxelCoordinatesAndFlattenedIndicesCPU(const core::Tensor &buf_indices, const core::Tensor &block_keys, core::Tensor &voxel_coords, core::Tensor &flattened_indices, index_t block_resolution, float voxel_size)
Definition: VoxelBlockGridImpl.h:57
Definition: TBBHashBackend.h:41
int index_t
Definition: VoxelBlockGrid.h:41
const char const char value recording_handle imu_sample recording_handle uint8_t size_t data_size k4a_record_configuration_t config target_format k4a_capture_t capture_handle k4a_imu_sample_t imu_sample playback_handle k4a_logging_message_cb_t void min_level device_handle k4a_imu_sample_t timeout_in_ms capture_handle capture_handle capture_handle image_handle temperature_c int
Definition: K4aPlugin.cpp:479
static Tensor Zeros(const SizeVector &shape, Dtype dtype, const Device &device=Device("CPU:0"))
Create a tensor fill with zeros.
Definition: Tensor.cpp:380
Definition: VoxelBlockGridImpl.h:298
Definition: PinholeCameraIntrinsic.cpp:35
const char const char value recording_handle imu_sample recording_handle uint8_t size_t data_size k4a_record_configuration_t config target_format k4a_capture_t capture_handle k4a_imu_sample_t imu_sample playback_handle k4a_logging_message_cb_t void min_level device_handle k4a_imu_sample_t timeout_in_ms capture_handle capture_handle capture_handle image_handle float
Definition: K4aPlugin.cpp:465
T * GetDataPtr()
Definition: Tensor.h:1074
#define OPEN3D_ASSERT(...)
Definition: Macro.h:67
Definition: MiniVec.h:43
index_t x
Definition: VoxelBlockGridImpl.h:299
index_t z
Definition: VoxelBlockGridImpl.h:301
index_t y
Definition: VoxelBlockGridImpl.h:300
int64_t GetLength() const
Definition: Tensor.h:1055
OPEN3D_HOST_DEVICE bool InBoundary(float x, float y) const
Definition: GeometryIndexer.h:314
index_t block_idx
Definition: VoxelBlockGridImpl.h:302
#define LogError(...)
Definition: Logging.h:72
Definition: TensorMap.h:49