Tpetra parallel linear algebra Version of the Day
Loading...
Searching...
No Matches
Tpetra_CrsGraph_def.hpp
Go to the documentation of this file.
1// @HEADER
2// *****************************************************************************
3// Tpetra: Templated Linear Algebra Services Package
4//
5// Copyright 2008 NTESS and the Tpetra contributors.
6// SPDX-License-Identifier: BSD-3-Clause
7// *****************************************************************************
8// @HEADER
9
10#ifndef TPETRA_CRSGRAPH_DEF_HPP
11#define TPETRA_CRSGRAPH_DEF_HPP
12
15
16#ifdef KOKKOS_ENABLE_SYCL
17#include <sycl/sycl.hpp>
18#endif
19
20#include "Tpetra_Details_Behavior.hpp"
21#include "Tpetra_Details_computeOffsets.hpp"
22#include "Tpetra_Details_copyOffsets.hpp"
23#include "Tpetra_Details_gathervPrint.hpp"
24#include "Tpetra_Details_getGraphDiagOffsets.hpp"
25#include "Tpetra_Details_getGraphOffRankOffsets.hpp"
26#include "Tpetra_Details_makeColMap.hpp"
27#include "Tpetra_Details_Profiling.hpp"
28#include "Tpetra_Details_getEntryOnHost.hpp"
29#include "Tpetra_Details_crsUtils.hpp"
30#include "Tpetra_Distributor.hpp"
31#include "Teuchos_SerialDenseMatrix.hpp"
32#include "Tpetra_Vector.hpp"
33#include "Tpetra_Import_Util.hpp"
34#include "Tpetra_Import_Util2.hpp"
35#include "Tpetra_Details_packCrsGraph.hpp"
36#include "Tpetra_Details_unpackCrsGraphAndCombine.hpp"
37#include "Tpetra_Details_CrsPadding.hpp"
38#include "Tpetra_Util.hpp"
39#include <algorithm>
40#include <limits>
41#include <map>
42#include <sstream>
43#include <string>
44#include <type_traits>
45#include <utility>
46#include <vector>
47
48namespace Tpetra {
49namespace Details {
50namespace Impl {
51
52template <class MapIter>
53void verbosePrintMap(std::ostream& out,
54 MapIter beg,
55 MapIter end,
56 const size_t numEnt,
57 const char mapName[]) {
58 using ::Tpetra::Details::Behavior;
59 using ::Tpetra::Details::verbosePrintArray;
60
61 out << mapName << ": {";
62 const size_t maxNumToPrint =
63 Behavior::verbosePrintCountThreshold();
64 if (maxNumToPrint == 0) {
65 if (numEnt != 0) {
66 out << "...";
67 }
68 } else {
69 const size_t numToPrint = numEnt > maxNumToPrint ? maxNumToPrint : numEnt;
70 size_t count = 0;
71 for (MapIter it = beg; it != end; ++it) {
72 out << "(" << (*it).first << ", ";
73 verbosePrintArray(out, (*it).second, "gblColInds",
74 maxNumToPrint);
75 out << ")";
76 if (count + size_t(1) < numToPrint) {
77 out << ", ";
78 }
79 ++count;
80 }
81 if (count < numEnt) {
82 out << ", ...";
83 }
84 }
85 out << "}";
86}
87
88template <class LO, class GO, class Node>
89Teuchos::ArrayView<GO>
90getRowGraphGlobalRow(
91 std::vector<GO>& gblColIndsStorage,
92 const RowGraph<LO, GO, Node>& graph,
93 const GO gblRowInd) {
94 size_t origNumEnt = graph.getNumEntriesInGlobalRow(gblRowInd);
95 if (gblColIndsStorage.size() < origNumEnt) {
96 gblColIndsStorage.resize(origNumEnt);
97 }
98 typename CrsGraph<LO, GO, Node>::nonconst_global_inds_host_view_type gblColInds(gblColIndsStorage.data(),
99 origNumEnt);
100 graph.getGlobalRowCopy(gblRowInd, gblColInds, origNumEnt);
101 Teuchos::ArrayView<GO> retval(gblColIndsStorage.data(), origNumEnt);
102 return retval;
103}
104
105template <class LO, class GO, class DT, class OffsetType, class NumEntType>
106class ConvertColumnIndicesFromGlobalToLocal {
107 public:
108 ConvertColumnIndicesFromGlobalToLocal(const ::Kokkos::View<LO*, DT>& lclColInds,
109 const ::Kokkos::View<const GO*, DT>& gblColInds,
110 const ::Kokkos::View<const OffsetType*, DT>& ptr,
111 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
112 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt)
113 : lclColInds_(lclColInds)
114 , gblColInds_(gblColInds)
115 , ptr_(ptr)
116 , lclColMap_(lclColMap)
117 , numRowEnt_(numRowEnt) {}
118
119 KOKKOS_FUNCTION void
120 operator()(const LO& lclRow, OffsetType& curNumBad) const {
121 const OffsetType offset = ptr_(lclRow);
122 // NOTE (mfh 26 Jun 2016) It's always legal to cast the number
123 // of entries in a row to LO, as long as the row doesn't have
124 // too many duplicate entries.
125 const LO numEnt = static_cast<LO>(numRowEnt_(lclRow));
126 for (LO j = 0; j < numEnt; ++j) {
127 const GO gid = gblColInds_(offset + j);
128 const LO lid = lclColMap_.getLocalElement(gid);
129 lclColInds_(offset + j) = lid;
130 if (lid == ::Tpetra::Details::OrdinalTraits<LO>::invalid()) {
131 ++curNumBad;
132 }
133 }
134 }
135
136 static OffsetType
137 run(const ::Kokkos::View<LO*, DT>& lclColInds,
138 const ::Kokkos::View<const GO*, DT>& gblColInds,
139 const ::Kokkos::View<const OffsetType*, DT>& ptr,
140 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
141 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt) {
142 typedef ::Kokkos::RangePolicy<typename DT::execution_space, LO> range_type;
143 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> functor_type;
144
145 const LO lclNumRows = ptr.extent(0) == 0 ? static_cast<LO>(0) : static_cast<LO>(ptr.extent(0) - 1);
146 OffsetType numBad = 0;
147 // Count of "bad" column indices is a reduction over rows.
148 ::Kokkos::parallel_reduce(range_type(0, lclNumRows),
149 functor_type(lclColInds, gblColInds, ptr,
150 lclColMap, numRowEnt),
151 numBad);
152 return numBad;
153 }
154
155 private:
156 ::Kokkos::View<LO*, DT> lclColInds_;
157 ::Kokkos::View<const GO*, DT> gblColInds_;
158 ::Kokkos::View<const OffsetType*, DT> ptr_;
159 ::Tpetra::Details::LocalMap<LO, GO, DT> lclColMap_;
160 ::Kokkos::View<const NumEntType*, DT> numRowEnt_;
161};
162
163} // namespace Impl
164
179template <class LO, class GO, class DT, class OffsetType, class NumEntType>
180OffsetType
181convertColumnIndicesFromGlobalToLocal(const Kokkos::View<LO*, DT>& lclColInds,
182 const Kokkos::View<const GO*, DT>& gblColInds,
183 const Kokkos::View<const OffsetType*, DT>& ptr,
184 const LocalMap<LO, GO, DT>& lclColMap,
185 const Kokkos::View<const NumEntType*, DT>& numRowEnt) {
186 using Impl::ConvertColumnIndicesFromGlobalToLocal;
187 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> impl_type;
188 return impl_type::run(lclColInds, gblColInds, ptr, lclColMap, numRowEnt);
189}
190
191template <class ViewType, class LO>
192class MaxDifference {
193 public:
194 MaxDifference(const ViewType& ptr)
195 : ptr_(ptr) {}
196
197 KOKKOS_INLINE_FUNCTION void init(LO& dst) const {
198 dst = 0;
199 }
200
201 KOKKOS_INLINE_FUNCTION void
202 join(LO& dst, const LO& src) const {
203 dst = (src > dst) ? src : dst;
204 }
205
206 KOKKOS_INLINE_FUNCTION void
207 operator()(const LO lclRow, LO& maxNumEnt) const {
208 const LO numEnt = static_cast<LO>(ptr_(lclRow + 1) - ptr_(lclRow));
209 maxNumEnt = (numEnt > maxNumEnt) ? numEnt : maxNumEnt;
210 }
211
212 private:
213 typename ViewType::const_type ptr_;
214};
215
216template <class ViewType, class LO>
217typename ViewType::non_const_value_type
218maxDifference(const char kernelLabel[],
219 const ViewType& ptr,
220 const LO lclNumRows) {
221 if (lclNumRows == 0) {
222 // mfh 07 May 2018: Weirdly, I need this special case,
223 // otherwise I get the wrong answer.
224 return static_cast<LO>(0);
225 } else {
226 using execution_space = typename ViewType::execution_space;
227 using range_type = Kokkos::RangePolicy<execution_space, LO>;
228 LO theMaxNumEnt{0};
229 Kokkos::parallel_reduce(kernelLabel,
230 range_type(0, lclNumRows),
231 MaxDifference<ViewType, LO>(ptr),
232 theMaxNumEnt);
233 return theMaxNumEnt;
234 }
235}
236
237} // namespace Details
238
239template <class LocalOrdinal, class GlobalOrdinal, class Node>
240bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
241 getDebug() {
242 return Details::Behavior::debug("CrsGraph");
243}
244
245template <class LocalOrdinal, class GlobalOrdinal, class Node>
246bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
247 getVerbose() {
248 return Details::Behavior::verbose("CrsGraph");
249}
250
251template <class LocalOrdinal, class GlobalOrdinal, class Node>
252CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
253 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
254 const size_t maxNumEntriesPerRow,
255 const Teuchos::RCP<Teuchos::ParameterList>& params)
256 : dist_object_type(rowMap)
257 , rowMap_(rowMap)
258 , numAllocForAllRows_(maxNumEntriesPerRow) {
259 const char tfecfFuncName[] =
260 "CrsGraph(rowMap,maxNumEntriesPerRow,params): ";
261 staticAssertions();
262 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid(),
263 std::invalid_argument,
264 "The allocation hint maxNumEntriesPerRow must be "
265 "a valid size_t value, which in this case means it must not be "
266 "Teuchos::OrdinalTraits<size_t>::invalid().");
267 resumeFill(params);
268 checkInternalState();
269}
270
271template <class LocalOrdinal, class GlobalOrdinal, class Node>
272CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
273 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
274 const Teuchos::RCP<const map_type>& colMap,
275 const size_t maxNumEntriesPerRow,
276 const Teuchos::RCP<Teuchos::ParameterList>& params)
277 : dist_object_type(rowMap)
278 , rowMap_(rowMap)
279 , colMap_(colMap)
280 , numAllocForAllRows_(maxNumEntriesPerRow) {
281 const char tfecfFuncName[] =
282 "CrsGraph(rowMap,colMap,maxNumEntriesPerRow,params): ";
283 staticAssertions();
284 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
285 maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid(),
286 std::invalid_argument,
287 "The allocation hint maxNumEntriesPerRow must be "
288 "a valid size_t value, which in this case means it must not be "
289 "Teuchos::OrdinalTraits<size_t>::invalid().");
290 resumeFill(params);
291 checkInternalState();
292}
293
294template <class LocalOrdinal, class GlobalOrdinal, class Node>
295CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
296 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
297 const Teuchos::ArrayView<const size_t>& numEntPerRow,
298 const Teuchos::RCP<Teuchos::ParameterList>& params)
299 : dist_object_type(rowMap)
300 , rowMap_(rowMap)
301 , numAllocForAllRows_(0) {
302 const char tfecfFuncName[] =
303 "CrsGraph(rowMap,numEntPerRow,params): ";
304 staticAssertions();
305
306 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
307 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
308 static_cast<size_t>(numEntPerRow.size()) != lclNumRows,
309 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.size() << " != the local number of rows " << lclNumRows << " as specified by "
310 "the input row Map.");
311
312 if (debug_) {
313 for (size_t r = 0; r < lclNumRows; ++r) {
314 const size_t curRowCount = numEntPerRow[r];
315 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
316 std::invalid_argument, "numEntPerRow(" << r << ") "
317 "specifies an invalid number of entries "
318 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
319 }
320 }
321
322 // Deep-copy the (host-accessible) input into k_numAllocPerRow_.
323 // The latter is a const View, so we have to copy into a nonconst
324 // View first, then assign.
325 typedef decltype(k_numAllocPerRow_) out_view_type;
326 typedef typename out_view_type::non_const_type nc_view_type;
327 typedef Kokkos::View<const size_t*,
328 typename nc_view_type::array_layout,
329 Kokkos::HostSpace,
330 Kokkos::MemoryUnmanaged>
331 in_view_type;
332 in_view_type numAllocPerRowIn(numEntPerRow.getRawPtr(), lclNumRows);
333 nc_view_type numAllocPerRowOut("Tpetra::CrsGraph::numAllocPerRow",
334 lclNumRows);
335 // DEEP_COPY REVIEW - HOST-TO-HOSTMIRROR
336 using exec_space = typename nc_view_type::execution_space;
337 Kokkos::deep_copy(exec_space(), numAllocPerRowOut, numAllocPerRowIn);
338 k_numAllocPerRow_ = numAllocPerRowOut;
339
340 resumeFill(params);
341 checkInternalState();
342}
343
344template <class LocalOrdinal, class GlobalOrdinal, class Node>
345CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
346 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
347 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
348 const Teuchos::RCP<Teuchos::ParameterList>& params)
349 : dist_object_type(rowMap)
350 , rowMap_(rowMap)
351 , k_numAllocPerRow_(numEntPerRow.view_host())
352 , numAllocForAllRows_(0) {
353 const char tfecfFuncName[] =
354 "CrsGraph(rowMap,numEntPerRow,params): ";
355 staticAssertions();
356
357 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
358 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
359 static_cast<size_t>(numEntPerRow.extent(0)) != lclNumRows,
360 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.extent(0) << " != the local number of rows " << lclNumRows << " as specified by "
361 "the input row Map.");
362
363 if (debug_) {
364 for (size_t r = 0; r < lclNumRows; ++r) {
365 const size_t curRowCount = numEntPerRow.view_host()(r);
366 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
367 std::invalid_argument, "numEntPerRow(" << r << ") "
368 "specifies an invalid number of entries "
369 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
370 }
371 }
372
373 resumeFill(params);
374 checkInternalState();
375}
376
377template <class LocalOrdinal, class GlobalOrdinal, class Node>
378CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
379 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
380 const Teuchos::RCP<const map_type>& colMap,
381 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
382 const Teuchos::RCP<Teuchos::ParameterList>& params)
383 : dist_object_type(rowMap)
384 , rowMap_(rowMap)
385 , colMap_(colMap)
386 , k_numAllocPerRow_(numEntPerRow.view_host())
387 , numAllocForAllRows_(0) {
388 const char tfecfFuncName[] =
389 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
390 staticAssertions();
391
392 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
393 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
394 static_cast<size_t>(numEntPerRow.extent(0)) != lclNumRows,
395 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.extent(0) << " != the local number of rows " << lclNumRows << " as specified by "
396 "the input row Map.");
397
398 if (debug_) {
399 for (size_t r = 0; r < lclNumRows; ++r) {
400 const size_t curRowCount = numEntPerRow.view_host()(r);
401 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
402 std::invalid_argument, "numEntPerRow(" << r << ") "
403 "specifies an invalid number of entries "
404 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
405 }
406 }
407
408 resumeFill(params);
409 checkInternalState();
410}
411
412template <class LocalOrdinal, class GlobalOrdinal, class Node>
413CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
414 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
415 const Teuchos::RCP<const map_type>& colMap,
416 const Teuchos::ArrayView<const size_t>& numEntPerRow,
417 const Teuchos::RCP<Teuchos::ParameterList>& params)
418 : dist_object_type(rowMap)
419 , rowMap_(rowMap)
420 , colMap_(colMap)
421 , numAllocForAllRows_(0) {
422 const char tfecfFuncName[] =
423 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
424 staticAssertions();
425
426 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
427 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
428 static_cast<size_t>(numEntPerRow.size()) != lclNumRows,
429 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.size() << " != the local number of rows " << lclNumRows << " as specified by "
430 "the input row Map.");
431
432 if (debug_) {
433 for (size_t r = 0; r < lclNumRows; ++r) {
434 const size_t curRowCount = numEntPerRow[r];
435 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
436 std::invalid_argument, "numEntPerRow(" << r << ") "
437 "specifies an invalid number of entries "
438 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
439 }
440 }
441
442 // Deep-copy the (host-accessible) input into k_numAllocPerRow_.
443 // The latter is a const View, so we have to copy into a nonconst
444 // View first, then assign.
445 typedef decltype(k_numAllocPerRow_) out_view_type;
446 typedef typename out_view_type::non_const_type nc_view_type;
447 typedef Kokkos::View<const size_t*,
448 typename nc_view_type::array_layout,
449 Kokkos::HostSpace,
450 Kokkos::MemoryUnmanaged>
451 in_view_type;
452 in_view_type numAllocPerRowIn(numEntPerRow.getRawPtr(), lclNumRows);
453 nc_view_type numAllocPerRowOut("Tpetra::CrsGraph::numAllocPerRow",
454 lclNumRows);
455 // DEEP_COPY REVIEW - HOST-TO-HOSTMIRROR
456 using exec_space = typename nc_view_type::execution_space;
457 Kokkos::deep_copy(exec_space(), numAllocPerRowOut, numAllocPerRowIn);
458 k_numAllocPerRow_ = numAllocPerRowOut;
459
460 resumeFill(params);
461 checkInternalState();
462}
463
464template <class LocalOrdinal, class GlobalOrdinal, class Node>
465CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
466 CrsGraph(CrsGraph<local_ordinal_type, global_ordinal_type, node_type>& originalGraph,
467 const Teuchos::RCP<const map_type>& rowMap,
468 const Teuchos::RCP<Teuchos::ParameterList>& params)
469 : dist_object_type(rowMap)
470 , rowMap_(rowMap)
471 , colMap_(originalGraph.colMap_)
472 , numAllocForAllRows_(originalGraph.numAllocForAllRows_)
473 , storageStatus_(originalGraph.storageStatus_)
474 , indicesAreAllocated_(originalGraph.indicesAreAllocated_)
475 , indicesAreLocal_(originalGraph.indicesAreLocal_)
476 , indicesAreSorted_(originalGraph.indicesAreSorted_) {
477 staticAssertions();
478
479 int numRows = rowMap->getLocalNumElements();
480 size_t numNonZeros = originalGraph.getRowPtrsPackedHost()(numRows);
481 auto rowsToUse = Kokkos::pair<size_t, size_t>(0, numRows + 1);
482
483 this->setRowPtrsUnpacked(Kokkos::subview(originalGraph.getRowPtrsUnpackedDevice(), rowsToUse));
484 this->setRowPtrsPacked(Kokkos::subview(originalGraph.getRowPtrsPackedDevice(), rowsToUse));
485
486 if (indicesAreLocal_) {
487 lclIndsUnpacked_wdv = local_inds_wdv_type(originalGraph.lclIndsUnpacked_wdv, 0, numNonZeros);
488 lclIndsPacked_wdv = local_inds_wdv_type(originalGraph.lclIndsPacked_wdv, 0, numNonZeros);
489 } else {
490 gblInds_wdv = global_inds_wdv_type(originalGraph.gblInds_wdv, 0, numNonZeros);
491 }
492
493 checkInternalState();
494}
495
496template <class LocalOrdinal, class GlobalOrdinal, class Node>
497CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
498 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
499 const Teuchos::RCP<const map_type>& colMap,
500 const typename local_graph_device_type::row_map_type& rowPointers,
501 const typename local_graph_device_type::entries_type::non_const_type& columnIndices,
502 const Teuchos::RCP<Teuchos::ParameterList>& params)
503 : dist_object_type(rowMap)
504 , rowMap_(rowMap)
505 , colMap_(colMap)
506 , numAllocForAllRows_(0)
507 , storageStatus_(Details::STORAGE_1D_PACKED)
508 , indicesAreAllocated_(true)
509 , indicesAreLocal_(true) {
510 staticAssertions();
511 if (!params.is_null() && params->isParameter("sorted") &&
512 !params->get<bool>("sorted")) {
513 indicesAreSorted_ = false;
514 } else {
515 indicesAreSorted_ = true;
516 }
517 setAllIndices(rowPointers, columnIndices);
518 checkInternalState();
519}
520
521template <class LocalOrdinal, class GlobalOrdinal, class Node>
522CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
523 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
524 const Teuchos::RCP<const map_type>& colMap,
525 const Teuchos::ArrayRCP<size_t>& rowPointers,
526 const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices,
527 const Teuchos::RCP<Teuchos::ParameterList>& params)
528 : dist_object_type(rowMap)
529 , rowMap_(rowMap)
530 , colMap_(colMap)
531 , numAllocForAllRows_(0)
532 , storageStatus_(Details::STORAGE_1D_PACKED)
533 , indicesAreAllocated_(true)
534 , indicesAreLocal_(true) {
535 staticAssertions();
536 if (!params.is_null() && params->isParameter("sorted") &&
537 !params->get<bool>("sorted")) {
538 indicesAreSorted_ = false;
539 } else {
540 indicesAreSorted_ = true;
541 }
542 setAllIndices(rowPointers, columnIndices);
543 checkInternalState();
544}
545
546template <class LocalOrdinal, class GlobalOrdinal, class Node>
547CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
548 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
549 const Teuchos::RCP<const map_type>& colMap,
550 const local_graph_device_type& k_local_graph_,
551 const Teuchos::RCP<Teuchos::ParameterList>& params)
552 : CrsGraph(k_local_graph_,
553 rowMap,
554 colMap,
555 Teuchos::null,
556 Teuchos::null,
557 params) {}
558
559template <class LocalOrdinal, class GlobalOrdinal, class Node>
560CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
561 CrsGraph(const local_graph_device_type& k_local_graph_,
562 const Teuchos::RCP<const map_type>& rowMap,
563 const Teuchos::RCP<const map_type>& colMap,
564 const Teuchos::RCP<const map_type>& domainMap,
565 const Teuchos::RCP<const map_type>& rangeMap,
566 const Teuchos::RCP<Teuchos::ParameterList>& params)
567 : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
568 , rowMap_(rowMap)
569 , colMap_(colMap)
570 , numAllocForAllRows_(0)
571 , storageStatus_(Details::STORAGE_1D_PACKED)
572 , indicesAreAllocated_(true)
573 , indicesAreLocal_(true) {
574 staticAssertions();
575 const char tfecfFuncName[] = "CrsGraph(Kokkos::LocalStaticCrsGraph,Map,Map,Map,Map)";
576
577 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
578 colMap.is_null(), std::runtime_error,
579 ": The input column Map must be nonnull.");
580 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
581 k_local_graph_.numRows() != rowMap->getLocalNumElements(),
582 std::runtime_error,
583 ": The input row Map and the input local graph need to have the same "
584 "number of rows. The row Map claims "
585 << rowMap->getLocalNumElements()
586 << " row(s), but the local graph claims " << k_local_graph_.numRows()
587 << " row(s).");
588
589 // NOTE (mfh 17 Mar 2014) getLocalNumRows() returns
590 // rowMap_->getLocalNumElements(), but it doesn't have to.
591 // TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
592 // k_local_graph_.numRows () != getLocalNumRows (), std::runtime_error,
593 // ": The input row Map and the input local graph need to have the same "
594 // "number of rows. The row Map claims " << getLocalNumRows () << " row(s), "
595 // "but the local graph claims " << k_local_graph_.numRows () << " row(s).");
596 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
597 lclIndsUnpacked_wdv.extent(0) != 0 || gblInds_wdv.extent(0) != 0, std::logic_error,
598 ": cannot have 1D data structures allocated.");
599
600 if (!params.is_null() && params->isParameter("sorted") &&
601 !params->get<bool>("sorted")) {
602 indicesAreSorted_ = false;
603 } else {
604 indicesAreSorted_ = true;
605 }
606
607 setDomainRangeMaps(domainMap.is_null() ? rowMap_ : domainMap,
608 rangeMap.is_null() ? rowMap_ : rangeMap);
609 Teuchos::Array<int> remotePIDs(0); // unused output argument
610 this->makeImportExport(remotePIDs, false);
611
612 lclIndsPacked_wdv = local_inds_wdv_type(k_local_graph_.entries);
613 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
614 this->setRowPtrs(k_local_graph_.row_map);
615
616 set_need_sync_host_uvm_access(); // lclGraph_ potentially still in a kernel
617
618 const bool callComputeGlobalConstants = params.get() == nullptr ||
619 params->get("compute global constants", true);
620
621 if (callComputeGlobalConstants) {
622 this->computeGlobalConstants();
623 }
624 this->fillComplete_ = true;
625 this->checkInternalState();
626}
627
628template <class LocalOrdinal, class GlobalOrdinal, class Node>
629CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
630 CrsGraph(const local_graph_device_type& lclGraph,
631 const Teuchos::RCP<const map_type>& rowMap,
632 const Teuchos::RCP<const map_type>& colMap,
633 const Teuchos::RCP<const map_type>& domainMap,
634 const Teuchos::RCP<const map_type>& rangeMap,
635 const Teuchos::RCP<const import_type>& importer,
636 const Teuchos::RCP<const export_type>& exporter,
637 const Teuchos::RCP<Teuchos::ParameterList>& params)
638 : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
639 , rowMap_(rowMap)
640 , colMap_(colMap)
641 , rangeMap_(rangeMap.is_null() ? rowMap : rangeMap)
642 , domainMap_(domainMap.is_null() ? rowMap : domainMap)
643 , importer_(importer)
644 , exporter_(exporter)
645 , numAllocForAllRows_(0)
646 , storageStatus_(Details::STORAGE_1D_PACKED)
647 , indicesAreAllocated_(true)
648 , indicesAreLocal_(true) {
649 staticAssertions();
650 const char tfecfFuncName[] =
651 "Tpetra::CrsGraph(local_graph_device_type,"
652 "Map,Map,Map,Map,Import,Export,params): ";
653
654 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(colMap.is_null(), std::runtime_error,
655 "The input column Map must be nonnull.");
656
657 lclIndsPacked_wdv = local_inds_wdv_type(lclGraph.entries);
658 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
659 setRowPtrs(lclGraph.row_map);
660
661 set_need_sync_host_uvm_access(); // lclGraph_ potentially still in a kernel
662
663 if (!params.is_null() && params->isParameter("sorted") &&
664 !params->get<bool>("sorted")) {
665 indicesAreSorted_ = false;
666 } else {
667 indicesAreSorted_ = true;
668 }
669
670 const bool callComputeGlobalConstants =
671 params.get() == nullptr ||
672 params->get("compute global constants", true);
673 if (callComputeGlobalConstants) {
674 this->computeGlobalConstants();
675 }
676 fillComplete_ = true;
677 checkInternalState();
678}
679
680template <class LocalOrdinal, class GlobalOrdinal, class Node>
681CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
682 CrsGraph(const row_ptrs_device_view_type& rowPointers,
683 const local_inds_wdv_type& columnIndices,
684 const Teuchos::RCP<const map_type>& rowMap,
685 const Teuchos::RCP<const map_type>& colMap,
686 const Teuchos::RCP<const map_type>& domainMap,
687 const Teuchos::RCP<const map_type>& rangeMap,
688 const Teuchos::RCP<const import_type>& importer,
689 const Teuchos::RCP<const export_type>& exporter,
690 const Teuchos::RCP<Teuchos::ParameterList>& params)
691 : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
692 , rowMap_(rowMap)
693 , colMap_(colMap)
694 , rangeMap_(rangeMap.is_null() ? rowMap : rangeMap)
695 , domainMap_(domainMap.is_null() ? rowMap : domainMap)
696 , importer_(importer)
697 , exporter_(exporter)
698 , numAllocForAllRows_(0)
699 , storageStatus_(Details::STORAGE_1D_PACKED)
700 , indicesAreAllocated_(true)
701 , indicesAreLocal_(true) {
702 staticAssertions();
703 const char tfecfFuncName[] =
704 "Tpetra::CrsGraph(row_ptrs_device_view_type,local_inds_wdv_type"
705 "Map,Map,Map,Map,Import,Export,params): ";
706
707 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(colMap.is_null(), std::runtime_error,
708 "The input column Map must be nonnull.");
709
710 lclIndsPacked_wdv = columnIndices;
711 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
712 setRowPtrs(rowPointers);
713
714 set_need_sync_host_uvm_access(); // lclGraph_ potentially still in a kernel
715
716 if (!params.is_null() && params->isParameter("sorted") &&
717 !params->get<bool>("sorted")) {
718 indicesAreSorted_ = false;
719 } else {
720 indicesAreSorted_ = true;
721 }
722
723 const bool callComputeGlobalConstants =
724 params.get() == nullptr ||
725 params->get("compute global constants", true);
726 if (callComputeGlobalConstants) {
727 this->computeGlobalConstants();
728 }
729 fillComplete_ = true;
730 checkInternalState();
731}
732
733template <class LocalOrdinal, class GlobalOrdinal, class Node>
734Teuchos::RCP<const Teuchos::ParameterList>
735CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
736 getValidParameters() const {
737 using Teuchos::ParameterList;
738 using Teuchos::parameterList;
739 using Teuchos::RCP;
740
741 RCP<ParameterList> params = parameterList("Tpetra::CrsGraph");
742
743 // Make a sublist for the Import.
744 RCP<ParameterList> importSublist = parameterList("Import");
745
746 // FIXME (mfh 02 Apr 2012) We should really have the Import and
747 // Export objects fill in these lists. However, we don't want to
748 // create an Import or Export unless we need them. For now, we
749 // know that the Import and Export just pass the list directly to
750 // their Distributor, so we can create a Distributor here
751 // (Distributor's constructor is a lightweight operation) and have
752 // it fill in the list.
753
754 // Fill in Distributor default parameters by creating a
755 // Distributor and asking it to do the work.
756 Distributor distributor(rowMap_->getComm(), importSublist);
757 params->set("Import", *importSublist, "How the Import performs communication.");
758
759 // Make a sublist for the Export. For now, it's a clone of the
760 // Import sublist. It's not a shallow copy, though, since we
761 // might like the Import to do communication differently than the
762 // Export.
763 params->set("Export", *importSublist, "How the Export performs communication.");
764
765 return params;
766}
767
768template <class LocalOrdinal, class GlobalOrdinal, class Node>
769void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
770 setParameterList(const Teuchos::RCP<Teuchos::ParameterList>& params) {
771 Teuchos::RCP<const Teuchos::ParameterList> validParams =
772 getValidParameters();
773 params->validateParametersAndSetDefaults(*validParams);
774 this->setMyParamList(params);
775}
776
777template <class LocalOrdinal, class GlobalOrdinal, class Node>
778global_size_t
779CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
780 getGlobalNumRows() const {
781 return rowMap_->getGlobalNumElements();
782}
783
784template <class LocalOrdinal, class GlobalOrdinal, class Node>
785global_size_t
786CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
787 getGlobalNumCols() const {
788 const char tfecfFuncName[] = "getGlobalNumCols: ";
789 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
790 !isFillComplete() || getDomainMap().is_null(), std::runtime_error,
791 "The graph does not have a domain Map. You may not call this method in "
792 "that case.");
793 return getDomainMap()->getGlobalNumElements();
794}
795
796template <class LocalOrdinal, class GlobalOrdinal, class Node>
797size_t
798CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
799 getLocalNumRows() const {
800 return this->rowMap_.is_null() ? static_cast<size_t>(0) : this->rowMap_->getLocalNumElements();
801}
802
803template <class LocalOrdinal, class GlobalOrdinal, class Node>
804size_t
805CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
806 getLocalNumCols() const {
807 const char tfecfFuncName[] = "getLocalNumCols: ";
808 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
809 !hasColMap(), std::runtime_error,
810 "The graph does not have a column Map. You may not call this method "
811 "unless the graph has a column Map. This requires either that a custom "
812 "column Map was given to the constructor, or that fillComplete() has "
813 "been called.");
814 return colMap_.is_null() ? static_cast<size_t>(0) : colMap_->getLocalNumElements();
815}
816
817template <class LocalOrdinal, class GlobalOrdinal, class Node>
818Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
823
824template <class LocalOrdinal, class GlobalOrdinal, class Node>
825Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
830
831template <class LocalOrdinal, class GlobalOrdinal, class Node>
832Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
837
838template <class LocalOrdinal, class GlobalOrdinal, class Node>
839Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
844
845template <class LocalOrdinal, class GlobalOrdinal, class Node>
846Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::import_type>
851
852template <class LocalOrdinal, class GlobalOrdinal, class Node>
853Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::export_type>
858
859template <class LocalOrdinal, class GlobalOrdinal, class Node>
860bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
861 hasColMap() const {
862 return !colMap_.is_null();
863}
864
865template <class LocalOrdinal, class GlobalOrdinal, class Node>
866bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
867 isStorageOptimized() const {
868 // FIXME (mfh 07 Aug 2014) Why wouldn't storage be optimized if
869 // getLocalNumRows() is zero?
870
871 const bool isOpt = indicesAreAllocated_ &&
872 k_numRowEntries_.extent(0) == 0 &&
873 getLocalNumRows() > 0;
874
875 return isOpt;
876}
877
878template <class LocalOrdinal, class GlobalOrdinal, class Node>
879global_size_t
880CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
881 getGlobalNumEntries() const {
882 const char tfecfFuncName[] = "getGlobalNumEntries: ";
883 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_, std::logic_error,
884 "The graph does not have global constants computed, "
885 "but the user has requested them.");
886
887 return globalNumEntries_;
888}
889
890template <class LocalOrdinal, class GlobalOrdinal, class Node>
891size_t
892CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
893 getLocalNumEntries() const {
894 const char tfecfFuncName[] = "getLocalNumEntries: ";
895 typedef LocalOrdinal LO;
896
897 Details::ProfilingRegion regionGLNE("Tpetra::CrsGraph::getLocalNumEntries");
898
899 if (this->indicesAreAllocated_) {
900 const LO lclNumRows = this->getLocalNumRows();
901 if (lclNumRows == 0) {
902 return static_cast<size_t>(0);
903 } else {
904 // Avoid the "*this capture" issue by creating a local Kokkos::View.
905 auto numEntPerRow = this->k_numRowEntries_;
906 const LO numNumEntPerRow = numEntPerRow.extent(0);
907 if (numNumEntPerRow == 0) {
908 if (static_cast<LO>(this->getRowPtrsPackedDevice().extent(0)) <
909 static_cast<LO>(lclNumRows + 1)) {
910 return static_cast<size_t>(0);
911 } else {
912 // indices are allocated and k_numRowEntries_ is not allocated,
913 // so we have packed storage and the length of lclIndsPacked_wdv
914 // must be the number of local entries.
915 if (debug_) {
916 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->getRowPtrsPackedHost()(lclNumRows) != lclIndsPacked_wdv.extent(0), std::logic_error,
917 "Final entry of packed host rowptrs doesn't match the length of lclIndsPacked");
918 }
919 return lclIndsPacked_wdv.extent(0);
920 }
921 } else { // k_numRowEntries_ is populated
922 // k_numRowEntries_ is actually be a host View, so we run
923 // the sum in its native execution space. This also means
924 // that we can use explicit capture (which could perhaps
925 // improve build time) instead of KOKKOS_LAMBDA, and avoid
926 // any CUDA build issues with trying to run a __device__ -
927 // only function on host.
928 typedef typename num_row_entries_type::execution_space
929 host_exec_space;
930 typedef Kokkos::RangePolicy<host_exec_space, LO> range_type;
931
932 const LO upperLoopBound = lclNumRows < numNumEntPerRow ? lclNumRows : numNumEntPerRow;
933 size_t nodeNumEnt = 0;
934 Kokkos::parallel_reduce(
935 "Tpetra::CrsGraph::getNumNodeEntries",
936 range_type(0, upperLoopBound),
937 [=](const LO& k, size_t& lclSum) {
938 lclSum += numEntPerRow(k);
939 },
940 nodeNumEnt);
941 return nodeNumEnt;
942 }
943 }
944 } else { // nothing allocated on this process, so no entries
945 return static_cast<size_t>(0);
946 }
947}
948
949template <class LocalOrdinal, class GlobalOrdinal, class Node>
950global_size_t
951CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
952 getGlobalMaxNumRowEntries() const {
953 const char tfecfFuncName[] = "getGlobalMaxNumRowEntries: ";
954 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_, std::logic_error,
955 "The graph does not have global constants computed, "
956 "but the user has requested them.");
957
958 return globalMaxNumRowEntries_;
959}
960
961template <class LocalOrdinal, class GlobalOrdinal, class Node>
962size_t
967
968template <class LocalOrdinal, class GlobalOrdinal, class Node>
969bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
970 isFillComplete() const {
971 return fillComplete_;
972}
973
974template <class LocalOrdinal, class GlobalOrdinal, class Node>
975bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
976 isFillActive() const {
977 return !fillComplete_;
978}
979
980template <class LocalOrdinal, class GlobalOrdinal, class Node>
981bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
982 isLocallyIndexed() const {
983 return indicesAreLocal_;
984}
985
986template <class LocalOrdinal, class GlobalOrdinal, class Node>
987bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
988 isGloballyIndexed() const {
989 return indicesAreGlobal_;
990}
991
992template <class LocalOrdinal, class GlobalOrdinal, class Node>
993size_t
994CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
995 getLocalAllocationSize() const {
996 typedef LocalOrdinal LO;
997
998 if (this->indicesAreAllocated_) {
999 const LO lclNumRows = this->getLocalNumRows();
1000 if (lclNumRows == 0) {
1001 return static_cast<size_t>(0);
1002 } else if (storageStatus_ == Details::STORAGE_1D_PACKED) {
1003 if (static_cast<LO>(this->getRowPtrsPackedDevice().extent(0)) <
1004 static_cast<LO>(lclNumRows + 1)) {
1005 return static_cast<size_t>(0);
1006 } else {
1007 if (this->isLocallyIndexed())
1008 return lclIndsPacked_wdv.extent(0);
1009 else
1010 return gblInds_wdv.extent(0);
1011 }
1012 } else if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
1013 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1014 if (rowPtrsUnpacked_host.extent(0) == 0) {
1015 return static_cast<size_t>(0);
1016 } else {
1017 if (this->isLocallyIndexed())
1018 return lclIndsUnpacked_wdv.extent(0);
1019 else
1020 return gblInds_wdv.extent(0);
1021 }
1022 } else {
1023 return static_cast<size_t>(0);
1024 }
1025 } else {
1026 return Tpetra::Details::OrdinalTraits<size_t>::invalid();
1027 }
1028}
1029
1030template <class LocalOrdinal, class GlobalOrdinal, class Node>
1031Teuchos::RCP<const Teuchos::Comm<int>>
1032CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1033 getComm() const {
1034 return this->rowMap_.is_null() ? Teuchos::null : this->rowMap_->getComm();
1035}
1036
1037template <class LocalOrdinal, class GlobalOrdinal, class Node>
1038GlobalOrdinal
1039CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1040 getIndexBase() const {
1041 return rowMap_->getIndexBase();
1042}
1043
1044template <class LocalOrdinal, class GlobalOrdinal, class Node>
1045bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1046 indicesAreAllocated() const {
1047 return indicesAreAllocated_;
1048}
1049
1050template <class LocalOrdinal, class GlobalOrdinal, class Node>
1055
1056template <class LocalOrdinal, class GlobalOrdinal, class Node>
1061
1062template <class LocalOrdinal, class GlobalOrdinal, class Node>
1063void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1064 setLocallyModified() {
1065 // FIXME (mfh 07 May 2013) How do we know that the change
1066 // introduced a redundancy, or even that it invalidated the sorted
1067 // order of indices? CrsGraph has always made this conservative
1068 // guess. It could be a bit costly to check at insertion time,
1069 // though.
1070 indicesAreSorted_ = false;
1071 noRedundancies_ = false;
1072
1073 // We've modified the graph, so we'll have to recompute local
1074 // constants like the number of diagonal entries on this process.
1075 haveLocalConstants_ = false;
1076}
1077
1078template <class LocalOrdinal, class GlobalOrdinal, class Node>
1079void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1080 allocateIndices(const ELocalGlobal lg, const bool verbose) {
1081 using std::endl;
1082 using Teuchos::arcp;
1083 using Teuchos::Array;
1084 using Teuchos::ArrayRCP;
1085 typedef Teuchos::ArrayRCP<size_t>::size_type size_type;
1086 typedef typename local_graph_device_type::row_map_type::non_const_type
1087 non_const_row_map_type;
1088 const char tfecfFuncName[] = "allocateIndices: ";
1089 const char suffix[] =
1090 " Please report this bug to the Tpetra developers.";
1091
1092 Details::ProfilingRegion profRegion("Tpetra::CrsGraph::allocateIndices");
1093
1094 std::unique_ptr<std::string> prefix;
1095 if (verbose) {
1096 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
1097 std::ostringstream os;
1098 os << *prefix << "Start: lg="
1099 << (lg == GlobalIndices ? "GlobalIndices" : "LocalIndices")
1100 << ", numRows: " << this->getLocalNumRows() << endl;
1101 std::cerr << os.str();
1102 }
1103
1104 // This is a protected function, only callable by us. If it was
1105 // called incorrectly, it is our fault. That's why the tests
1106 // below throw std::logic_error instead of std::invalid_argument.
1107 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isLocallyIndexed() && lg == GlobalIndices, std::logic_error,
1108 ": The graph is locally indexed, but Tpetra code is calling "
1109 "this method with lg=GlobalIndices."
1110 << suffix);
1111 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed() && lg == LocalIndices, std::logic_error,
1112 ": The graph is globally indexed, but Tpetra code is calling "
1113 "this method with lg=LocalIndices."
1114 << suffix);
1115 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreAllocated(), std::logic_error,
1116 ": The graph's "
1117 "indices are already allocated, but Tpetra is calling "
1118 "allocateIndices again."
1119 << suffix);
1120 const size_t numRows = this->getLocalNumRows();
1121
1122 //
1123 // STATIC ALLOCATION PROFILE
1124 //
1125 size_type numInds = 0;
1126 {
1127 if (verbose) {
1128 std::ostringstream os;
1129 os << *prefix << "Allocate k_rowPtrs: " << (numRows + 1) << endl;
1130 std::cerr << os.str();
1131 }
1132 non_const_row_map_type k_rowPtrs("Tpetra::CrsGraph::ptr", numRows + 1);
1133
1134 if (this->k_numAllocPerRow_.extent(0) != 0) {
1135 // It's OK to throw std::invalid_argument here, because we
1136 // haven't incurred any side effects yet. Throwing that
1137 // exception (and not, say, std::logic_error) implies that the
1138 // instance can recover.
1139 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->k_numAllocPerRow_.extent(0) != numRows,
1140 std::invalid_argument,
1141 "k_numAllocPerRow_ is allocated, that is, "
1142 "has nonzero length "
1143 << this->k_numAllocPerRow_.extent(0)
1144 << ", but its length != numRows = " << numRows << ".");
1145
1146 // k_numAllocPerRow_ is a host View, but k_rowPtrs (the thing
1147 // we want to compute here) lives on device. That's OK;
1148 // computeOffsetsFromCounts can handle this case.
1149 using Details::computeOffsetsFromCounts;
1150
1151 // FIXME (mfh 27 Jun 2016) Currently, computeOffsetsFromCounts
1152 // doesn't attempt to check its input for "invalid" flag
1153 // values. For now, we omit that feature of the sequential
1154 // code disabled below.
1155 numInds = computeOffsetsFromCounts(k_rowPtrs, k_numAllocPerRow_);
1156 } else {
1157 // It's OK to throw std::invalid_argument here, because we
1158 // haven't incurred any side effects yet. Throwing that
1159 // exception (and not, say, std::logic_error) implies that the
1160 // instance can recover.
1161 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->numAllocForAllRows_ ==
1162 Tpetra::Details::OrdinalTraits<size_t>::invalid(),
1163 std::invalid_argument,
1164 "numAllocForAllRows_ has an invalid value, "
1165 "namely Tpetra::Details::OrdinalTraits<size_t>::invalid() = "
1166 << Tpetra::Details::OrdinalTraits<size_t>::invalid() << ".");
1167
1168 using Details::computeOffsetsFromConstantCount;
1169 numInds = computeOffsetsFromConstantCount(k_rowPtrs, this->numAllocForAllRows_);
1170 }
1171 // "Commit" the resulting row offsets.
1172 setRowPtrsUnpacked(k_rowPtrs);
1173 }
1174 if (debug_) {
1175 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numInds != size_type(this->getRowPtrsUnpackedHost()(numRows)), std::logic_error,
1176 ": Number of indices produced by computeOffsetsFrom[Constant]Counts "
1177 "does not match final entry of rowptrs unpacked");
1178 }
1179
1180 if (lg == LocalIndices) {
1181 if (verbose) {
1182 std::ostringstream os;
1183 os << *prefix << "Allocate local column indices "
1184 "lclIndsUnpacked_wdv: "
1185 << numInds << endl;
1186 std::cerr << os.str();
1187 }
1188 lclIndsUnpacked_wdv = local_inds_wdv_type(
1189 local_inds_dualv_type("Tpetra::CrsGraph::lclInd", numInds));
1190 } else {
1191 if (verbose) {
1192 std::ostringstream os;
1193 os << *prefix << "Allocate global column indices "
1194 "gblInds_wdv: "
1195 << numInds << endl;
1196 std::cerr << os.str();
1197 }
1198 gblInds_wdv = global_inds_wdv_type(
1199 global_inds_dualv_type("Tpetra::CrsGraph::gblInd", numInds));
1200 }
1201 storageStatus_ = Details::STORAGE_1D_UNPACKED;
1202
1203 this->indicesAreLocal_ = (lg == LocalIndices);
1204 this->indicesAreGlobal_ = (lg == GlobalIndices);
1205
1206 if (numRows > 0) { // reallocate k_numRowEntries_ & fill w/ 0s
1207 using Kokkos::ViewAllocateWithoutInitializing;
1208 const char label[] = "Tpetra::CrsGraph::numRowEntries";
1209 if (verbose) {
1210 std::ostringstream os;
1211 os << *prefix << "Allocate k_numRowEntries_: " << numRows
1212 << endl;
1213 std::cerr << os.str();
1214 }
1215 num_row_entries_type numRowEnt(ViewAllocateWithoutInitializing(label), numRows);
1216 // DEEP_COPY REVIEW - VALUE-TO-HOSTMIRROR
1217 Kokkos::deep_copy(execution_space(), numRowEnt, static_cast<size_t>(0)); // fill w/ 0s
1218 Kokkos::fence("CrsGraph::allocateIndices"); // TODO: Need to understand downstream failure points and move this fence.
1219 this->k_numRowEntries_ = numRowEnt; // "commit" our allocation
1220 }
1221
1222 // Once indices are allocated, CrsGraph needs to free this information.
1223 this->numAllocForAllRows_ = 0;
1224 this->k_numAllocPerRow_ = decltype(k_numAllocPerRow_)();
1225 this->indicesAreAllocated_ = true;
1226
1227 try {
1228 this->checkInternalState();
1229 } catch (std::logic_error& e) {
1230 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
1231 "At end of allocateIndices, "
1232 "checkInternalState threw std::logic_error: "
1233 << e.what());
1234 } catch (std::exception& e) {
1235 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1236 "At end of allocateIndices, "
1237 "checkInternalState threw std::exception: "
1238 << e.what());
1239 } catch (...) {
1240 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1241 "At end of allocateIndices, "
1242 "checkInternalState threw an exception "
1243 "not a subclass of std::exception.");
1244 }
1245
1246 if (verbose) {
1247 std::ostringstream os;
1248 os << *prefix << "Done" << endl;
1249 std::cerr << os.str();
1250 }
1251}
1252
1253template <class LocalOrdinal, class GlobalOrdinal, class Node>
1254typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1255 local_inds_dualv_type::t_host::const_type
1256 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1257 getLocalIndsViewHost(const RowInfo& rowinfo) const {
1258 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1259 return typename local_inds_dualv_type::t_host::const_type();
1260 else
1261 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1262 rowinfo.allocSize,
1263 Access::ReadOnly);
1264}
1265
1266template <class LocalOrdinal, class GlobalOrdinal, class Node>
1267typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1268 local_inds_dualv_type::t_host
1269 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1270 getLocalIndsViewHostNonConst(const RowInfo& rowinfo) {
1271 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1272 return typename local_inds_dualv_type::t_host();
1273 else
1274 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1275 rowinfo.allocSize,
1276 Access::ReadWrite);
1277}
1278
1279template <class LocalOrdinal, class GlobalOrdinal, class Node>
1280typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1281 global_inds_dualv_type::t_host::const_type
1282 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1283 getGlobalIndsViewHost(const RowInfo& rowinfo) const {
1284 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1285 return typename global_inds_dualv_type::t_host::const_type();
1286 else
1287 return gblInds_wdv.getHostSubview(rowinfo.offset1D,
1288 rowinfo.allocSize,
1289 Access::ReadOnly);
1290}
1291
1292template <class LocalOrdinal, class GlobalOrdinal, class Node>
1293typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1294 local_inds_dualv_type::t_dev::const_type
1295 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1296 getLocalIndsViewDevice(const RowInfo& rowinfo) const {
1297 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1298 return typename local_inds_dualv_type::t_dev::const_type();
1299 else
1300 return lclIndsUnpacked_wdv.getDeviceSubview(rowinfo.offset1D,
1301 rowinfo.allocSize,
1302 Access::ReadOnly);
1303}
1304
1305template <class LocalOrdinal, class GlobalOrdinal, class Node>
1306typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1307 global_inds_dualv_type::t_dev::const_type
1308 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1309 getGlobalIndsViewDevice(const RowInfo& rowinfo) const {
1310 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1311 return typename global_inds_dualv_type::t_dev::const_type();
1312 else
1313 return gblInds_wdv.getDeviceSubview(rowinfo.offset1D,
1314 rowinfo.allocSize,
1315 Access::ReadOnly);
1316}
1317
1318template <class LocalOrdinal, class GlobalOrdinal, class Node>
1319RowInfo
1320CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1321 getRowInfo(const LocalOrdinal myRow) const {
1322 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid();
1323 RowInfo ret;
1324 if (this->rowMap_.is_null() || !this->rowMap_->isNodeLocalElement(myRow)) {
1325 ret.localRow = STINV;
1326 ret.allocSize = 0;
1327 ret.numEntries = 0;
1328 ret.offset1D = STINV;
1329 return ret;
1330 }
1331
1332 ret.localRow = static_cast<size_t>(myRow);
1333 if (this->indicesAreAllocated()) {
1334 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1335 // Offsets tell us the allocation size in this case.
1336 if (rowPtrsUnpacked_host.extent(0) == 0) {
1337 ret.offset1D = 0;
1338 ret.allocSize = 0;
1339 } else {
1340 ret.offset1D = rowPtrsUnpacked_host(myRow);
1341 ret.allocSize = rowPtrsUnpacked_host(myRow + 1) - rowPtrsUnpacked_host(myRow);
1342 }
1343
1344 ret.numEntries = (this->k_numRowEntries_.extent(0) == 0) ? ret.allocSize : this->k_numRowEntries_(myRow);
1345 } else { // haven't performed allocation yet; probably won't hit this code
1346 // FIXME (mfh 07 Aug 2014) We want graph's constructors to
1347 // allocate, rather than doing lazy allocation at first insert.
1348 // This will make k_numAllocPerRow_ obsolete.
1349 ret.allocSize = (this->k_numAllocPerRow_.extent(0) != 0) ? this->k_numAllocPerRow_(myRow) : // this is a host View
1350 this->numAllocForAllRows_;
1351 ret.numEntries = 0;
1352 ret.offset1D = STINV;
1353 }
1354
1355 return ret;
1356}
1357
1358template <class LocalOrdinal, class GlobalOrdinal, class Node>
1359RowInfo
1360CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1361 getRowInfoFromGlobalRowIndex(const GlobalOrdinal gblRow) const {
1362 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid();
1363 RowInfo ret;
1364 if (this->rowMap_.is_null()) {
1365 ret.localRow = STINV;
1366 ret.allocSize = 0;
1367 ret.numEntries = 0;
1368 ret.offset1D = STINV;
1369 return ret;
1370 }
1371 const LocalOrdinal myRow = this->rowMap_->getLocalElement(gblRow);
1372 if (myRow == Teuchos::OrdinalTraits<LocalOrdinal>::invalid()) {
1373 ret.localRow = STINV;
1374 ret.allocSize = 0;
1375 ret.numEntries = 0;
1376 ret.offset1D = STINV;
1377 return ret;
1378 }
1379
1380 ret.localRow = static_cast<size_t>(myRow);
1381 if (this->indicesAreAllocated()) {
1382 // graph data structures have the info that we need
1383 //
1384 // if static graph, offsets tell us the allocation size
1385 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1386 if (rowPtrsUnpacked_host.extent(0) == 0) {
1387 ret.offset1D = 0;
1388 ret.allocSize = 0;
1389 } else {
1390 ret.offset1D = rowPtrsUnpacked_host(myRow);
1391 ret.allocSize = rowPtrsUnpacked_host(myRow + 1) - rowPtrsUnpacked_host(myRow);
1392 }
1393
1394 ret.numEntries = (this->k_numRowEntries_.extent(0) == 0) ? ret.allocSize : this->k_numRowEntries_(myRow);
1395 } else { // haven't performed allocation yet; probably won't hit this code
1396 // FIXME (mfh 07 Aug 2014) We want graph's constructors to
1397 // allocate, rather than doing lazy allocation at first insert.
1398 // This will make k_numAllocPerRow_ obsolete.
1399 ret.allocSize = (this->k_numAllocPerRow_.extent(0) != 0) ? this->k_numAllocPerRow_(myRow) : // this is a host View
1400 this->numAllocForAllRows_;
1401 ret.numEntries = 0;
1402 ret.offset1D = STINV;
1403 }
1404
1405 return ret;
1406}
1407
1408template <class LocalOrdinal, class GlobalOrdinal, class Node>
1409void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1410 staticAssertions() const {
1411 using Teuchos::OrdinalTraits;
1412 typedef LocalOrdinal LO;
1413 typedef GlobalOrdinal GO;
1414 typedef global_size_t GST;
1415
1416 // Assumption: sizeof(GlobalOrdinal) >= sizeof(LocalOrdinal):
1417 // This is so that we can store local indices in the memory
1418 // formerly occupied by global indices.
1419 static_assert(sizeof(GlobalOrdinal) >= sizeof(LocalOrdinal),
1420 "Tpetra::CrsGraph: sizeof(GlobalOrdinal) must be >= sizeof(LocalOrdinal).");
1421 // Assumption: max(size_t) >= max(LocalOrdinal)
1422 // This is so that we can represent any LocalOrdinal as a size_t.
1423 static_assert(sizeof(size_t) >= sizeof(LocalOrdinal),
1424 "Tpetra::CrsGraph: sizeof(size_t) must be >= sizeof(LocalOrdinal).");
1425 static_assert(sizeof(GST) >= sizeof(size_t),
1426 "Tpetra::CrsGraph: sizeof(Tpetra::global_size_t) must be >= sizeof(size_t).");
1427
1428 // FIXME (mfh 30 Sep 2015) We're not using
1429 // Teuchos::CompileTimeAssert any more. Can we do these checks
1430 // with static_assert?
1431
1432 // can't call max() with CompileTimeAssert, because it isn't a
1433 // constant expression; will need to make this a runtime check
1434 const char msg[] =
1435 "Tpetra::CrsGraph: Object cannot be created with the "
1436 "given template arguments: size assumptions are not valid.";
1437 TEUCHOS_TEST_FOR_EXCEPTION(
1438 static_cast<size_t>(Teuchos::OrdinalTraits<LO>::max()) > Teuchos::OrdinalTraits<size_t>::max(),
1439 std::runtime_error, msg);
1440 TEUCHOS_TEST_FOR_EXCEPTION(
1441 static_cast<GST>(Teuchos::OrdinalTraits<LO>::max()) > static_cast<GST>(Teuchos::OrdinalTraits<GO>::max()),
1442 std::runtime_error, msg);
1443 TEUCHOS_TEST_FOR_EXCEPTION(
1444 static_cast<size_t>(Teuchos::OrdinalTraits<GO>::max()) > Teuchos::OrdinalTraits<GST>::max(),
1445 std::runtime_error, msg);
1446 TEUCHOS_TEST_FOR_EXCEPTION(
1447 Teuchos::OrdinalTraits<size_t>::max() > Teuchos::OrdinalTraits<GST>::max(),
1448 std::runtime_error, msg);
1449}
1450
1451template <class LocalOrdinal, class GlobalOrdinal, class Node>
1452size_t
1453CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1454 insertIndices(RowInfo& rowinfo,
1455 const SLocalGlobalViews& newInds,
1456 const ELocalGlobal lg,
1457 const ELocalGlobal I) {
1458 using Teuchos::ArrayView;
1459 typedef LocalOrdinal LO;
1460 typedef GlobalOrdinal GO;
1461 const char tfecfFuncName[] = "insertIndices: ";
1462
1463 size_t oldNumEnt = 0;
1464 if (debug_) {
1465 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(lg != GlobalIndices && lg != LocalIndices, std::invalid_argument,
1466 "lg must be either GlobalIndices or LocalIndices.");
1467 oldNumEnt = this->getNumEntriesInLocalRow(rowinfo.localRow);
1468 }
1469
1470 size_t numNewInds = 0;
1471 if (lg == GlobalIndices) { // input indices are global
1472 ArrayView<const GO> new_ginds = newInds.ginds;
1473 numNewInds = new_ginds.size();
1474 if (I == GlobalIndices) { // store global indices
1475 auto gind_view = gblInds_wdv.getHostView(Access::ReadWrite);
1476 if (debug_) {
1477 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(gind_view.size()) <
1478 rowinfo.numEntries + numNewInds,
1479 std::logic_error,
1480 "gind_view.size() = " << gind_view.size()
1481 << " < rowinfo.numEntries (= " << rowinfo.numEntries
1482 << ") + numNewInds (= " << numNewInds << ").");
1483 }
1484 GO* const gblColInds_out = gind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
1485 for (size_t k = 0; k < numNewInds; ++k) {
1486 gblColInds_out[k] = new_ginds[k];
1487 }
1488 } else if (I == LocalIndices) { // store local indices
1489 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1490 if (debug_) {
1491 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(lind_view.size()) <
1492 rowinfo.numEntries + numNewInds,
1493 std::logic_error,
1494 "lind_view.size() = " << lind_view.size()
1495 << " < rowinfo.numEntries (= " << rowinfo.numEntries
1496 << ") + numNewInds (= " << numNewInds << ").");
1497 }
1498 LO* const lclColInds_out = lind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
1499 for (size_t k = 0; k < numNewInds; ++k) {
1500 lclColInds_out[k] = colMap_->getLocalElement(new_ginds[k]);
1501 }
1502 }
1503 } else if (lg == LocalIndices) { // input indices are local
1504 ArrayView<const LO> new_linds = newInds.linds;
1505 numNewInds = new_linds.size();
1506 if (I == LocalIndices) { // store local indices
1507 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1508 if (debug_) {
1509 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(lind_view.size()) <
1510 rowinfo.numEntries + numNewInds,
1511 std::logic_error,
1512 "lind_view.size() = " << lind_view.size()
1513 << " < rowinfo.numEntries (= " << rowinfo.numEntries
1514 << ") + numNewInds (= " << numNewInds << ").");
1515 }
1516 LO* const lclColInds_out = lind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
1517 for (size_t k = 0; k < numNewInds; ++k) {
1518 lclColInds_out[k] = new_linds[k];
1519 }
1520 } else if (I == GlobalIndices) {
1521 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
1522 "The case where the input indices are local "
1523 "and the indices to write are global (lg=LocalIndices, I="
1524 "GlobalIndices) is not implemented, because it does not make sense."
1525 << std::endl
1526 << "If you have correct local column indices, that "
1527 "means the graph has a column Map. In that case, you should be "
1528 "storing local indices.");
1529 }
1530 }
1531
1532 rowinfo.numEntries += numNewInds;
1533 this->k_numRowEntries_(rowinfo.localRow) += numNewInds;
1534 this->setLocallyModified();
1535
1536 if (debug_) {
1537 const size_t chkNewNumEnt =
1538 this->getNumEntriesInLocalRow(rowinfo.localRow);
1539 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(chkNewNumEnt != oldNumEnt + numNewInds, std::logic_error,
1540 "chkNewNumEnt = " << chkNewNumEnt
1541 << " != oldNumEnt (= " << oldNumEnt
1542 << ") + numNewInds (= " << numNewInds << ").");
1543 }
1544
1545 return numNewInds;
1546}
1547
1548template <class LocalOrdinal, class GlobalOrdinal, class Node>
1549size_t
1550CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1551 insertGlobalIndicesImpl(const LocalOrdinal lclRow,
1552 const GlobalOrdinal inputGblColInds[],
1553 const size_t numInputInds) {
1554 return this->insertGlobalIndicesImpl(this->getRowInfo(lclRow),
1555 inputGblColInds, numInputInds);
1556}
1557
1558template <class LocalOrdinal, class GlobalOrdinal, class Node>
1559size_t
1560CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1561 insertGlobalIndicesImpl(const RowInfo& rowInfo,
1562 const GlobalOrdinal inputGblColInds[],
1563 const size_t numInputInds,
1564 std::function<void(const size_t, const size_t, const size_t)> fun) {
1565 using Details::verbosePrintArray;
1566 using Kokkos::MemoryUnmanaged;
1567 using Kokkos::subview;
1568 using Kokkos::View;
1569 using Teuchos::ArrayView;
1570 using LO = LocalOrdinal;
1571 using GO = GlobalOrdinal;
1572
1573 const char tfecfFuncName[] = "insertGlobalIndicesImpl: ";
1574 const LO lclRow = static_cast<LO>(rowInfo.localRow);
1575
1576 auto numEntries = rowInfo.numEntries;
1577 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1578 inp_view_type inputInds(inputGblColInds, numInputInds);
1579 size_t numInserted;
1580 {
1581 auto gblIndsHostView = this->gblInds_wdv.getHostView(Access::ReadWrite);
1582 numInserted = Details::insertCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
1583 gblIndsHostView,
1584 numEntries, inputInds, fun);
1585 }
1586
1587 const bool insertFailed =
1588 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1589 if (insertFailed) {
1590 constexpr size_t ONE(1);
1591 const int myRank = this->getComm()->getRank();
1592 std::ostringstream os;
1593
1594 os << "Proc " << myRank << ": Not enough capacity to insert "
1595 << numInputInds
1596 << " ind" << (numInputInds != ONE ? "ices" : "ex")
1597 << " into local row " << lclRow << ", which currently has "
1598 << rowInfo.numEntries
1599 << " entr" << (rowInfo.numEntries != ONE ? "ies" : "y")
1600 << " and total allocation size " << rowInfo.allocSize
1601 << ". ";
1602 const size_t maxNumToPrint =
1603 Details::Behavior::verbosePrintCountThreshold();
1604 ArrayView<const GO> inputGblColIndsView(inputGblColInds,
1605 numInputInds);
1606 verbosePrintArray(os, inputGblColIndsView,
1607 "Input global "
1608 "column indices",
1609 maxNumToPrint);
1610 os << ", ";
1611 auto curGblColInds = getGlobalIndsViewHost(rowInfo);
1612 ArrayView<const GO> curGblColIndsView(curGblColInds.data(),
1613 rowInfo.numEntries);
1614 verbosePrintArray(os, curGblColIndsView,
1615 "Current global "
1616 "column indices",
1617 maxNumToPrint);
1618 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
1619 }
1620
1621 this->k_numRowEntries_(lclRow) += numInserted;
1622
1623 this->setLocallyModified();
1624 return numInserted;
1625}
1626
1627template <class LocalOrdinal, class GlobalOrdinal, class Node>
1628void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1629 insertLocalIndicesImpl(const LocalOrdinal myRow,
1630 const Teuchos::ArrayView<const LocalOrdinal>& indices,
1631 std::function<void(const size_t, const size_t, const size_t)> fun) {
1632 using Kokkos::MemoryUnmanaged;
1633 using Kokkos::subview;
1634 using Kokkos::View;
1635 using LO = LocalOrdinal;
1636
1637 const char tfecfFuncName[] = "insertLocallIndicesImpl: ";
1638
1639 const RowInfo rowInfo = this->getRowInfo(myRow);
1640
1641 size_t numNewInds = 0;
1642 size_t newNumEntries = 0;
1643
1644 auto numEntries = rowInfo.numEntries;
1645 // Note: Teuchos::ArrayViews are in HostSpace
1646 using inp_view_type = View<const LO*, Kokkos::HostSpace, MemoryUnmanaged>;
1647 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1648 size_t numInserted = 0;
1649 {
1650 auto lclInds = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1651 numInserted = Details::insertCrsIndices(myRow, this->getRowPtrsUnpackedHost(), lclInds,
1652 numEntries, inputInds, fun);
1653 }
1654
1655 const bool insertFailed =
1656 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1657 if (insertFailed) {
1658 constexpr size_t ONE(1);
1659 const size_t numInputInds(indices.size());
1660 const int myRank = this->getComm()->getRank();
1661 std::ostringstream os;
1662 os << "On MPI Process " << myRank << ": Not enough capacity to "
1663 "insert "
1664 << numInputInds
1665 << " ind" << (numInputInds != ONE ? "ices" : "ex")
1666 << " into local row " << myRow << ", which currently has "
1667 << rowInfo.numEntries
1668 << " entr" << (rowInfo.numEntries != ONE ? "ies" : "y")
1669 << " and total allocation size " << rowInfo.allocSize << ".";
1670 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
1671 }
1672 numNewInds = numInserted;
1673 newNumEntries = rowInfo.numEntries + numNewInds;
1674
1675 this->k_numRowEntries_(myRow) += numNewInds;
1676 this->setLocallyModified();
1677
1678 if (debug_) {
1679 const size_t chkNewNumEntries = this->getNumEntriesInLocalRow(myRow);
1680 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(chkNewNumEntries != newNumEntries, std::logic_error,
1681 "getNumEntriesInLocalRow(" << myRow << ") = " << chkNewNumEntries
1682 << " != newNumEntries = " << newNumEntries
1683 << ". Please report this bug to the Tpetra developers.");
1684 }
1685}
1686
1687template <class LocalOrdinal, class GlobalOrdinal, class Node>
1688size_t
1689CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1690 findGlobalIndices(const RowInfo& rowInfo,
1691 const Teuchos::ArrayView<const GlobalOrdinal>& indices,
1692 std::function<void(const size_t, const size_t, const size_t)> fun) const {
1693 using GO = GlobalOrdinal;
1694 using Kokkos::MemoryUnmanaged;
1695 using Kokkos::View;
1696
1697 auto invalidCount = Teuchos::OrdinalTraits<size_t>::invalid();
1698
1699 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1700 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1701
1702 size_t numFound = 0;
1703 LocalOrdinal lclRow = rowInfo.localRow;
1704 if (this->isLocallyIndexed()) {
1705 if (this->colMap_.is_null())
1706 return invalidCount;
1707 const auto& colMap = *(this->colMap_);
1708 auto map = [&](GO const gblInd) { return colMap.getLocalElement(gblInd); };
1709 if (this->isSorted()) {
1710 numFound = Details::findCrsIndicesSorted(
1711 lclRow,
1712 this->getRowPtrsUnpackedHost(),
1713 rowInfo.numEntries,
1714 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly),
1715 inputInds,
1716 map,
1717 fun);
1718 } else {
1719 numFound = Details::findCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
1720 rowInfo.numEntries,
1721 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly), inputInds, map, fun);
1722 }
1723 } else if (this->isGloballyIndexed()) {
1724 numFound = Details::findCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
1725 rowInfo.numEntries,
1726 gblInds_wdv.getHostView(Access::ReadOnly), inputInds, fun);
1727 }
1728 return numFound;
1729}
1730
1731template <class LocalOrdinal, class GlobalOrdinal, class Node>
1732void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1733 setDomainRangeMaps(const Teuchos::RCP<const map_type>& domainMap,
1734 const Teuchos::RCP<const map_type>& rangeMap) {
1735 // simple pointer comparison for equality
1736 if (domainMap_ != domainMap) {
1737 domainMap_ = domainMap;
1738 importer_ = Teuchos::null;
1739 }
1740 if (rangeMap_ != rangeMap) {
1741 rangeMap_ = rangeMap;
1742 exporter_ = Teuchos::null;
1743 }
1744}
1745
1746template <class LocalOrdinal, class GlobalOrdinal, class Node>
1747void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1748 clearGlobalConstants() {
1749 const auto INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
1750
1751 globalNumEntries_ = INV;
1752 globalMaxNumRowEntries_ = INV;
1753 haveGlobalConstants_ = false;
1754}
1755
1756template <class LocalOrdinal, class GlobalOrdinal, class Node>
1757void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1758 checkInternalState() const {
1759 if (debug_) {
1760 using std::endl;
1761 const char tfecfFuncName[] = "checkInternalState: ";
1762 const char suffix[] = " Please report this bug to the Tpetra developers.";
1763
1764 std::unique_ptr<std::string> prefix;
1765 if (verbose_) {
1766 prefix = this->createPrefix("CrsGraph", "checkInternalState");
1767 std::ostringstream os;
1768 os << *prefix << "Start" << endl;
1769 std::cerr << os.str();
1770 }
1771
1772 const global_size_t GSTI = Teuchos::OrdinalTraits<global_size_t>::invalid();
1773 // const size_t STI = Teuchos::OrdinalTraits<size_t>::invalid (); // unused
1774 // check the internal state of this data structure
1775 // this is called by numerous state-changing methods, in a debug build, to ensure that the object
1776 // always remains in a valid state
1777
1778 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->rowMap_.is_null(), std::logic_error,
1779 "Row Map is null." << suffix);
1780 // This may access the row Map, so we need to check first (above)
1781 // whether the row Map is null.
1782 const LocalOrdinal lclNumRows =
1783 static_cast<LocalOrdinal>(this->getLocalNumRows());
1784
1785 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillActive() == this->isFillComplete(), std::logic_error,
1786 "Graph cannot be both fill active and fill complete." << suffix);
1787 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillComplete() &&
1788 (this->colMap_.is_null() ||
1789 this->rangeMap_.is_null() ||
1790 this->domainMap_.is_null()),
1791 std::logic_error,
1792 "Graph is full complete, but at least one of {column, range, domain} "
1793 "Map is null."
1794 << suffix);
1795 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isStorageOptimized() && !this->indicesAreAllocated(),
1796 std::logic_error,
1797 "Storage is optimized, but indices are not "
1798 "allocated, not even trivially."
1799 << suffix);
1800
1801 size_t nodeAllocSize = 0;
1802 try {
1803 nodeAllocSize = this->getLocalAllocationSize();
1804 } catch (std::logic_error& e) {
1805 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1806 "getLocalAllocationSize threw "
1807 "std::logic_error: "
1808 << e.what());
1809 } catch (std::exception& e) {
1810 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1811 "getLocalAllocationSize threw an "
1812 "std::exception: "
1813 << e.what());
1814 } catch (...) {
1815 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1816 "getLocalAllocationSize threw an exception "
1817 "not a subclass of std::exception.");
1818 }
1819
1820 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isStorageOptimized() &&
1821 nodeAllocSize != this->getLocalNumEntries(),
1822 std::logic_error,
1823 "Storage is optimized, but "
1824 "this->getLocalAllocationSize() = "
1825 << nodeAllocSize
1826 << " != this->getLocalNumEntries() = " << this->getLocalNumEntries()
1827 << "." << suffix);
1828 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_ &&
1829 (this->globalNumEntries_ != GSTI ||
1830 this->globalMaxNumRowEntries_ != GSTI),
1831 std::logic_error,
1832 "Graph claims not to have global constants, but "
1833 "some of the global constants are not marked as invalid."
1834 << suffix);
1835 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->haveGlobalConstants_ &&
1836 (this->globalNumEntries_ == GSTI ||
1837 this->globalMaxNumRowEntries_ == GSTI),
1838 std::logic_error,
1839 "Graph claims to have global constants, but "
1840 "some of them are marked as invalid."
1841 << suffix);
1842 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->haveGlobalConstants_ &&
1843 (this->globalNumEntries_ < this->getLocalNumEntries() ||
1844 this->globalMaxNumRowEntries_ < this->nodeMaxNumRowEntries_),
1845 std::logic_error,
1846 "Graph claims to have global constants, and "
1847 "all of the values of the global constants are valid, but "
1848 "some of the local constants are greater than "
1849 "their corresponding global constants."
1850 << suffix);
1851 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreAllocated() &&
1852 (this->numAllocForAllRows_ != 0 ||
1853 this->k_numAllocPerRow_.extent(0) != 0),
1854 std::logic_error,
1855 "The graph claims that its indices are allocated, but "
1856 "either numAllocForAllRows_ (= "
1857 << this->numAllocForAllRows_ << ") is "
1858 "nonzero, or k_numAllocPerRow_ has nonzero dimension. In other words, "
1859 "the graph is supposed to release its \"allocation specifications\" "
1860 "when it allocates its indices."
1861 << suffix);
1862 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1863 auto rowPtrsUnpacked_dev = this->getRowPtrsUnpackedDevice();
1864 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked_host.extent(0) != rowPtrsUnpacked_dev.extent(0),
1865 std::logic_error,
1866 "The host and device views of k_rowPtrs_ have "
1867 "different sizes; rowPtrsUnpacked_host_ has size "
1868 << rowPtrsUnpacked_host.extent(0)
1869 << ", but rowPtrsUnpacked_dev_ has size "
1870 << rowPtrsUnpacked_dev.extent(0)
1871 << "." << suffix);
1872 if (isGloballyIndexed() && rowPtrsUnpacked_host.extent(0) != 0) {
1873 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(size_t(rowPtrsUnpacked_host.extent(0)) != size_t(lclNumRows + 1),
1874 std::logic_error,
1875 "The graph is globally indexed and "
1876 "k_rowPtrs has nonzero size "
1877 << rowPtrsUnpacked_host.extent(0)
1878 << ", but that size does not equal lclNumRows+1 = "
1879 << (lclNumRows + 1) << "." << suffix);
1880 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked_host(lclNumRows) != size_t(gblInds_wdv.extent(0)),
1881 std::logic_error,
1882 "The graph is globally indexed and "
1883 "k_rowPtrs_ has nonzero size "
1884 << rowPtrsUnpacked_host.extent(0)
1885 << ", but k_rowPtrs_(lclNumRows=" << lclNumRows << ")="
1886 << rowPtrsUnpacked_host(lclNumRows)
1887 << " != gblInds_wdv.extent(0)="
1888 << gblInds_wdv.extent(0) << "." << suffix);
1889 }
1890 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() &&
1891 rowPtrsUnpacked_host.extent(0) != 0 &&
1892 (static_cast<size_t>(rowPtrsUnpacked_host.extent(0)) !=
1893 static_cast<size_t>(lclNumRows + 1) ||
1894 rowPtrsUnpacked_host(lclNumRows) !=
1895 static_cast<size_t>(this->lclIndsUnpacked_wdv.extent(0))),
1896 std::logic_error,
1897 "If k_rowPtrs_ has nonzero size and "
1898 "the graph is locally indexed, then "
1899 "k_rowPtrs_ must have N+1 rows, and "
1900 "k_rowPtrs_(N) must equal lclIndsUnpacked_wdv.extent(0)."
1901 << suffix);
1902
1903 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreAllocated() &&
1904 nodeAllocSize > 0 &&
1905 this->lclIndsUnpacked_wdv.extent(0) == 0 &&
1906 this->gblInds_wdv.extent(0) == 0,
1907 std::logic_error,
1908 "Graph is allocated nontrivially, but "
1909 "but 1-D allocations are not present."
1910 << suffix);
1911
1912 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->indicesAreAllocated() &&
1913 ((rowPtrsUnpacked_host.extent(0) != 0 ||
1914 this->k_numRowEntries_.extent(0) != 0) ||
1915 this->lclIndsUnpacked_wdv.extent(0) != 0 ||
1916 this->gblInds_wdv.extent(0) != 0),
1917 std::logic_error,
1918 "If indices are not allocated, "
1919 "then none of the buffers should be."
1920 << suffix);
1921 // indices may be local or global only if they are allocated
1922 // (numAllocated is redundant; could simply be indicesAreLocal_ ||
1923 // indicesAreGlobal_)
1924 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC((this->indicesAreLocal_ || this->indicesAreGlobal_) &&
1925 !this->indicesAreAllocated_,
1926 std::logic_error,
1927 "Indices may be local or global only if they are "
1928 "allocated."
1929 << suffix);
1930 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreLocal_ && this->indicesAreGlobal_,
1931 std::logic_error, "Indices may not be both local and global." << suffix);
1932 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreLocal_ && gblInds_wdv.extent(0) != 0,
1933 std::logic_error,
1934 "Indices are local, but "
1935 "gblInds_wdv.extent(0) (= "
1936 << gblInds_wdv.extent(0)
1937 << ") != 0. In other words, if indices are local, then "
1938 "allocations of global indices should not be present."
1939 << suffix);
1940 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreGlobal_ && lclIndsUnpacked_wdv.extent(0) != 0,
1941 std::logic_error,
1942 "Indices are global, but "
1943 "lclIndsUnpacked_wdv.extent(0) (= "
1944 << lclIndsUnpacked_wdv.extent(0)
1945 << ") != 0. In other words, if indices are global, "
1946 "then allocations for local indices should not be present."
1947 << suffix);
1948 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreLocal_ && nodeAllocSize > 0 &&
1949 lclIndsUnpacked_wdv.extent(0) == 0 && getLocalNumRows() > 0,
1950 std::logic_error,
1951 "Indices are local and "
1952 "getLocalAllocationSize() = "
1953 << nodeAllocSize << " > 0, but "
1954 "lclIndsUnpacked_wdv.extent(0) = 0 and getLocalNumRows() = "
1955 << getLocalNumRows() << " > 0." << suffix);
1956 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreGlobal_ && nodeAllocSize > 0 &&
1957 gblInds_wdv.extent(0) == 0 && getLocalNumRows() > 0,
1958 std::logic_error,
1959 "Indices are global and "
1960 "getLocalAllocationSize() = "
1961 << nodeAllocSize << " > 0, but "
1962 "gblInds_wdv.extent(0) = 0 and getLocalNumRows() = "
1963 << getLocalNumRows() << " > 0." << suffix);
1964 // check the actual allocations
1965 if (this->indicesAreAllocated() &&
1966 rowPtrsUnpacked_host.extent(0) != 0) {
1967 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(rowPtrsUnpacked_host.extent(0)) !=
1968 this->getLocalNumRows() + 1,
1969 std::logic_error,
1970 "Indices are allocated and "
1971 "k_rowPtrs_ has nonzero length, but rowPtrsUnpacked_host_.extent(0) = "
1972 << rowPtrsUnpacked_host.extent(0) << " != getLocalNumRows()+1 = "
1973 << (this->getLocalNumRows() + 1) << "." << suffix);
1974 const size_t actualNumAllocated =
1975 rowPtrsUnpacked_host(this->getLocalNumRows());
1976 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() &&
1977 static_cast<size_t>(this->lclIndsUnpacked_wdv.extent(0)) != actualNumAllocated,
1978 std::logic_error,
1979 "Graph is locally indexed, indices are "
1980 "are allocated, and k_rowPtrs_ has nonzero length, but "
1981 "lclIndsUnpacked_wdv.extent(0) = "
1982 << this->lclIndsUnpacked_wdv.extent(0)
1983 << " != actualNumAllocated = " << actualNumAllocated << suffix);
1984 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isGloballyIndexed() &&
1985 static_cast<size_t>(this->gblInds_wdv.extent(0)) != actualNumAllocated,
1986 std::logic_error,
1987 "Graph is globally indexed, indices "
1988 "are allocated, and k_rowPtrs_ has nonzero length, but "
1989 "gblInds_wdv.extent(0) = "
1990 << this->gblInds_wdv.extent(0)
1991 << " != actualNumAllocated = " << actualNumAllocated << suffix);
1992 }
1993
1994 if (verbose_) {
1995 std::ostringstream os;
1996 os << *prefix << "Done" << endl;
1997 std::cerr << os.str();
1998 }
1999 }
2000}
2001
2002template <class LocalOrdinal, class GlobalOrdinal, class Node>
2003size_t
2004CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2005 getNumEntriesInGlobalRow(GlobalOrdinal globalRow) const {
2006 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(globalRow);
2007 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2008 return Teuchos::OrdinalTraits<size_t>::invalid();
2009 } else {
2010 return rowInfo.numEntries;
2011 }
2012}
2013
2014template <class LocalOrdinal, class GlobalOrdinal, class Node>
2015size_t
2016CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2017 getNumEntriesInLocalRow(LocalOrdinal localRow) const {
2018 const RowInfo rowInfo = this->getRowInfo(localRow);
2019 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2020 return Teuchos::OrdinalTraits<size_t>::invalid();
2021 } else {
2022 return rowInfo.numEntries;
2023 }
2024}
2025
2026template <class LocalOrdinal, class GlobalOrdinal, class Node>
2027size_t
2028CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2029 getNumAllocatedEntriesInGlobalRow(GlobalOrdinal globalRow) const {
2030 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(globalRow);
2031 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2032 return Teuchos::OrdinalTraits<size_t>::invalid();
2033 } else {
2034 return rowInfo.allocSize;
2035 }
2036}
2037
2038template <class LocalOrdinal, class GlobalOrdinal, class Node>
2039size_t
2040CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2041 getNumAllocatedEntriesInLocalRow(LocalOrdinal localRow) const {
2042 const RowInfo rowInfo = this->getRowInfo(localRow);
2043 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2044 return Teuchos::OrdinalTraits<size_t>::invalid();
2045 } else {
2046 return rowInfo.allocSize;
2047 }
2048}
2049
2050template <class LocalOrdinal, class GlobalOrdinal, class Node>
2051typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_host_view_type
2056
2057template <class LocalOrdinal, class GlobalOrdinal, class Node>
2058typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_device_view_type
2063
2064template <class LocalOrdinal, class GlobalOrdinal, class Node>
2065typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_host_view_type
2066CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2067 getLocalIndicesHost() const {
2068 return lclIndsPacked_wdv.getHostView(Access::ReadOnly);
2069}
2070
2071template <class LocalOrdinal, class GlobalOrdinal, class Node>
2072typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_device_view_type
2073CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2074 getLocalIndicesDevice() const {
2075 return lclIndsPacked_wdv.getDeviceView(Access::ReadOnly);
2076}
2077
2078template <class LocalOrdinal, class GlobalOrdinal, class Node>
2079void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2080 getLocalRowCopy(LocalOrdinal localRow,
2081 nonconst_local_inds_host_view_type& indices,
2082 size_t& numEntries) const {
2083 using Teuchos::ArrayView;
2084 const char tfecfFuncName[] = "getLocalRowCopy: ";
2085
2086 TEUCHOS_TEST_FOR_EXCEPTION(
2087 isGloballyIndexed() && !hasColMap(), std::runtime_error,
2088 "Tpetra::CrsGraph::getLocalRowCopy: The graph is globally indexed and "
2089 "does not have a column Map yet. That means we don't have local indices "
2090 "for columns yet, so it doesn't make sense to call this method. If the "
2091 "graph doesn't have a column Map yet, you should call fillComplete on "
2092 "it first.");
2093
2094 // This does the right thing (reports an empty row) if the input
2095 // row is invalid.
2096 const RowInfo rowinfo = this->getRowInfo(localRow);
2097 // No side effects on error.
2098 const size_t theNumEntries = rowinfo.numEntries;
2099 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) < theNumEntries, std::runtime_error,
2100 "Specified storage (size==" << indices.size() << ") does not suffice "
2101 "to hold all "
2102 << theNumEntries << " entry/ies for this row.");
2103 numEntries = theNumEntries;
2104
2105 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid()) {
2106 if (isLocallyIndexed()) {
2107 auto lclInds = getLocalIndsViewHost(rowinfo);
2108 for (size_t j = 0; j < theNumEntries; ++j) {
2109 indices[j] = lclInds(j);
2110 }
2111 } else if (isGloballyIndexed()) {
2112 auto gblInds = getGlobalIndsViewHost(rowinfo);
2113 for (size_t j = 0; j < theNumEntries; ++j) {
2114 indices[j] = colMap_->getLocalElement(gblInds(j));
2115 }
2116 }
2117 }
2118}
2119
2120template <class LocalOrdinal, class GlobalOrdinal, class Node>
2121void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2122 getGlobalRowCopy(GlobalOrdinal globalRow,
2123 nonconst_global_inds_host_view_type& indices,
2124 size_t& numEntries) const {
2125 using Teuchos::ArrayView;
2126 const char tfecfFuncName[] = "getGlobalRowCopy: ";
2127
2128 // This does the right thing (reports an empty row) if the input
2129 // row is invalid.
2130 const RowInfo rowinfo = getRowInfoFromGlobalRowIndex(globalRow);
2131 const size_t theNumEntries = rowinfo.numEntries;
2132 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2133 static_cast<size_t>(indices.size()) < theNumEntries, std::runtime_error,
2134 "Specified storage (size==" << indices.size() << ") does not suffice "
2135 "to hold all "
2136 << theNumEntries << " entry/ies for this row.");
2137 numEntries = theNumEntries; // first side effect
2138
2139 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid()) {
2140 if (isLocallyIndexed()) {
2141 auto lclInds = getLocalIndsViewHost(rowinfo);
2142 bool err = colMap_->getGlobalElements(lclInds.data(), theNumEntries, indices.data());
2143 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(err, std::runtime_error, "getGlobalElements error");
2144 } else if (isGloballyIndexed()) {
2145 auto gblInds = getGlobalIndsViewHost(rowinfo);
2146 std::memcpy(
2147 (void*)indices.data(),
2148 (const void*)gblInds.data(),
2149 theNumEntries * sizeof(*indices.data()));
2150 }
2151 }
2152}
2153
2154template <class LocalOrdinal, class GlobalOrdinal, class Node>
2155void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2156 getLocalRowView(
2157 const LocalOrdinal localRow,
2158 local_inds_host_view_type& indices) const {
2159 const char tfecfFuncName[] = "getLocalRowView: ";
2160
2161 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed(), std::runtime_error,
2162 "The graph's indices are "
2163 "currently stored as global indices, so we cannot return a view with "
2164 "local column indices, whether or not the graph has a column Map. If "
2165 "the graph _does_ have a column Map, use getLocalRowCopy() instead.");
2166
2167 const RowInfo rowInfo = getRowInfo(localRow);
2168 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid() &&
2169 rowInfo.numEntries > 0) {
2170 indices = lclIndsUnpacked_wdv.getHostSubview(rowInfo.offset1D,
2171 rowInfo.numEntries,
2172 Access::ReadOnly);
2173 } else {
2174 // This does the right thing (reports an empty row) if the input
2175 // row is invalid.
2176 indices = local_inds_host_view_type();
2177 }
2178
2179 if (debug_) {
2180 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) !=
2181 getNumEntriesInLocalRow(localRow),
2182 std::logic_error,
2183 "indices.size() "
2184 "= " << indices.extent(0)
2185 << " != getNumEntriesInLocalRow(localRow=" << localRow << ") = " << getNumEntriesInLocalRow(localRow) << ". Please report this bug to the Tpetra developers.");
2186 }
2187}
2188
2189template <class LocalOrdinal, class GlobalOrdinal, class Node>
2190void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2191 getGlobalRowView(
2192 const GlobalOrdinal globalRow,
2193 global_inds_host_view_type& indices) const {
2194 const char tfecfFuncName[] = "getGlobalRowView: ";
2195
2196 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isLocallyIndexed(), std::runtime_error,
2197 "The graph's indices are "
2198 "currently stored as local indices, so we cannot return a view with "
2199 "global column indices. Use getGlobalRowCopy() instead.");
2200
2201 // This does the right thing (reports an empty row) if the input
2202 // row is invalid.
2203 const RowInfo rowInfo = getRowInfoFromGlobalRowIndex(globalRow);
2204 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid() &&
2205 rowInfo.numEntries > 0) {
2206 indices = gblInds_wdv.getHostSubview(rowInfo.offset1D,
2207 rowInfo.numEntries,
2208 Access::ReadOnly);
2209 } else {
2210 indices = typename global_inds_dualv_type::t_host::const_type();
2211 }
2212 if (debug_) {
2213 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) !=
2214 getNumEntriesInGlobalRow(globalRow),
2215 std::logic_error, "indices.size() = " << indices.extent(0) << " != getNumEntriesInGlobalRow(globalRow=" << globalRow << ") = " << getNumEntriesInGlobalRow(globalRow) << ". Please report this bug to the Tpetra developers.");
2216 }
2217}
2218
2219template <class LocalOrdinal, class GlobalOrdinal, class Node>
2220void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2221 insertLocalIndices(const LocalOrdinal localRow,
2222 const Teuchos::ArrayView<const LocalOrdinal>& indices) {
2223 const char tfecfFuncName[] = "insertLocalIndices: ";
2224
2225 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive(), std::runtime_error, "Fill must be active.");
2226 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed(), std::runtime_error,
2227 "Graph indices are global; use insertGlobalIndices().");
2228 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error,
2229 "Cannot insert local indices without a column Map.");
2230 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!rowMap_->isNodeLocalElement(localRow), std::runtime_error,
2231 "Local row index " << localRow << " is not in the row Map "
2232 "on the calling process.");
2233 if (!indicesAreAllocated()) {
2234 allocateIndices(LocalIndices, verbose_);
2235 }
2236
2237 if (debug_) {
2238 // In debug mode, if the graph has a column Map, test whether any
2239 // of the given column indices are not in the column Map. Keep
2240 // track of the invalid column indices so we can tell the user
2241 // about them.
2242 if (hasColMap()) {
2243 using std::endl;
2244 using Teuchos::Array;
2245 using Teuchos::toString;
2246 typedef typename Teuchos::ArrayView<const LocalOrdinal>::size_type size_type;
2247
2248 const map_type& colMap = *colMap_;
2249 Array<LocalOrdinal> badColInds;
2250 bool allInColMap = true;
2251 for (size_type k = 0; k < indices.size(); ++k) {
2252 if (!colMap.isNodeLocalElement(indices[k])) {
2253 allInColMap = false;
2254 badColInds.push_back(indices[k]);
2255 }
2256 }
2257 if (!allInColMap) {
2258 std::ostringstream os;
2259 os << "Tpetra::CrsGraph::insertLocalIndices: You attempted to insert "
2260 "entries in owned row "
2261 << localRow << ", at the following column "
2262 "indices: "
2263 << toString(indices) << "." << endl;
2264 os << "Of those, the following indices are not in the column Map on "
2265 "this process: "
2266 << toString(badColInds) << "." << endl
2267 << "Since "
2268 "the graph has a column Map already, it is invalid to insert entries "
2269 "at those locations.";
2270 TEUCHOS_TEST_FOR_EXCEPTION(!allInColMap, std::invalid_argument, os.str());
2271 }
2272 }
2273 }
2274
2275 insertLocalIndicesImpl(localRow, indices);
2276
2277 if (debug_) {
2278 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!indicesAreAllocated() || !isLocallyIndexed(), std::logic_error,
2279 "At the end of insertLocalIndices, ! indicesAreAllocated() || "
2280 "! isLocallyIndexed() is true. Please report this bug to the "
2281 "Tpetra developers.");
2282 }
2283}
2284
2285template <class LocalOrdinal, class GlobalOrdinal, class Node>
2286void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2287 insertLocalIndices(const LocalOrdinal localRow,
2288 const LocalOrdinal numEnt,
2289 const LocalOrdinal inds[]) {
2290 Teuchos::ArrayView<const LocalOrdinal> indsT(inds, numEnt);
2291 this->insertLocalIndices(localRow, indsT);
2292}
2293
2294template <class LocalOrdinal, class GlobalOrdinal, class Node>
2295void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2296 insertGlobalIndices(const GlobalOrdinal gblRow,
2297 const LocalOrdinal numInputInds,
2298 const GlobalOrdinal inputGblColInds[]) {
2299 typedef LocalOrdinal LO;
2300 const char tfecfFuncName[] = "insertGlobalIndices: ";
2301
2302 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed(), std::runtime_error,
2303 "graph indices are local; use insertLocalIndices().");
2304 // This can't really be satisfied for now, because if we are
2305 // fillComplete(), then we are local. In the future, this may
2306 // change. However, the rule that modification require active
2307 // fill will not change.
2308 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->isFillActive(), std::runtime_error,
2309 "You are not allowed to call this method if fill is not active. "
2310 "If fillComplete has been called, you must first call resumeFill "
2311 "before you may insert indices.");
2312 if (!indicesAreAllocated()) {
2313 allocateIndices(GlobalIndices, verbose_);
2314 }
2315 const LO lclRow = this->rowMap_->getLocalElement(gblRow);
2316 if (lclRow != Tpetra::Details::OrdinalTraits<LO>::invalid()) {
2317 if (debug_) {
2318 if (this->hasColMap()) {
2319 using std::endl;
2320 const map_type& colMap = *(this->colMap_);
2321 // In a debug build, keep track of the nonowned ("bad") column
2322 // indices, so that we can display them in the exception
2323 // message. In a release build, just ditch the loop early if
2324 // we encounter a nonowned column index.
2325 std::vector<GlobalOrdinal> badColInds;
2326 bool allInColMap = true;
2327 for (LO k = 0; k < numInputInds; ++k) {
2328 if (!colMap.isNodeGlobalElement(inputGblColInds[k])) {
2329 allInColMap = false;
2330 badColInds.push_back(inputGblColInds[k]);
2331 }
2332 }
2333 if (!allInColMap) {
2334 std::ostringstream os;
2335 os << "You attempted to insert entries in owned row " << gblRow
2336 << ", at the following column indices: [";
2337 for (LO k = 0; k < numInputInds; ++k) {
2338 os << inputGblColInds[k];
2339 if (k + static_cast<LO>(1) < numInputInds) {
2340 os << ",";
2341 }
2342 }
2343 os << "]." << endl
2344 << "Of those, the following indices are not in "
2345 "the column Map on this process: [";
2346 for (size_t k = 0; k < badColInds.size(); ++k) {
2347 os << badColInds[k];
2348 if (k + size_t(1) < badColInds.size()) {
2349 os << ",";
2350 }
2351 }
2352 os << "]." << endl
2353 << "Since the matrix has a column Map already, "
2354 "it is invalid to insert entries at those locations.";
2355 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::invalid_argument, os.str());
2356 }
2357 }
2358 } // debug_
2359 this->insertGlobalIndicesImpl(lclRow, inputGblColInds, numInputInds);
2360 } else { // a nonlocal row
2361 this->insertGlobalIndicesIntoNonownedRows(gblRow, inputGblColInds,
2362 numInputInds);
2363 }
2364}
2365
2366template <class LocalOrdinal, class GlobalOrdinal, class Node>
2367void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2368 insertGlobalIndices(const GlobalOrdinal gblRow,
2369 const Teuchos::ArrayView<const GlobalOrdinal>& inputGblColInds) {
2370 this->insertGlobalIndices(gblRow, inputGblColInds.size(),
2371 inputGblColInds.getRawPtr());
2372}
2373
2374template <class LocalOrdinal, class GlobalOrdinal, class Node>
2375void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2376 insertGlobalIndicesFiltered(const LocalOrdinal lclRow,
2377 const GlobalOrdinal gblColInds[],
2378 const LocalOrdinal numGblColInds) {
2379 typedef LocalOrdinal LO;
2380 typedef GlobalOrdinal GO;
2381 const char tfecfFuncName[] = "insertGlobalIndicesFiltered: ";
2382
2383 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed(), std::runtime_error,
2384 "Graph indices are local; use insertLocalIndices().");
2385 // This can't really be satisfied for now, because if we are
2386 // fillComplete(), then we are local. In the future, this may
2387 // change. However, the rule that modification require active
2388 // fill will not change.
2389 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->isFillActive(), std::runtime_error,
2390 "You are not allowed to call this method if fill is not active. "
2391 "If fillComplete has been called, you must first call resumeFill "
2392 "before you may insert indices.");
2393 if (!indicesAreAllocated()) {
2394 allocateIndices(GlobalIndices, verbose_);
2395 }
2396
2397 Teuchos::ArrayView<const GO> gblColInds_av(gblColInds, numGblColInds);
2398 // If we have a column Map, use it to filter the entries.
2399 if (!colMap_.is_null()) {
2400 const map_type& colMap = *(this->colMap_);
2401
2402 LO curOffset = 0;
2403 while (curOffset < numGblColInds) {
2404 // Find a sequence of input indices that are in the column Map
2405 // on the calling process. Doing a sequence at a time,
2406 // instead of one at a time, amortizes some overhead.
2407 LO endOffset = curOffset;
2408 for (; endOffset < numGblColInds; ++endOffset) {
2409 const LO lclCol = colMap.getLocalElement(gblColInds[endOffset]);
2410 if (lclCol == Tpetra::Details::OrdinalTraits<LO>::invalid()) {
2411 break; // first entry, in current sequence, not in the column Map
2412 }
2413 }
2414 // curOffset, endOffset: half-exclusive range of indices in
2415 // the column Map on the calling process. If endOffset ==
2416 // curOffset, the range is empty.
2417 const LO numIndInSeq = (endOffset - curOffset);
2418 if (numIndInSeq != 0) {
2419 this->insertGlobalIndicesImpl(lclRow, gblColInds + curOffset,
2420 numIndInSeq);
2421 }
2422 // Invariant before this line: Either endOffset ==
2423 // numGblColInds, or gblColInds[endOffset] is not in the
2424 // column Map on the calling process.
2425 curOffset = endOffset + 1;
2426 }
2427 } else {
2428 this->insertGlobalIndicesImpl(lclRow, gblColInds_av.getRawPtr(),
2429 gblColInds_av.size());
2430 }
2431}
2432
2433template <class LocalOrdinal, class GlobalOrdinal, class Node>
2434void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2435 insertGlobalIndicesIntoNonownedRows(const GlobalOrdinal gblRow,
2436 const GlobalOrdinal gblColInds[],
2437 const LocalOrdinal numGblColInds) {
2438 // This creates the std::vector if it doesn't exist yet.
2439 // std::map's operator[] does a lookup each time, so it's better
2440 // to pull nonlocals_[grow] out of the loop.
2441 std::vector<GlobalOrdinal>& nonlocalRow = this->nonlocals_[gblRow];
2442 for (LocalOrdinal k = 0; k < numGblColInds; ++k) {
2443 // FIXME (mfh 20 Jul 2017) Would be better to use a set, in
2444 // order to avoid duplicates. globalAssemble() sorts these
2445 // anyway.
2446 nonlocalRow.push_back(gblColInds[k]);
2447 }
2448}
2449
2450template <class LocalOrdinal, class GlobalOrdinal, class Node>
2451void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2452 removeLocalIndices(LocalOrdinal lrow) {
2453 const char tfecfFuncName[] = "removeLocalIndices: ";
2454 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2455 !isFillActive(), std::runtime_error, "requires that fill is active.");
2456 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2457 isStorageOptimized(), std::runtime_error,
2458 "cannot remove indices after optimizeStorage() has been called.");
2459 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2460 isGloballyIndexed(), std::runtime_error, "graph indices are global.");
2461 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2462 !rowMap_->isNodeLocalElement(lrow), std::runtime_error,
2463 "Local row " << lrow << " is not in the row Map on the calling process.");
2464 if (!indicesAreAllocated()) {
2465 allocateIndices(LocalIndices, verbose_);
2466 }
2467
2468 if (k_numRowEntries_.extent(0) != 0) {
2469 this->k_numRowEntries_(lrow) = 0;
2470 }
2471
2472 if (debug_) {
2473 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(getNumEntriesInLocalRow(lrow) != 0 ||
2474 !indicesAreAllocated() ||
2475 !isLocallyIndexed(),
2476 std::logic_error,
2477 "Violated stated post-conditions. Please contact Tpetra team.");
2478 }
2479}
2480
2481template <class LocalOrdinal, class GlobalOrdinal, class Node>
2482void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2483 setAllIndices(const typename local_graph_device_type::row_map_type& rowPointers,
2484 const typename local_graph_device_type::entries_type::non_const_type& columnIndices) {
2485 using ProfilingRegion = Details::ProfilingRegion;
2486 ProfilingRegion region("Tpetra::CrsGraph::setAllIndices");
2487 const char tfecfFuncName[] = "setAllIndices: ";
2488 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2489 !hasColMap() || getColMap().is_null(), std::runtime_error,
2490 "The graph must have a column Map before you may call this method.");
2491 LocalOrdinal numLocalRows = this->getLocalNumRows();
2492 {
2493 LocalOrdinal rowPtrLen = rowPointers.size();
2494 if (numLocalRows == 0) {
2495 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2496 rowPtrLen != 0 && rowPtrLen != 1,
2497 std::runtime_error, "Have 0 local rows, but rowPointers.size() is neither 0 nor 1.");
2498 } else {
2499 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2500 rowPtrLen != numLocalRows + 1,
2501 std::runtime_error, "rowPointers.size() = " << rowPtrLen << " != this->getLocalNumRows()+1 = " << (numLocalRows + 1) << ".");
2502 }
2503 }
2504
2505 if (debug_) {
2506 using exec_space = typename local_graph_device_type::execution_space;
2507 int columnsOutOfBounds = 0;
2508 local_ordinal_type numLocalCols = this->getLocalNumCols();
2509 Kokkos::parallel_reduce(
2510 Kokkos::RangePolicy<exec_space>(0, columnIndices.extent(0)),
2511 KOKKOS_LAMBDA(const LocalOrdinal i, int& lOutOfBounds) {
2512 if (columnIndices(i) < 0 || columnIndices(i) >= numLocalCols)
2513 lOutOfBounds++;
2514 },
2515 columnsOutOfBounds);
2516 int globalColsOutOfBounds = 0;
2517 auto comm = this->getComm();
2518 Teuchos::reduceAll<int, int>(*comm, Teuchos::REDUCE_MAX, columnsOutOfBounds,
2519 Teuchos::outArg(globalColsOutOfBounds));
2520 if (globalColsOutOfBounds) {
2521 std::string message;
2522 if (columnsOutOfBounds) {
2523 // Only print message from ranks with the problem
2524 message = std::string("ERROR, rank ") + std::to_string(comm->getRank()) + ", CrsGraph::setAllIndices(): provided columnIndices are not all within range [0, getLocalNumCols())!\n";
2525 }
2526 Details::gathervPrint(std::cout, message, *comm);
2527 throw std::invalid_argument("CrsGraph::setAllIndices(): columnIndices are out of the valid range on at least one process.");
2528 }
2529 }
2530
2531 if (debug_ && this->isSorted()) {
2532 // Verify that the local indices are actually sorted
2533 int notSorted = 0;
2534 using exec_space = typename local_graph_device_type::execution_space;
2535 using size_type = typename local_graph_device_type::size_type;
2536 Kokkos::parallel_reduce(
2537 Kokkos::RangePolicy<exec_space>(0, numLocalRows),
2538 KOKKOS_LAMBDA(const LocalOrdinal i, int& lNotSorted) {
2539 size_type rowBegin = rowPointers(i);
2540 size_type rowEnd = rowPointers(i + 1);
2541 for (size_type j = rowBegin + 1; j < rowEnd; j++) {
2542 if (columnIndices(j - 1) > columnIndices(j)) {
2543 lNotSorted = 1;
2544 }
2545 }
2546 },
2547 notSorted);
2548 // All-reduce notSorted to avoid rank divergence
2549 int globalNotSorted = 0;
2550 auto comm = this->getComm();
2551 Teuchos::reduceAll<int, int>(*comm, Teuchos::REDUCE_MAX, notSorted,
2552 Teuchos::outArg(globalNotSorted));
2553 if (globalNotSorted) {
2554 std::string message;
2555 if (notSorted) {
2556 // Only print message from ranks with the problem
2557 message = std::string("ERROR, rank ") + std::to_string(comm->getRank()) + ", CrsGraph::setAllIndices(): provided columnIndices are not sorted!\n";
2558 }
2559 Details::gathervPrint(std::cout, message, *comm);
2560 throw std::invalid_argument("CrsGraph::setAllIndices(): provided columnIndices are not sorted within rows on at least one process.");
2561 }
2562 }
2563
2564 indicesAreAllocated_ = true;
2565 indicesAreLocal_ = true;
2566 indicesAreSorted_ = true;
2567 noRedundancies_ = true;
2568 lclIndsPacked_wdv = local_inds_wdv_type(columnIndices);
2569 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
2570 setRowPtrs(rowPointers);
2571
2572 set_need_sync_host_uvm_access(); // columnIndices and rowPointers potentially still in a kernel
2573
2574 // Storage MUST be packed, since the interface doesn't give any
2575 // way to indicate any extra space at the end of each row.
2576 storageStatus_ = Details::STORAGE_1D_PACKED;
2577
2578 // These normally get cleared out at the end of allocateIndices.
2579 // It makes sense to clear them out here, because at the end of
2580 // this method, the graph is allocated on the calling process.
2581 numAllocForAllRows_ = 0;
2582 k_numAllocPerRow_ = decltype(k_numAllocPerRow_)();
2583
2584 checkInternalState();
2585}
2586
2587template <class LocalOrdinal, class GlobalOrdinal, class Node>
2588void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2589 setAllIndices(const Teuchos::ArrayRCP<size_t>& rowPointers,
2590 const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices) {
2591 using Kokkos::View;
2592 typedef typename local_graph_device_type::row_map_type row_map_type;
2593 typedef typename row_map_type::array_layout layout_type;
2594 typedef typename row_map_type::non_const_value_type row_offset_type;
2595 typedef View<size_t*, layout_type, Kokkos::HostSpace,
2596 Kokkos::MemoryUnmanaged>
2597 input_view_type;
2598 typedef typename row_map_type::non_const_type nc_row_map_type;
2599
2600 const size_t size = static_cast<size_t>(rowPointers.size());
2601 constexpr bool same = std::is_same<size_t, row_offset_type>::value;
2602 input_view_type ptr_in(rowPointers.getRawPtr(), size);
2603
2604 nc_row_map_type ptr_rot("Tpetra::CrsGraph::ptr", size);
2605
2606 if constexpr (same) { // size_t == row_offset_type
2607 using lexecution_space = typename device_type::execution_space;
2608 Kokkos::deep_copy(lexecution_space(),
2609 ptr_rot,
2610 ptr_in);
2611 } else { // size_t != row_offset_type
2612 // CudaUvmSpace != HostSpace, so this will be false in that case.
2613 constexpr bool inHostMemory =
2614 std::is_same<typename row_map_type::memory_space,
2615 Kokkos::HostSpace>::value;
2616 if (inHostMemory) {
2617 // Copy (with cast from size_t to row_offset_type, with bounds
2618 // checking if necessary) to ptr_rot.
2619 ::Tpetra::Details::copyOffsets(ptr_rot, ptr_in);
2620 } else { // Copy input row offsets to device first.
2621 //
2622 // FIXME (mfh 24 Mar 2015) If CUDA UVM, running in the host's
2623 // execution space would avoid the double copy.
2624 //
2625 View<size_t*, layout_type, device_type> ptr_st("Tpetra::CrsGraph::ptr", size);
2626
2627 // DEEP_COPY REVIEW - NOT TESTED
2628 Kokkos::deep_copy(ptr_st, ptr_in);
2629 // Copy on device (casting from size_t to row_offset_type,
2630 // with bounds checking if necessary) to ptr_rot. This
2631 // executes in the output View's execution space, which is the
2632 // same as execution_space.
2633 ::Tpetra::Details::copyOffsets(ptr_rot, ptr_st);
2634 }
2635 }
2636
2637 Kokkos::View<LocalOrdinal*, layout_type, device_type> k_ind =
2638 Kokkos::Compat::getKokkosViewDeepCopy<device_type>(columnIndices());
2639 setAllIndices(ptr_rot, k_ind);
2640}
2641
2642template <class LocalOrdinal, class GlobalOrdinal, class Node>
2643void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2644 globalAssemble() {
2645 using std::endl;
2646 using Teuchos::Comm;
2647 using Teuchos::outArg;
2648 using Teuchos::RCP;
2649 using Teuchos::rcp;
2650 using Teuchos::REDUCE_MAX;
2651 using Teuchos::REDUCE_MIN;
2652 using Teuchos::reduceAll;
2653 using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
2654 using LO = local_ordinal_type;
2655 using GO = global_ordinal_type;
2656 using size_type = typename Teuchos::Array<GO>::size_type;
2657
2658 const char tfecfFuncName[] = "globalAssemble: "; // for exception macro
2659
2660 Details::ProfilingRegion regionGA("Tpetra::CrsGraph::globalAssemble");
2661
2662 std::unique_ptr<std::string> prefix;
2663 if (verbose_) {
2664 prefix = this->createPrefix("CrsGraph", "globalAssemble");
2665 std::ostringstream os;
2666 os << *prefix << "Start" << endl;
2667 std::cerr << os.str();
2668 }
2669 RCP<const Comm<int>> comm = getComm();
2670
2671 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive(), std::runtime_error,
2672 "Fill must be active before "
2673 "you may call this method.");
2674
2675 const size_t myNumNonlocalRows = this->nonlocals_.size();
2676
2677 // If no processes have nonlocal rows, then we don't have to do
2678 // anything. Checking this is probably cheaper than constructing
2679 // the Map of nonlocal rows (see below) and noticing that it has
2680 // zero global entries.
2681 {
2682 const int iHaveNonlocalRows = (myNumNonlocalRows == 0) ? 0 : 1;
2683 int someoneHasNonlocalRows = 0;
2684 reduceAll<int, int>(*comm, REDUCE_MAX, iHaveNonlocalRows,
2685 outArg(someoneHasNonlocalRows));
2686 if (someoneHasNonlocalRows == 0) {
2687 if (verbose_) {
2688 std::ostringstream os;
2689 os << *prefix << "Done: No nonlocal rows" << endl;
2690 std::cerr << os.str();
2691 }
2692 return;
2693 } else if (verbose_) {
2694 std::ostringstream os;
2695 os << *prefix << "At least 1 process has nonlocal rows"
2696 << endl;
2697 std::cerr << os.str();
2698 }
2699 }
2700
2701 // 1. Create a list of the "nonlocal" rows on each process. this
2702 // requires iterating over nonlocals_, so while we do this,
2703 // deduplicate the entries and get a count for each nonlocal
2704 // row on this process.
2705 // 2. Construct a new row Map corresponding to those rows. This
2706 // Map is likely overlapping. We know that the Map is not
2707 // empty on all processes, because the above all-reduce and
2708 // return exclude that case.
2709
2710 RCP<const map_type> nonlocalRowMap;
2711 // Keep this for CrsGraph's constructor.
2712 Teuchos::Array<size_t> numEntPerNonlocalRow(myNumNonlocalRows);
2713 {
2714 Teuchos::Array<GO> myNonlocalGblRows(myNumNonlocalRows);
2715 size_type curPos = 0;
2716 for (auto mapIter = this->nonlocals_.begin();
2717 mapIter != this->nonlocals_.end();
2718 ++mapIter, ++curPos) {
2719 myNonlocalGblRows[curPos] = mapIter->first;
2720 std::vector<GO>& gblCols = mapIter->second; // by ref; change in place
2721 std::sort(gblCols.begin(), gblCols.end());
2722 auto vecLast = std::unique(gblCols.begin(), gblCols.end());
2723 gblCols.erase(vecLast, gblCols.end());
2724 numEntPerNonlocalRow[curPos] = gblCols.size();
2725 }
2726
2727 // Currently, Map requires that its indexBase be the global min
2728 // of all its global indices. Map won't compute this for us, so
2729 // we must do it. If our process has no nonlocal rows, set the
2730 // "min" to the max possible GO value. This ensures that if
2731 // some process has at least one nonlocal row, then it will pick
2732 // that up as the min. We know that at least one process has a
2733 // nonlocal row, since the all-reduce and return at the top of
2734 // this method excluded that case.
2735 GO myMinNonlocalGblRow = std::numeric_limits<GO>::max();
2736 {
2737 auto iter = std::min_element(myNonlocalGblRows.begin(),
2738 myNonlocalGblRows.end());
2739 if (iter != myNonlocalGblRows.end()) {
2740 myMinNonlocalGblRow = *iter;
2741 }
2742 }
2743 GO gblMinNonlocalGblRow = 0;
2744 reduceAll<int, GO>(*comm, REDUCE_MIN, myMinNonlocalGblRow,
2745 outArg(gblMinNonlocalGblRow));
2746 const GO indexBase = gblMinNonlocalGblRow;
2747 const global_size_t INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
2748 nonlocalRowMap = rcp(new map_type(INV, myNonlocalGblRows(), indexBase, comm));
2749 }
2750
2751 if (verbose_) {
2752 std::ostringstream os;
2753 os << *prefix << "nonlocalRowMap->getIndexBase()="
2754 << nonlocalRowMap->getIndexBase() << endl;
2755 std::cerr << os.str();
2756 }
2757
2758 // 3. Use the column indices for each nonlocal row, as stored in
2759 // nonlocals_, to construct a CrsGraph corresponding to
2760 // nonlocal rows. We need, but we have, exact counts of the
2761 // number of entries in each nonlocal row.
2762
2763 RCP<crs_graph_type> nonlocalGraph =
2764 rcp(new crs_graph_type(nonlocalRowMap, numEntPerNonlocalRow()));
2765 {
2766 size_type curPos = 0;
2767 for (auto mapIter = this->nonlocals_.begin();
2768 mapIter != this->nonlocals_.end();
2769 ++mapIter, ++curPos) {
2770 const GO gblRow = mapIter->first;
2771 std::vector<GO>& gblCols = mapIter->second; // by ref just to avoid copy
2772 const LO numEnt = static_cast<LO>(numEntPerNonlocalRow[curPos]);
2773 nonlocalGraph->insertGlobalIndices(gblRow, numEnt, gblCols.data());
2774 }
2775 }
2776 if (verbose_) {
2777 std::ostringstream os;
2778 os << *prefix << "Built nonlocal graph" << endl;
2779 std::cerr << os.str();
2780 }
2781 // There's no need to fill-complete the nonlocals graph.
2782 // We just use it as a temporary container for the Export.
2783
2784 // 4. If the original row Map is one to one, then we can Export
2785 // directly from nonlocalGraph into this. Otherwise, we have
2786 // to create a temporary graph with a one-to-one row Map,
2787 // Export into that, then Import from the temporary graph into
2788 // *this.
2789
2790 auto origRowMap = this->getRowMap();
2791 const bool origRowMapIsOneToOne = origRowMap->isOneToOne();
2792
2793 if (origRowMapIsOneToOne) {
2794 if (verbose_) {
2795 std::ostringstream os;
2796 os << *prefix << "Original row Map is 1-to-1" << endl;
2797 std::cerr << os.str();
2798 }
2799 export_type exportToOrig(nonlocalRowMap, origRowMap);
2800 this->doExport(*nonlocalGraph, exportToOrig, Tpetra::INSERT);
2801 // We're done at this point!
2802 } else {
2803 if (verbose_) {
2804 std::ostringstream os;
2805 os << *prefix << "Original row Map is NOT 1-to-1" << endl;
2806 std::cerr << os.str();
2807 }
2808 // If you ask a Map whether it is one to one, it does some
2809 // communication and stashes intermediate results for later use
2810 // by createOneToOne. Thus, calling createOneToOne doesn't cost
2811 // much more then the original cost of calling isOneToOne.
2812 auto oneToOneRowMap = Tpetra::createOneToOne(origRowMap);
2813 export_type exportToOneToOne(nonlocalRowMap, oneToOneRowMap);
2814
2815 // Create a temporary graph with the one-to-one row Map.
2816 //
2817 // TODO (mfh 09 Sep 2016) Estimate the number of entries in each
2818 // row, to avoid reallocation during the Export operation.
2819 crs_graph_type oneToOneGraph(oneToOneRowMap, 0);
2820
2821 // Export from graph of nonlocals into the temp one-to-one graph.
2822 if (verbose_) {
2823 std::ostringstream os;
2824 os << *prefix << "Export nonlocal graph" << endl;
2825 std::cerr << os.str();
2826 }
2827 oneToOneGraph.doExport(*nonlocalGraph, exportToOneToOne, Tpetra::INSERT);
2828
2829 // We don't need the graph of nonlocals anymore, so get rid of
2830 // it, to keep the memory high-water mark down.
2831 nonlocalGraph = Teuchos::null;
2832
2833 // Import from the one-to-one graph to the original graph.
2834 import_type importToOrig(oneToOneRowMap, origRowMap);
2835 if (verbose_) {
2836 std::ostringstream os;
2837 os << *prefix << "Import nonlocal graph" << endl;
2838 std::cerr << os.str();
2839 }
2840 this->doImport(oneToOneGraph, importToOrig, Tpetra::INSERT);
2841 }
2842
2843 // It's safe now to clear out nonlocals_, since we've already
2844 // committed side effects to *this. The standard idiom for
2845 // clearing a Container like std::map, is to swap it with an empty
2846 // Container and let the swapped Container fall out of scope.
2847 decltype(this->nonlocals_) newNonlocals;
2848 std::swap(this->nonlocals_, newNonlocals);
2849
2850 checkInternalState();
2851 if (verbose_) {
2852 std::ostringstream os;
2853 os << *prefix << "Done" << endl;
2854 std::cerr << os.str();
2855 }
2856}
2857
2858template <class LocalOrdinal, class GlobalOrdinal, class Node>
2859void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2860 resumeFill(const Teuchos::RCP<Teuchos::ParameterList>& params) {
2861 clearGlobalConstants();
2862 if (params != Teuchos::null) this->setParameterList(params);
2863 // either still sorted/merged or initially sorted/merged
2864 indicesAreSorted_ = true;
2865 noRedundancies_ = true;
2866 fillComplete_ = false;
2867}
2868
2869template <class LocalOrdinal, class GlobalOrdinal, class Node>
2870void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2871 fillComplete(const Teuchos::RCP<Teuchos::ParameterList>& params) {
2872 // If the graph already has domain and range Maps, don't clobber
2873 // them. If it doesn't, use the current row Map for both the
2874 // domain and range Maps.
2875 //
2876 // NOTE (mfh 28 Sep 2014): If the graph was constructed without a
2877 // column Map, and column indices are inserted which are not in
2878 // the row Map on any process, this will cause troubles. However,
2879 // that is not a common case for most applications that we
2880 // encounter, and checking for it might require more
2881 // communication.
2882 Teuchos::RCP<const map_type> domMap = this->getDomainMap();
2883 if (domMap.is_null()) {
2884 domMap = this->getRowMap();
2885 }
2886 Teuchos::RCP<const map_type> ranMap = this->getRangeMap();
2887 if (ranMap.is_null()) {
2888 ranMap = this->getRowMap();
2889 }
2890 this->fillComplete(domMap, ranMap, params);
2891}
2892
2893template <class LocalOrdinal, class GlobalOrdinal, class Node>
2894void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2895 fillComplete(const Teuchos::RCP<const map_type>& domainMap,
2896 const Teuchos::RCP<const map_type>& rangeMap,
2897 const Teuchos::RCP<Teuchos::ParameterList>& params) {
2898 using std::endl;
2899
2900 const char tfecfFuncName[] = "fillComplete: ";
2901 const bool verbose = verbose_;
2902
2903 Details::ProfilingRegion regionFC("Tpetra::CrsGraph::fillComplete");
2904
2905 std::unique_ptr<std::string> prefix;
2906 if (verbose) {
2907 prefix = this->createPrefix("CrsGraph", "fillComplete");
2908 std::ostringstream os;
2909 os << *prefix << "Start" << endl;
2910 std::cerr << os.str();
2911 }
2912
2913 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive() || isFillComplete(), std::runtime_error,
2914 "Graph fill state must be active (isFillActive() "
2915 "must be true) before calling fillComplete().");
2916
2917 const int numProcs = getComm()->getSize();
2918
2919 //
2920 // Read and set parameters
2921 //
2922
2923 // Does the caller want to sort remote GIDs (within those owned by
2924 // the same process) in makeColMap()?
2925 if (!params.is_null()) {
2926 if (params->isParameter("sort column map ghost gids")) {
2927 sortGhostsAssociatedWithEachProcessor_ =
2928 params->get<bool>("sort column map ghost gids",
2929 sortGhostsAssociatedWithEachProcessor_);
2930 } else if (params->isParameter("Sort column Map ghost GIDs")) {
2931 sortGhostsAssociatedWithEachProcessor_ =
2932 params->get<bool>("Sort column Map ghost GIDs",
2933 sortGhostsAssociatedWithEachProcessor_);
2934 }
2935 }
2936
2937 // If true, the caller promises that no process did nonlocal
2938 // changes since the last call to fillComplete.
2939 bool assertNoNonlocalInserts = false;
2940 if (!params.is_null()) {
2941 assertNoNonlocalInserts =
2942 params->get<bool>("No Nonlocal Changes", assertNoNonlocalInserts);
2943 }
2944
2945 //
2946 // Allocate indices, if they haven't already been allocated
2947 //
2948 if (!indicesAreAllocated()) {
2949 if (hasColMap()) {
2950 // We have a column Map, so use local indices.
2951 allocateIndices(LocalIndices, verbose);
2952 } else {
2953 // We don't have a column Map, so use global indices.
2954 allocateIndices(GlobalIndices, verbose);
2955 }
2956 }
2957
2958 //
2959 // Do global assembly, if requested and if the communicator
2960 // contains more than one process.
2961 //
2962 const bool mayNeedGlobalAssemble = !assertNoNonlocalInserts && numProcs > 1;
2963 if (mayNeedGlobalAssemble) {
2964 // This first checks if we need to do global assembly.
2965 // The check costs a single all-reduce.
2966 globalAssemble();
2967 } else {
2968 const size_t numNonlocals = nonlocals_.size();
2969 if (verbose) {
2970 std::ostringstream os;
2971 os << *prefix << "Do not need to call globalAssemble; "
2972 "assertNoNonlocalInserts="
2973 << (assertNoNonlocalInserts ? "true" : "false")
2974 << "numProcs=" << numProcs
2975 << ", nonlocals_.size()=" << numNonlocals << endl;
2976 std::cerr << os.str();
2977 }
2978 const int lclNeededGlobalAssemble =
2979 (numProcs > 1 && numNonlocals != 0) ? 1 : 0;
2980 if (lclNeededGlobalAssemble != 0 && verbose) {
2981 std::ostringstream os;
2982 os << *prefix;
2983 Details::Impl::verbosePrintMap(
2984 os, nonlocals_.begin(), nonlocals_.end(),
2985 nonlocals_.size(), "nonlocals_");
2986 std::cerr << os.str() << endl;
2987 }
2988
2989 if (debug_) {
2990 auto map = this->getMap();
2991 auto comm = map.is_null() ? Teuchos::null : map->getComm();
2992 int gblNeededGlobalAssemble = lclNeededGlobalAssemble;
2993 if (!comm.is_null()) {
2994 using Teuchos::REDUCE_MAX;
2995 using Teuchos::reduceAll;
2996 reduceAll(*comm, REDUCE_MAX, lclNeededGlobalAssemble,
2997 Teuchos::outArg(gblNeededGlobalAssemble));
2998 }
2999 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(gblNeededGlobalAssemble != 0, std::runtime_error,
3000 "nonlocals_.size()=" << numNonlocals << " != 0 on at "
3001 "least one process in the CrsGraph's communicator. This "
3002 "means either that you incorrectly set the "
3003 "\"No Nonlocal Changes\" fillComplete parameter to true, "
3004 "or that you inserted invalid entries. "
3005 "Rerun with the environment variable TPETRA_VERBOSE="
3006 "CrsGraph set to see the entries of nonlocals_ on every "
3007 "MPI process (WARNING: lots of output).");
3008 } else {
3009 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(lclNeededGlobalAssemble != 0, std::runtime_error,
3010 "nonlocals_.size()=" << numNonlocals << " != 0 on the "
3011 "calling process. This means either that you incorrectly "
3012 "set the \"No Nonlocal Changes\" fillComplete parameter "
3013 "to true, or that you inserted invalid entries. "
3014 "Rerun with the environment "
3015 "variable TPETRA_VERBOSE=CrsGraph set to see the entries "
3016 "of nonlocals_ on every MPI process (WARNING: lots of "
3017 "output).");
3018 }
3019 }
3020
3021 // Set domain and range Map. This may clear the Import / Export
3022 // objects if the new Maps differ from any old ones.
3023 setDomainRangeMaps(domainMap, rangeMap);
3024
3025 // If the graph does not already have a column Map (either from
3026 // the user constructor calling the version of the constructor
3027 // that takes a column Map, or from a previous fillComplete call),
3028 // then create it.
3029 Teuchos::Array<int> remotePIDs(0);
3030 const bool mustBuildColMap = !this->hasColMap();
3031 if (mustBuildColMap) {
3032 this->makeColMap(remotePIDs); // resized on output
3033 }
3034
3035 // Make indices local, if they aren't already.
3036 // The method doesn't do any work if the indices are already local.
3037 const std::pair<size_t, std::string> makeIndicesLocalResult =
3038 this->makeIndicesLocal(verbose);
3039
3040 if (debug_) {
3041 using Details::gathervPrint;
3042 using Teuchos::outArg;
3043 using Teuchos::RCP;
3044 using Teuchos::REDUCE_MIN;
3045 using Teuchos::reduceAll;
3046
3047 RCP<const map_type> map = this->getMap();
3048 RCP<const Teuchos::Comm<int>> comm;
3049 if (!map.is_null()) {
3050 comm = map->getComm();
3051 }
3052 if (comm.is_null()) {
3053 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(makeIndicesLocalResult.first != 0, std::runtime_error,
3054 makeIndicesLocalResult.second);
3055 } else {
3056 const int lclSuccess = (makeIndicesLocalResult.first == 0);
3057 int gblSuccess = 0; // output argument
3058 reduceAll(*comm, REDUCE_MIN, lclSuccess, outArg(gblSuccess));
3059 if (gblSuccess != 1) {
3060 std::ostringstream os;
3061 gathervPrint(os, makeIndicesLocalResult.second, *comm);
3062 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
3063 }
3064 }
3065 } else {
3066 // TODO (mfh 20 Jul 2017) Instead of throwing here, pass along
3067 // the error state to makeImportExport or
3068 // computeGlobalConstants, which may do all-reduces and thus may
3069 // have the opportunity to communicate that error state.
3070 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(makeIndicesLocalResult.first != 0, std::runtime_error,
3071 makeIndicesLocalResult.second);
3072 }
3073
3074 // If this process has no indices, then CrsGraph considers it
3075 // already trivially sorted and merged. Thus, this method need
3076 // not be called on all processes in the row Map's communicator.
3077 this->sortAndMergeAllIndices(this->isSorted(), this->isMerged());
3078
3079 // Make Import and Export objects, if they haven't been made
3080 // already. If we made a column Map above, reuse information from
3081 // that process to avoid communiation in the Import setup.
3082 this->makeImportExport(remotePIDs, mustBuildColMap);
3083
3084 // Create the KokkosSparse::StaticCrsGraph, if it doesn't already exist.
3085 this->fillLocalGraph(params);
3086
3087 const bool callComputeGlobalConstants = params.get() == nullptr ||
3088 params->get("compute global constants", true);
3089 if (callComputeGlobalConstants) {
3090 this->computeGlobalConstants();
3091 } else {
3092 this->computeLocalConstants();
3093 }
3094 this->fillComplete_ = true;
3095 this->checkInternalState();
3096
3097 if (verbose) {
3098 std::ostringstream os;
3099 os << *prefix << "Done" << endl;
3100 std::cerr << os.str();
3101 }
3102}
3103
3104template <class LocalOrdinal, class GlobalOrdinal, class Node>
3105void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3106 expertStaticFillComplete(const Teuchos::RCP<const map_type>& domainMap,
3107 const Teuchos::RCP<const map_type>& rangeMap,
3108 const Teuchos::RCP<const import_type>& importer,
3109 const Teuchos::RCP<const export_type>& exporter,
3110 const Teuchos::RCP<Teuchos::ParameterList>& params) {
3111 const char tfecfFuncName[] = "expertStaticFillComplete: ";
3112#ifdef HAVE_TPETRA_MMM_TIMINGS
3113 std::string label;
3114 if (!params.is_null())
3115 label = params->get("Timer Label", label);
3116 std::string prefix = std::string("Tpetra ") + label + std::string(": ");
3117 using Teuchos::TimeMonitor;
3118 Teuchos::RCP<Teuchos::TimeMonitor> MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-Setup"))));
3119#endif
3120
3121 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3122 domainMap.is_null() || rangeMap.is_null(),
3123 std::runtime_error, "The input domain Map and range Map must be nonnull.");
3124 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3125 isFillComplete() || !hasColMap(), std::runtime_error,
3126 "You may not "
3127 "call this method unless the graph has a column Map.");
3128 auto rowPtrsUnpackedLength = this->getRowPtrsUnpackedDevice().extent(0);
3129 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3130 getLocalNumRows() > 0 && rowPtrsUnpackedLength == 0,
3131 std::runtime_error, "The calling process has getLocalNumRows() = " << getLocalNumRows() << " > 0 rows, but the row offsets array has not "
3132 "been set.");
3133 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3134 static_cast<size_t>(rowPtrsUnpackedLength) != getLocalNumRows() + 1,
3135 std::runtime_error, "The row offsets array has length " << rowPtrsUnpackedLength << " != getLocalNumRows()+1 = " << (getLocalNumRows() + 1) << ".");
3136
3137 // Note: We don't need to do the following things which are normally done in fillComplete:
3138 // allocateIndices, globalAssemble, makeColMap, makeIndicesLocal, sortAndMergeAllIndices
3139
3140 // Constants from allocateIndices
3141 //
3142 // mfh 08 Aug 2014: numAllocForAllRows_ and k_numAllocPerRow_ go
3143 // away once the graph is allocated. expertStaticFillComplete
3144 // either presumes that the graph is allocated, or "allocates" it.
3145 //
3146 // FIXME (mfh 08 Aug 2014) The goal for the Kokkos refactor
3147 // version of CrsGraph is to allocate in the constructor, not
3148 // lazily on first insert. That will make both
3149 // numAllocForAllRows_ and k_numAllocPerRow_ obsolete.
3150 numAllocForAllRows_ = 0;
3151 k_numAllocPerRow_ = decltype(k_numAllocPerRow_)();
3152 indicesAreAllocated_ = true;
3153
3154 // Constants from makeIndicesLocal
3155 //
3156 // The graph has a column Map, so its indices had better be local.
3157 indicesAreLocal_ = true;
3158 indicesAreGlobal_ = false;
3159
3160 // set domain/range map: may clear the import/export objects
3161#ifdef HAVE_TPETRA_MMM_TIMINGS
3162 MM = Teuchos::null;
3163 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-Maps"))));
3164#endif
3165 setDomainRangeMaps(domainMap, rangeMap);
3166
3167 // Presume the user sorted and merged the arrays first
3168 indicesAreSorted_ = true;
3169 noRedundancies_ = true;
3170
3171 // makeImportExport won't create a new importer/exporter if I set one here first.
3172#ifdef HAVE_TPETRA_MMM_TIMINGS
3173 MM = Teuchos::null;
3174 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-mIXcheckI"))));
3175#endif
3176
3177 importer_ = Teuchos::null;
3178 exporter_ = Teuchos::null;
3179 if (importer != Teuchos::null) {
3180 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3181 !importer->getSourceMap()->isSameAs(*getDomainMap()) ||
3182 !importer->getTargetMap()->isSameAs(*getColMap()),
3183 std::invalid_argument, ": importer does not match matrix maps.");
3184 importer_ = importer;
3185 }
3186
3187#ifdef HAVE_TPETRA_MMM_TIMINGS
3188 MM = Teuchos::null;
3189 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-mIXcheckE"))));
3190#endif
3191
3192 if (exporter != Teuchos::null) {
3193 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3194 !exporter->getSourceMap()->isSameAs(*getRowMap()) ||
3195 !exporter->getTargetMap()->isSameAs(*getRangeMap()),
3196 std::invalid_argument, ": exporter does not match matrix maps.");
3197 exporter_ = exporter;
3198 }
3199
3200#ifdef HAVE_TPETRA_MMM_TIMINGS
3201 MM = Teuchos::null;
3202 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-mIXmake"))));
3203#endif
3204 Teuchos::Array<int> remotePIDs(0); // unused output argument
3205 this->makeImportExport(remotePIDs, false);
3206
3207#ifdef HAVE_TPETRA_MMM_TIMINGS
3208 MM = Teuchos::null;
3209 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-fLG"))));
3210#endif
3211 this->fillLocalGraph(params);
3212
3213 const bool callComputeGlobalConstants = params.get() == nullptr ||
3214 params->get("compute global constants", true);
3215
3216 if (callComputeGlobalConstants) {
3217#ifdef HAVE_TPETRA_MMM_TIMINGS
3218 MM = Teuchos::null;
3219 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-cGC (const)"))));
3220#endif // HAVE_TPETRA_MMM_TIMINGS
3221 this->computeGlobalConstants();
3222 } else {
3223#ifdef HAVE_TPETRA_MMM_TIMINGS
3224 MM = Teuchos::null;
3225 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-cGC (noconst)"))));
3226#endif // HAVE_TPETRA_MMM_TIMINGS
3227 this->computeLocalConstants();
3228 }
3229
3230 fillComplete_ = true;
3231
3232#ifdef HAVE_TPETRA_MMM_TIMINGS
3233 MM = Teuchos::null;
3234 MM = Teuchos::rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-G-cIS"))));
3235#endif
3236 checkInternalState();
3237}
3238
3239template <class LocalOrdinal, class GlobalOrdinal, class Node>
3240void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3241 fillLocalGraph(const Teuchos::RCP<Teuchos::ParameterList>& params) {
3242 using ::Tpetra::Details::computeOffsetsFromCounts;
3243 typedef typename local_graph_device_type::row_map_type row_map_type;
3244 typedef typename row_map_type::non_const_type non_const_row_map_type;
3245 typedef typename local_graph_device_type::entries_type::non_const_type lclinds_1d_type;
3246 const char tfecfFuncName[] =
3247 "fillLocalGraph (called from fillComplete or "
3248 "expertStaticFillComplete): ";
3249 const size_t lclNumRows = this->getLocalNumRows();
3250
3251 Details::ProfilingRegion regionFLG("Tpetra::CrsGraph::fillLocalGraph");
3252
3253 // This method's goal is to fill in the two arrays (compressed
3254 // sparse row format) that define the sparse graph's structure.
3255
3256 bool requestOptimizedStorage = true;
3257 if (!params.is_null() && !params->get("Optimize Storage", true)) {
3258 requestOptimizedStorage = false;
3259 }
3260
3261 // The graph's column indices are currently stored in a 1-D
3262 // format, with row offsets in rowPtrsUnpacked_host_ and local column indices
3263 // in k_lclInds1D_.
3264
3265 if (debug_) {
3266 auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
3267 // The graph's array of row offsets must already be allocated.
3268 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked.extent(0) == 0, std::logic_error,
3269 "rowPtrsUnpacked_host_ has size zero, but shouldn't");
3270 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked.extent(0) != lclNumRows + 1, std::logic_error,
3271 "rowPtrsUnpacked_host_.extent(0) = "
3272 << rowPtrsUnpacked.extent(0) << " != (lclNumRows + 1) = "
3273 << (lclNumRows + 1) << ".");
3274 const size_t numOffsets = rowPtrsUnpacked.extent(0);
3275 const auto valToCheck = rowPtrsUnpacked(numOffsets - 1);
3276 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numOffsets != 0 &&
3277 lclIndsUnpacked_wdv.extent(0) != valToCheck,
3278 std::logic_error, "numOffsets=" << numOffsets << " != 0 "
3279 " and lclIndsUnpacked_wdv.extent(0)="
3280 << lclIndsUnpacked_wdv.extent(0) << " != rowPtrsUnpacked_host_(" << numOffsets << ")=" << valToCheck << ".");
3281 }
3282
3283 size_t allocSize = 0;
3284 try {
3285 allocSize = this->getLocalAllocationSize();
3286 } catch (std::logic_error& e) {
3287 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
3288 "getLocalAllocationSize threw "
3289 "std::logic_error: "
3290 << e.what());
3291 } catch (std::runtime_error& e) {
3292 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
3293 "getLocalAllocationSize threw "
3294 "std::runtime_error: "
3295 << e.what());
3296 } catch (std::exception& e) {
3297 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
3298 "getLocalAllocationSize threw "
3299 "std::exception: "
3300 << e.what());
3301 } catch (...) {
3302 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
3303 "getLocalAllocationSize threw "
3304 "an exception not a subclass of std::exception.");
3305 }
3306
3307 if (this->getLocalNumEntries() != allocSize) {
3308 // Use the nonconst version of row_map_type for ptr_d, because
3309 // the latter is const and we need to modify ptr_d here.
3310 non_const_row_map_type ptr_d;
3311 row_map_type ptr_d_const;
3312
3313 // The graph's current 1-D storage is "unpacked." This means
3314 // the row offsets may differ from what the final row offsets
3315 // should be. This could happen, for example, if the user set
3316 // an upper bound on the number of entries in each row, but
3317 // didn't fill all those entries.
3318
3319 if (debug_) {
3320 auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
3321 if (rowPtrsUnpacked.extent(0) != 0) {
3322 const size_t numOffsets =
3323 static_cast<size_t>(rowPtrsUnpacked.extent(0));
3324 const auto valToCheck = rowPtrsUnpacked(numOffsets - 1);
3325 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != size_t(lclIndsUnpacked_wdv.extent(0)),
3326 std::logic_error,
3327 "(Unpacked branch) Before allocating "
3328 "or packing, k_rowPtrs_("
3329 << (numOffsets - 1) << ")="
3330 << valToCheck << " != lclIndsUnpacked_wdv.extent(0)="
3331 << lclIndsUnpacked_wdv.extent(0) << ".");
3332 }
3333 }
3334
3335 // Pack the row offsets into ptr_d, by doing a sum-scan of the
3336 // array of valid entry counts per row (k_numRowEntries_).
3337
3338 // Total number of entries in the matrix on the calling
3339 // process. We will compute this in the loop below. It's
3340 // cheap to compute and useful as a sanity check.
3341 size_t lclTotalNumEntries = 0;
3342 {
3343 // Allocate the packed row offsets array.
3344 ptr_d =
3345 non_const_row_map_type("Tpetra::CrsGraph::ptr", lclNumRows + 1);
3346 ptr_d_const = ptr_d;
3347
3348 // It's ok that k_numRowEntries_ is a host View; the
3349 // function can handle this.
3350 typename num_row_entries_type::const_type numRowEnt_h = k_numRowEntries_;
3351 if (debug_) {
3352 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(size_t(numRowEnt_h.extent(0)) != lclNumRows,
3353 std::logic_error,
3354 "(Unpacked branch) "
3355 "numRowEnt_h.extent(0)="
3356 << numRowEnt_h.extent(0)
3357 << " != getLocalNumRows()=" << lclNumRows << "");
3358 }
3359
3360 lclTotalNumEntries = computeOffsetsFromCounts(ptr_d, numRowEnt_h);
3361
3362 if (debug_) {
3363 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(ptr_d.extent(0)) != lclNumRows + 1,
3364 std::logic_error,
3365 "(Unpacked branch) After allocating "
3366 "ptr_d, ptr_d.extent(0) = "
3367 << ptr_d.extent(0)
3368 << " != lclNumRows+1 = " << (lclNumRows + 1) << ".");
3369 const auto valToCheck =
3370 ::Tpetra::Details::getEntryOnHost(ptr_d, lclNumRows);
3371 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != lclTotalNumEntries, std::logic_error,
3372 "Tpetra::CrsGraph::fillLocalGraph: In unpacked branch, "
3373 "after filling ptr_d, ptr_d(lclNumRows="
3374 << lclNumRows
3375 << ") = " << valToCheck << " != total number of entries "
3376 "on the calling process = "
3377 << lclTotalNumEntries
3378 << ".");
3379 }
3380 }
3381
3382 // Allocate the array of packed column indices.
3383 lclinds_1d_type ind_d =
3384 lclinds_1d_type("Tpetra::CrsGraph::lclInd", lclTotalNumEntries);
3385
3386 // k_rowPtrs_ and lclIndsUnpacked_wdv are currently unpacked. Pack
3387 // them, using the packed row offsets array ptr_d that we
3388 // created above.
3389 //
3390 // FIXME (mfh 08 Aug 2014) If "Optimize Storage" is false (in
3391 // CrsMatrix?), we need to keep around the unpacked row
3392 // offsets and column indices.
3393
3394 // Pack the column indices from unpacked lclIndsUnpacked_wdv into
3395 // packed ind_d. We will replace lclIndsUnpacked_wdv below.
3396 typedef pack_functor<
3397 typename local_graph_device_type::entries_type::non_const_type,
3398 typename local_inds_dualv_type::t_dev::const_type,
3399 row_map_type,
3400 typename local_graph_device_type::row_map_type>
3401 inds_packer_type;
3402 inds_packer_type f(ind_d,
3403 lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly),
3404 ptr_d, this->getRowPtrsUnpackedDevice());
3405 {
3406 typedef typename decltype(ind_d)::execution_space exec_space;
3407 typedef Kokkos::RangePolicy<exec_space, LocalOrdinal> range_type;
3408 Kokkos::parallel_for(range_type(0, lclNumRows), f);
3409 }
3410
3411 if (debug_) {
3412 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(ptr_d.extent(0) == 0, std::logic_error,
3413 "(\"Optimize Storage\"=true branch) After packing, "
3414 "ptr_d.extent(0)=0.");
3415 if (ptr_d.extent(0) != 0) {
3416 const size_t numOffsets = static_cast<size_t>(ptr_d.extent(0));
3417 const auto valToCheck =
3418 ::Tpetra::Details::getEntryOnHost(ptr_d, numOffsets - 1);
3419 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(valToCheck) != ind_d.extent(0),
3420 std::logic_error,
3421 "(\"Optimize Storage\"=true branch) "
3422 "After packing, ptr_d("
3423 << (numOffsets - 1) << ")="
3424 << valToCheck << " != ind_d.extent(0)="
3425 << ind_d.extent(0) << ".");
3426 }
3427 }
3428 // Build the local graph.
3429 if (requestOptimizedStorage)
3430 setRowPtrs(ptr_d_const);
3431 else
3432 setRowPtrsPacked(ptr_d_const);
3433 lclIndsPacked_wdv = local_inds_wdv_type(ind_d);
3434 } else { // We don't have to pack, so just set the pointers.
3435 // Set both packed and unpacked rowptrs to this
3436 this->setRowPtrs(rowPtrsUnpacked_dev_);
3437 lclIndsPacked_wdv = lclIndsUnpacked_wdv;
3438
3439 if (debug_) {
3440 auto rowPtrsPacked_dev = this->getRowPtrsPackedDevice();
3441 auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
3442 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsPacked_dev.extent(0) == 0, std::logic_error,
3443 "(\"Optimize Storage\"=false branch) "
3444 "rowPtrsPacked_dev_.extent(0) = 0.");
3445 if (rowPtrsPacked_dev.extent(0) != 0) {
3446 const size_t numOffsets =
3447 static_cast<size_t>(rowPtrsPacked_dev.extent(0));
3448 const size_t valToCheck =
3449 rowPtrsPacked_host(numOffsets - 1);
3450 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != size_t(lclIndsPacked_wdv.extent(0)),
3451 std::logic_error,
3452 "(\"Optimize Storage\"=false branch) "
3453 "rowPtrsPacked_dev_("
3454 << (numOffsets - 1) << ")="
3455 << valToCheck
3456 << " != lclIndsPacked_wdv.extent(0)="
3457 << lclIndsPacked_wdv.extent(0) << ".");
3458 }
3459 }
3460 }
3461
3462 if (debug_) {
3463 auto rowPtrsPacked_dev = this->getRowPtrsPackedDevice();
3464 auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
3465 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(rowPtrsPacked_dev.extent(0)) != lclNumRows + 1,
3466 std::logic_error, "After packing, rowPtrsPacked_dev_.extent(0) = " << rowPtrsPacked_dev.extent(0) << " != lclNumRows+1 = " << (lclNumRows + 1) << ".");
3467 if (rowPtrsPacked_dev.extent(0) != 0) {
3468 const size_t numOffsets = static_cast<size_t>(rowPtrsPacked_dev.extent(0));
3469 const auto valToCheck = rowPtrsPacked_host(numOffsets - 1);
3470 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(valToCheck) != lclIndsPacked_wdv.extent(0),
3471 std::logic_error, "After packing, rowPtrsPacked_dev_(" << (numOffsets - 1) << ") = " << valToCheck << " != lclIndsPacked_wdv.extent(0) = " << lclIndsPacked_wdv.extent(0) << ".");
3472 }
3473 }
3474
3475 if (requestOptimizedStorage) {
3476 // With optimized storage, we don't need to store
3477 // the array of row entry counts.
3478
3479 // Free graph data structures that are only needed for
3480 // unpacked 1-D storage.
3481 k_numRowEntries_ = num_row_entries_type();
3482
3483 // Keep the new 1-D packed allocations.
3484 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
3485
3486 storageStatus_ = Details::STORAGE_1D_PACKED;
3487 }
3488
3489 set_need_sync_host_uvm_access(); // make sure kernel setup of indices is fenced before a host access
3490}
3491
3492template <class LocalOrdinal, class GlobalOrdinal, class Node>
3493void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3494 replaceColMap(const Teuchos::RCP<const map_type>& newColMap) {
3495 // NOTE: This safety check matches the code, but not the documentation of Crsgraph
3496 //
3497 // FIXME (mfh 18 Aug 2014) This will break if the calling process
3498 // has no entries, because in that case, currently it is neither
3499 // locally nor globally indexed. This will change once we get rid
3500 // of lazy allocation (so that the constructor allocates indices
3501 // and therefore commits to local vs. global).
3502 const char tfecfFuncName[] = "replaceColMap: ";
3503 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3504 isLocallyIndexed() || isGloballyIndexed(), std::runtime_error,
3505 "Requires matching maps and non-static graph.");
3506 colMap_ = newColMap;
3507}
3508
3509template <class LocalOrdinal, class GlobalOrdinal, class Node>
3510void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3511 reindexColumns(const Teuchos::RCP<const map_type>& newColMap,
3512 const Teuchos::RCP<const import_type>& newImport,
3513 const bool sortIndicesInEachRow) {
3514 using Teuchos::RCP;
3515 using Teuchos::REDUCE_MIN;
3516 using Teuchos::reduceAll;
3517 typedef GlobalOrdinal GO;
3518 typedef LocalOrdinal LO;
3519 using col_inds_type_dev = typename local_inds_dualv_type::t_dev;
3520 const char tfecfFuncName[] = "reindexColumns: ";
3521
3522 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3523 isFillComplete(), std::runtime_error,
3524 "The graph is fill complete "
3525 "(isFillComplete() returns true). You must call resumeFill() before "
3526 "you may call this method.");
3527
3528 // mfh 19 Aug 2014: This method does NOT redistribute data; it
3529 // doesn't claim to do the work of an Import or Export. This
3530 // means that for all processes, the calling process MUST own all
3531 // column indices, in both the old column Map (if it exists) and
3532 // the new column Map. We check this via an all-reduce.
3533 //
3534 // Some processes may be globally indexed, others may be locally
3535 // indexed, and others (that have no graph entries) may be
3536 // neither. This method will NOT change the graph's current
3537 // state. If it's locally indexed, it will stay that way, and
3538 // vice versa. It would easy to add an option to convert indices
3539 // from global to local, so as to save a global-to-local
3540 // conversion pass. However, we don't do this here. The intended
3541 // typical use case is that the graph already has a column Map and
3542 // is locally indexed, and this is the case for which we optimize.
3543
3544 const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
3545
3546 // Attempt to convert indices to the new column Map's version of
3547 // local. This will fail if on the calling process, the graph has
3548 // indices that are not on that process in the new column Map.
3549 // After the local conversion attempt, we will do an all-reduce to
3550 // see if any processes failed.
3551
3552 // If this is false, then either the graph contains a column index
3553 // which is invalid in the CURRENT column Map, or the graph is
3554 // locally indexed but currently has no column Map. In either
3555 // case, there is no way to convert the current local indices into
3556 // global indices, so that we can convert them into the new column
3557 // Map's local indices. It's possible for this to be true on some
3558 // processes but not others, due to replaceColMap.
3559 bool allCurColIndsValid = true;
3560 // On the calling process, are all valid current column indices
3561 // also in the new column Map on the calling process? In other
3562 // words, does local reindexing suffice, or should the user have
3563 // done an Import or Export instead?
3564 bool localSuffices = true;
3565
3566 {
3567 // Final arrays for the local indices. We will allocate exactly
3568 // one of these ONLY if the graph is locally indexed on the
3569 // calling process, and ONLY if the graph has one or more entries
3570 // (is not empty) on the calling process. In that case, we
3571 // allocate the first (1-D storage) if the graph has a static
3572 // profile, else we allocate the second (2-D storage).
3573 col_inds_type_dev newLclInds1D_dev;
3574
3575 // If indices aren't allocated, that means the calling process
3576 // owns no entries in the graph. Thus, there is nothing to
3577 // convert, and it trivially succeeds locally.
3578 if (indicesAreAllocated()) {
3579 if (isLocallyIndexed()) {
3580 if (hasColMap()) { // locally indexed, and currently has a column Map
3581 const map_type& oldColMap = *(getColMap());
3582
3583 // Allocate storage for the new local indices.
3584 const size_t allocSize = this->getLocalAllocationSize();
3585 auto oldLclInds1D = lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly);
3586 newLclInds1D_dev = col_inds_type_dev("Tpetra::CrsGraph::lclIndsReindexed",
3587 allocSize);
3588 auto oldLclColMap = oldColMap.getLocalMap();
3589 auto newLclColMap = newColMap->getLocalMap();
3590
3591 const auto LO_INVALID = Teuchos::OrdinalTraits<LO>::invalid();
3592 const auto GO_INVALID = Teuchos::OrdinalTraits<GO>::invalid();
3593
3594 const int NOT_ALL_LOCAL_INDICES_ARE_VALID = 1;
3595 const int LOCAL_DOES_NOT_SUFFICE = 2;
3596 int errorStatus = 0;
3597 Kokkos::parallel_reduce(
3598 "Tpetra::CrsGraph::reindexColumns",
3599 Kokkos::RangePolicy<LocalOrdinal, execution_space>(0, allocSize),
3600 KOKKOS_LAMBDA(const LocalOrdinal k, int& result) {
3601 const LocalOrdinal oldLclCol = oldLclInds1D(k);
3602 if (oldLclCol == LO_INVALID) {
3603 result &= NOT_ALL_LOCAL_INDICES_ARE_VALID;
3604 } else {
3605 const GO gblCol = oldLclColMap.getGlobalElement(oldLclCol);
3606 if (gblCol == GO_INVALID) {
3607 result &= LOCAL_DOES_NOT_SUFFICE;
3608 } else {
3609 const LocalOrdinal newLclCol = newLclColMap.getLocalElement(gblCol);
3610 if (newLclCol == LO_INVALID) {
3611 result &= NOT_ALL_LOCAL_INDICES_ARE_VALID;
3612 } else {
3613 newLclInds1D_dev(k) = newLclCol;
3614 }
3615 }
3616 }
3617 },
3618 Kokkos::LOr<int>(errorStatus));
3619 allCurColIndsValid = !(errorStatus & NOT_ALL_LOCAL_INDICES_ARE_VALID);
3620 localSuffices = !(errorStatus & LOCAL_DOES_NOT_SUFFICE);
3621 } else { // locally indexed, but no column Map
3622 // This case is only possible if replaceColMap() was called
3623 // with a null argument on the calling process. It's
3624 // possible, but it means that this method can't possibly
3625 // succeed, since we have no way of knowing how to convert
3626 // the current local indices to global indices.
3627 allCurColIndsValid = false;
3628 }
3629 } else { // globally indexed
3630 // If the graph is globally indexed, we don't need to save
3631 // local indices, but we _do_ need to know whether the current
3632 // global indices are valid in the new column Map. We may
3633 // need to do a getRemoteIndexList call to find this out.
3634 //
3635 // In this case, it doesn't matter whether the graph currently
3636 // has a column Map. We don't need the old column Map to
3637 // convert from global indices to the _new_ column Map's local
3638 // indices. Furthermore, we can use the same code, whether
3639 // the graph is static or dynamic profile.
3640
3641 // Test whether the current global indices are in the new
3642 // column Map on the calling process.
3643 for (LO lclRow = 0; lclRow < lclNumRows; ++lclRow) {
3644 const RowInfo rowInfo = this->getRowInfo(lclRow);
3645 auto oldGblRowView = this->getGlobalIndsViewHost(rowInfo);
3646 for (size_t k = 0; k < rowInfo.numEntries; ++k) {
3647 const GO gblCol = oldGblRowView(k);
3648 if (!newColMap->isNodeGlobalElement(gblCol)) {
3649 localSuffices = false;
3650 break; // Stop at the first invalid index
3651 }
3652 } // for each entry in the current row
3653 } // for each locally owned row
3654 } // locally or globally indexed
3655 } // whether indices are allocated
3656
3657 // Do an all-reduce to check both possible error conditions.
3658 int lclSuccess[2];
3659 lclSuccess[0] = allCurColIndsValid ? 1 : 0;
3660 lclSuccess[1] = localSuffices ? 1 : 0;
3661 int gblSuccess[2];
3662 gblSuccess[0] = 0;
3663 gblSuccess[1] = 0;
3664 RCP<const Teuchos::Comm<int>> comm =
3665 getRowMap().is_null() ? Teuchos::null : getRowMap()->getComm();
3666 if (!comm.is_null()) {
3667 reduceAll<int, int>(*comm, REDUCE_MIN, 2, lclSuccess, gblSuccess);
3668 }
3669
3670 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3671 gblSuccess[0] == 0, std::runtime_error,
3672 "It is not possible to continue."
3673 " The most likely reason is that the graph is locally indexed, but the "
3674 "column Map is missing (null) on some processes, due to a previous call "
3675 "to replaceColMap().");
3676
3677 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3678 gblSuccess[1] == 0, std::runtime_error,
3679 "On some process, the graph "
3680 "contains column indices that are in the old column Map, but not in the "
3681 "new column Map (on that process). This method does NOT redistribute "
3682 "data; it does not claim to do the work of an Import or Export operation."
3683 " This means that for all processess, the calling process MUST own all "
3684 "column indices, in both the old column Map and the new column Map. In "
3685 "this case, you will need to do an Import or Export operation to "
3686 "redistribute data.");
3687
3688 // Commit the results.
3689 if (isLocallyIndexed()) {
3690 lclIndsUnpacked_wdv = local_inds_wdv_type(newLclInds1D_dev);
3691 }
3692 // end of scope for newLclInds1D_dev
3693 // sortAndMergeAllIndices needs host access
3694 }
3695
3696 if (isLocallyIndexed()) {
3697 // We've reindexed, so we don't know if the indices are sorted.
3698 //
3699 // FIXME (mfh 17 Sep 2014) It could make sense to check this,
3700 // since we're already going through all the indices above. We
3701 // could also sort each row in place; that way, we would only
3702 // have to make one pass over the rows.
3703 indicesAreSorted_ = false;
3704 if (sortIndicesInEachRow) {
3705 // NOTE (mfh 17 Sep 2014) The graph must be locally indexed in
3706 // order to call this method.
3707 //
3708 // FIXME (mfh 17 Sep 2014) This violates the strong exception
3709 // guarantee. It would be better to sort the new index arrays
3710 // before committing them.
3711 const bool sorted = false; // need to resort
3712 const bool merged = true; // no need to merge, since no dups
3713 this->sortAndMergeAllIndices(sorted, merged);
3714 }
3715 }
3716 colMap_ = newColMap;
3717
3718 if (newImport.is_null()) {
3719 // FIXME (mfh 19 Aug 2014) Should use the above all-reduce to
3720 // check whether the input Import is null on any process.
3721 //
3722 // If the domain Map hasn't been set yet, we can't compute a new
3723 // Import object. Leave it what it is; it should be null, but
3724 // it doesn't matter. If the domain Map _has_ been set, then
3725 // compute a new Import object if necessary.
3726 if (!domainMap_.is_null()) {
3727 if (!domainMap_->isSameAs(*newColMap)) {
3728 importer_ = Teuchos::rcp(new import_type(domainMap_, newColMap));
3729 } else {
3730 importer_ = Teuchos::null; // don't need an Import
3731 }
3732 }
3733 } else {
3734 // The caller gave us an Import object. Assume that it's valid.
3735 importer_ = newImport;
3736 }
3737}
3738
3739template <class LocalOrdinal, class GlobalOrdinal, class Node>
3740void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3741 replaceDomainMap(const Teuchos::RCP<const map_type>& newDomainMap) {
3742 const char prefix[] = "Tpetra::CrsGraph::replaceDomainMap: ";
3743 TEUCHOS_TEST_FOR_EXCEPTION(
3744 colMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3745 "this method unless the graph already has a column Map.");
3746 TEUCHOS_TEST_FOR_EXCEPTION(
3747 newDomainMap.is_null(), std::invalid_argument,
3748 prefix << "The new domain Map must be nonnull.");
3749
3750 // Create a new importer, if needed
3751 Teuchos::RCP<const import_type> newImporter = Teuchos::null;
3752 if (newDomainMap != colMap_ && (!newDomainMap->isSameAs(*colMap_))) {
3753 newImporter = rcp(new import_type(newDomainMap, colMap_));
3754 }
3755 this->replaceDomainMapAndImporter(newDomainMap, newImporter);
3756}
3757
3758template <class LocalOrdinal, class GlobalOrdinal, class Node>
3759void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3760 replaceDomainMapAndImporter(const Teuchos::RCP<const map_type>& newDomainMap,
3761 const Teuchos::RCP<const import_type>& newImporter) {
3762 const char prefix[] = "Tpetra::CrsGraph::replaceDomainMapAndImporter: ";
3763 TEUCHOS_TEST_FOR_EXCEPTION(
3764 colMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3765 "this method unless the graph already has a column Map.");
3766 TEUCHOS_TEST_FOR_EXCEPTION(
3767 newDomainMap.is_null(), std::invalid_argument,
3768 prefix << "The new domain Map must be nonnull.");
3769
3770 if (debug_) {
3771 if (newImporter.is_null()) {
3772 // It's not a good idea to put expensive operations in a macro
3773 // clause, even if they are side effect - free, because macros
3774 // don't promise that they won't evaluate their arguments more
3775 // than once. It's polite for them to do so, but not required.
3776 const bool colSameAsDom = colMap_->isSameAs(*newDomainMap);
3777 TEUCHOS_TEST_FOR_EXCEPTION(!colSameAsDom, std::invalid_argument,
3778 "If the new Import is null, "
3779 "then the new domain Map must be the same as the current column Map.");
3780 } else {
3781 const bool colSameAsTgt =
3782 colMap_->isSameAs(*(newImporter->getTargetMap()));
3783 const bool newDomSameAsSrc =
3784 newDomainMap->isSameAs(*(newImporter->getSourceMap()));
3785 TEUCHOS_TEST_FOR_EXCEPTION(!colSameAsTgt || !newDomSameAsSrc, std::invalid_argument,
3786 "If the "
3787 "new Import is nonnull, then the current column Map must be the same "
3788 "as the new Import's target Map, and the new domain Map must be the "
3789 "same as the new Import's source Map.");
3790 }
3791 }
3792
3793 domainMap_ = newDomainMap;
3794 importer_ = Teuchos::rcp_const_cast<import_type>(newImporter);
3795}
3796
3797template <class LocalOrdinal, class GlobalOrdinal, class Node>
3798void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3799 replaceRangeMap(const Teuchos::RCP<const map_type>& newRangeMap) {
3800 const char prefix[] = "Tpetra::CrsGraph::replaceRangeMap: ";
3801 TEUCHOS_TEST_FOR_EXCEPTION(
3802 rowMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3803 "this method unless the graph already has a row Map.");
3804 TEUCHOS_TEST_FOR_EXCEPTION(
3805 newRangeMap.is_null(), std::invalid_argument,
3806 prefix << "The new range Map must be nonnull.");
3807
3808 // Create a new exporter, if needed
3809 Teuchos::RCP<const export_type> newExporter = Teuchos::null;
3810 if (newRangeMap != rowMap_ && (!newRangeMap->isSameAs(*rowMap_))) {
3811 newExporter = rcp(new export_type(rowMap_, newRangeMap));
3812 }
3813 this->replaceRangeMapAndExporter(newRangeMap, newExporter);
3814}
3815
3816template <class LocalOrdinal, class GlobalOrdinal, class Node>
3817void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3818 replaceRangeMapAndExporter(const Teuchos::RCP<const map_type>& newRangeMap,
3819 const Teuchos::RCP<const export_type>& newExporter) {
3820 const char prefix[] = "Tpetra::CrsGraph::replaceRangeMapAndExporter: ";
3821 TEUCHOS_TEST_FOR_EXCEPTION(
3822 rowMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3823 "this method unless the graph already has a column Map.");
3824 TEUCHOS_TEST_FOR_EXCEPTION(
3825 newRangeMap.is_null(), std::invalid_argument,
3826 prefix << "The new domain Map must be nonnull.");
3827
3828 if (debug_) {
3829 if (newExporter.is_null()) {
3830 // It's not a good idea to put expensive operations in a macro
3831 // clause, even if they are side effect - free, because macros
3832 // don't promise that they won't evaluate their arguments more
3833 // than once. It's polite for them to do so, but not required.
3834 const bool rowSameAsRange = rowMap_->isSameAs(*newRangeMap);
3835 TEUCHOS_TEST_FOR_EXCEPTION(!rowSameAsRange, std::invalid_argument,
3836 "If the new Export is null, "
3837 "then the new range Map must be the same as the current row Map.");
3838 } else {
3839 const bool newRangeSameAsTgt =
3840 newRangeMap->isSameAs(*(newExporter->getTargetMap()));
3841 const bool rowSameAsSrc =
3842 rowMap_->isSameAs(*(newExporter->getSourceMap()));
3843 TEUCHOS_TEST_FOR_EXCEPTION(!rowSameAsSrc || !newRangeSameAsTgt, std::invalid_argument,
3844 "If the "
3845 "new Export is nonnull, then the current row Map must be the same "
3846 "as the new Export's source Map, and the new range Map must be the "
3847 "same as the new Export's target Map.");
3848 }
3849 }
3850
3851 rangeMap_ = newRangeMap;
3852 exporter_ = Teuchos::rcp_const_cast<export_type>(newExporter);
3853}
3854
3855template <class LocalOrdinal, class GlobalOrdinal, class Node>
3856typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_graph_device_type
3857CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3858 getLocalGraphDevice() const {
3859 return local_graph_device_type(
3860 lclIndsPacked_wdv.getDeviceView(Access::ReadWrite),
3861 this->getRowPtrsPackedDevice());
3862}
3863
3864template <class LocalOrdinal, class GlobalOrdinal, class Node>
3865typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_graph_host_type
3866CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3867 getLocalGraphHost() const {
3868 return local_graph_host_type(
3869 lclIndsPacked_wdv.getHostView(Access::ReadWrite),
3870 this->getRowPtrsPackedHost());
3871}
3872
3873template <class LocalOrdinal, class GlobalOrdinal, class Node>
3874void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3875 computeGlobalConstants() {
3876 using Teuchos::ArrayView;
3877 using Teuchos::outArg;
3878 using Teuchos::reduceAll;
3879 using ::Tpetra::Details::ProfilingRegion;
3880 typedef global_size_t GST;
3881
3882 ProfilingRegion regionCGC("Tpetra::CrsGraph::computeGlobalConstants");
3883
3884 this->computeLocalConstants();
3885
3886 // Compute global constants from local constants. Processes that
3887 // already have local constants still participate in the
3888 // all-reduces, using their previously computed values.
3889 if (!this->haveGlobalConstants_) {
3890 const Teuchos::Comm<int>& comm = *(this->getComm());
3891 // Promote all the nodeNum* and nodeMaxNum* quantities from
3892 // size_t to global_size_t, when doing the all-reduces for
3893 // globalNum* / globalMaxNum* results.
3894 //
3895 // FIXME (mfh 07 May 2013) Unfortunately, we either have to do
3896 // this in two all-reduces (one for the sum and the other for
3897 // the max), or use a custom MPI_Op that combines the sum and
3898 // the max. The latter might even be slower than two
3899 // all-reduces on modern network hardware. It would also be a
3900 // good idea to use nonblocking all-reduces (MPI 3), so that we
3901 // don't have to wait around for the first one to finish before
3902 // starting the second one.
3903 GST lcl, gbl;
3904 lcl = static_cast<GST>(this->getLocalNumEntries());
3905
3906 reduceAll<int, GST>(comm, Teuchos::REDUCE_SUM, 1, &lcl, &gbl);
3907 this->globalNumEntries_ = gbl;
3908
3909 const GST lclMaxNumRowEnt = static_cast<GST>(this->nodeMaxNumRowEntries_);
3910 reduceAll<int, GST>(comm, Teuchos::REDUCE_MAX, lclMaxNumRowEnt,
3911 outArg(this->globalMaxNumRowEntries_));
3912 this->haveGlobalConstants_ = true;
3913 }
3914}
3915
3916template <class LocalOrdinal, class GlobalOrdinal, class Node>
3917void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3918 computeLocalConstants() {
3919 using ::Tpetra::Details::ProfilingRegion;
3920
3921 ProfilingRegion regionCLC("Tpetra::CrsGraph::computeLocalConstants");
3922 if (this->haveLocalConstants_) {
3923 return;
3924 }
3925
3926 // Reset local properties
3927 this->nodeMaxNumRowEntries_ =
3928 Teuchos::OrdinalTraits<size_t>::invalid();
3929
3930 using LO = local_ordinal_type;
3931
3932 auto ptr = this->getRowPtrsPackedDevice();
3933 const LO lclNumRows = ptr.extent(0) == 0 ? static_cast<LO>(0) : (static_cast<LO>(ptr.extent(0)) - static_cast<LO>(1));
3934
3935 const LO lclMaxNumRowEnt =
3936 ::Tpetra::Details::maxDifference("Tpetra::CrsGraph: nodeMaxNumRowEntries",
3937 ptr, lclNumRows);
3938 this->nodeMaxNumRowEntries_ = static_cast<size_t>(lclMaxNumRowEnt);
3939 this->haveLocalConstants_ = true;
3940}
3941
3942template <class LocalOrdinal, class GlobalOrdinal, class Node>
3943std::pair<size_t, std::string>
3944CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3945 makeIndicesLocal(const bool verbose) {
3946 using Details::ProfilingRegion;
3947 using std::endl;
3948 using Teuchos::arcp;
3949 using Teuchos::Array;
3950 typedef LocalOrdinal LO;
3951 typedef GlobalOrdinal GO;
3952 typedef device_type DT;
3953 typedef typename local_graph_device_type::row_map_type::non_const_value_type offset_type;
3954 typedef typename num_row_entries_type::non_const_value_type num_ent_type;
3955 const char tfecfFuncName[] = "makeIndicesLocal: ";
3956 ProfilingRegion regionMakeIndicesLocal("Tpetra::CrsGraph::makeIndicesLocal");
3957
3958 std::unique_ptr<std::string> prefix;
3959 if (verbose) {
3960 prefix = this->createPrefix("CrsGraph", "makeIndicesLocal");
3961 std::ostringstream os;
3962 os << *prefix << "lclNumRows: " << getLocalNumRows() << endl;
3963 std::cerr << os.str();
3964 }
3965
3966 // These are somewhat global properties, so it's safe to have
3967 // exception checks for them, rather than returning an error code.
3968 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::logic_error,
3969 "The graph does not have a "
3970 "column Map yet. This method should never be called in that case. "
3971 "Please report this bug to the Tpetra developers.");
3972 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->getColMap().is_null(), std::logic_error,
3973 "The graph claims "
3974 "that it has a column Map, because hasColMap() returns true. However, "
3975 "the result of getColMap() is null. This should never happen. Please "
3976 "report this bug to the Tpetra developers.");
3977
3978 // Return value 1: The number of column indices (counting
3979 // duplicates) that could not be converted to local indices,
3980 // because they were not in the column Map on the calling process.
3981 size_t lclNumErrs = 0;
3982 std::ostringstream errStrm; // for return value 2 (error string)
3983
3984 const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
3985 const map_type& colMap = *(this->getColMap());
3986
3987 if (this->isGloballyIndexed() && lclNumRows != 0) {
3988 // This is a host-accessible View.
3989 typename num_row_entries_type::const_type h_numRowEnt =
3990 this->k_numRowEntries_;
3991
3992 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
3993
3994 // Allocate space for local indices.
3995 if (rowPtrsUnpacked_host.extent(0) == 0) {
3996 errStrm << "Unpacked row pointers (rowPtrsUnpacked_dev_) has length 0. This should never "
3997 "happen here. Please report this bug to the Tpetra developers."
3998 << endl;
3999 // Need to return early.
4000 return std::make_pair(Tpetra::Details::OrdinalTraits<size_t>::invalid(),
4001 errStrm.str());
4002 }
4003 const auto numEnt = rowPtrsUnpacked_host(lclNumRows);
4004
4005 // mfh 17 Dec 2016: We don't need initial zero-fill of
4006 // lclIndsUnpacked_wdv, because we will fill it below anyway.
4007 // AllowPadding would only help for aligned access (e.g.,
4008 // for vectorization) if we also were to pad each row to the
4009 // same alignment, so we'll skip AllowPadding for now.
4010
4011 // using Kokkos::AllowPadding;
4012 using Kokkos::view_alloc;
4013 using Kokkos::WithoutInitializing;
4014
4015 // When giving the label as an argument to
4016 // Kokkos::view_alloc, the label must be a string and not a
4017 // char*, else the code won't compile. This is because
4018 // view_alloc also allows a raw pointer as its first
4019 // argument. See
4020 // https://github.com/kokkos/kokkos/issues/434. This is a
4021 // large allocation typically, so the overhead of creating
4022 // an std::string is minor.
4023 const std::string label("Tpetra::CrsGraph::lclInd");
4024 if (verbose) {
4025 std::ostringstream os;
4026 os << *prefix << "(Re)allocate lclInd_wdv: old="
4027 << lclIndsUnpacked_wdv.extent(0) << ", new=" << numEnt << endl;
4028 std::cerr << os.str();
4029 }
4030
4031 local_inds_dualv_type lclInds_dualv =
4032 local_inds_dualv_type(view_alloc(label, WithoutInitializing),
4033 numEnt);
4034 lclIndsUnpacked_wdv = local_inds_wdv_type(lclInds_dualv);
4035
4036 auto lclColMap = colMap.getLocalMap();
4037 // This is a "device mirror" of the host View h_numRowEnt.
4038 //
4039 // NOTE (mfh 27 Sep 2016) Currently, the right way to get a
4040 // Device instance is to use its default constructor. See the
4041 // following Kokkos issue:
4042 //
4043 // https://github.com/kokkos/kokkos/issues/442
4044 if (verbose) {
4045 std::ostringstream os;
4046 os << *prefix << "Allocate device mirror k_numRowEnt: "
4047 << h_numRowEnt.extent(0) << endl;
4048 std::cerr << os.str();
4049 }
4050 auto k_numRowEnt =
4051 Kokkos::create_mirror_view_and_copy(device_type(), h_numRowEnt);
4052
4053 using ::Tpetra::Details::convertColumnIndicesFromGlobalToLocal;
4054 lclNumErrs =
4055 convertColumnIndicesFromGlobalToLocal<LO, GO, DT, offset_type, num_ent_type>(
4056 lclIndsUnpacked_wdv.getDeviceView(Access::OverwriteAll),
4057 gblInds_wdv.getDeviceView(Access::ReadOnly),
4058 this->getRowPtrsUnpackedDevice(),
4059 lclColMap,
4060 k_numRowEnt);
4061 if (lclNumErrs != 0) {
4062 const int myRank = [this]() {
4063 auto map = this->getMap();
4064 if (map.is_null()) {
4065 return 0;
4066 } else {
4067 auto comm = map->getComm();
4068 return comm.is_null() ? 0 : comm->getRank();
4069 }
4070 }();
4071 const bool pluralNumErrs = (lclNumErrs != static_cast<size_t>(1));
4072 errStrm << "(Process " << myRank << ") When converting column "
4073 "indices from global to local, we encountered "
4074 << lclNumErrs
4075 << " ind" << (pluralNumErrs ? "ices" : "ex")
4076 << " that do" << (pluralNumErrs ? "es" : "")
4077 << " not live in the column Map on this process." << endl;
4078 }
4079
4080 // We've converted column indices from global to local, so we
4081 // can deallocate the global column indices (which we know are
4082 // in 1-D storage, because the graph has static profile).
4083 if (verbose) {
4084 std::ostringstream os;
4085 os << *prefix << "Free gblInds_wdv: "
4086 << gblInds_wdv.extent(0) << endl;
4087 std::cerr << os.str();
4088 }
4089 gblInds_wdv = global_inds_wdv_type();
4090 } // globallyIndexed() && lclNumRows > 0
4091
4092 this->indicesAreLocal_ = true;
4093 this->indicesAreGlobal_ = false;
4094 this->checkInternalState();
4095
4096 return std::make_pair(lclNumErrs, errStrm.str());
4097}
4098
4099template <class LocalOrdinal, class GlobalOrdinal, class Node>
4100void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4101 makeColMap(Teuchos::Array<int>& remotePIDs) {
4102 using Details::ProfilingRegion;
4103 using std::endl;
4104 const char tfecfFuncName[] = "makeColMap";
4105
4106 ProfilingRegion regionSortAndMerge("Tpetra::CrsGraph::makeColMap");
4107 std::unique_ptr<std::string> prefix;
4108 if (verbose_) {
4109 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4110 std::ostringstream os;
4111 os << *prefix << "Start" << endl;
4112 std::cerr << os.str();
4113 }
4114
4115 // this->colMap_ should be null at this point, but we accept the
4116 // future possibility that it might not be (esp. if we decide
4117 // later to support graph structure changes after first
4118 // fillComplete, which CrsGraph does not currently (as of 12 Feb
4119 // 2017) support).
4120 Teuchos::RCP<const map_type> colMap = this->colMap_;
4121 const bool sortEachProcsGids =
4122 this->sortGhostsAssociatedWithEachProcessor_;
4123
4124 // FIXME (mfh 12 Feb 2017) ::Tpetra::Details::makeColMap returns a
4125 // per-process error code. If an error does occur on a process,
4126 // ::Tpetra::Details::makeColMap does NOT promise that all processes will
4127 // notice that error. This is the caller's responsibility. For
4128 // now, we only propagate (to all processes) and report the error
4129 // in debug mode. In the future, we need to add the local/global
4130 // error handling scheme used in BlockCrsMatrix to this class.
4131 if (debug_) {
4132 using Teuchos::outArg;
4133 using Teuchos::REDUCE_MIN;
4134 using Teuchos::reduceAll;
4135
4136 std::ostringstream errStrm;
4137 const int lclErrCode =
4138 Details::makeColMap(colMap, remotePIDs,
4139 getDomainMap(), *this, sortEachProcsGids, &errStrm);
4140 auto comm = this->getComm();
4141 if (!comm.is_null()) {
4142 const int lclSuccess = (lclErrCode == 0) ? 1 : 0;
4143 int gblSuccess = 0; // output argument
4144 reduceAll<int, int>(*comm, REDUCE_MIN, lclSuccess,
4145 outArg(gblSuccess));
4146 if (gblSuccess != 1) {
4147 std::ostringstream os;
4148 Details::gathervPrint(os, errStrm.str(), *comm);
4149 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
4150 ": An error happened on at "
4151 "least one process in the CrsGraph's communicator. "
4152 "Here are all processes' error messages:"
4153 << std::endl
4154 << os.str());
4155 }
4156 }
4157 } else {
4158 (void)Details::makeColMap(colMap, remotePIDs,
4159 getDomainMap(), *this, sortEachProcsGids, nullptr);
4160 }
4161 // See above. We want to admit the possibility of makeColMap
4162 // actually revising an existing column Map, even though that
4163 // doesn't currently (as of 10 May 2017) happen.
4164 this->colMap_ = colMap;
4165
4166 checkInternalState();
4167 if (verbose_) {
4168 std::ostringstream os;
4169 os << *prefix << "Done" << endl;
4170 std::cerr << os.str();
4171 }
4172}
4173
4174template <class execution_space, class LO, class rowptr_type, class colinds_type, class numRowEntries_type>
4175void prepareSortMergeUnpackedGraph(rowptr_type rowptr, colinds_type colinds, numRowEntries_type numRowEntries) {
4176 using ATS = KokkosKernels::ArithTraits<LO>;
4177 const auto unused = ATS::max();
4178
4179 auto numRows = rowptr.extent(0) - 1;
4180
4181 // make sure that unused entries will get ordered last
4182 Kokkos::parallel_for(
4183 "flag_unused_entries", Kokkos::RangePolicy<execution_space, LO>(0, numRows), KOKKOS_LAMBDA(const LO rlid) {
4184 for (size_t jj = rowptr(rlid) + numRowEntries(rlid); jj < rowptr(rlid + 1); ++jj) {
4185 colinds(jj) = unused;
4186 }
4187 });
4188}
4189
4190template <class execution_space, class LO, class rowptr_type, class colinds_type, class numRowEntries_type>
4191void mergeUnpackedGraph(rowptr_type rowptr, colinds_type colinds, numRowEntries_type numRowEntries) {
4192 // For this to work correctly, we require that the unsused column entries have been filled
4193 // with indices that get ordered last.
4194
4195 auto numRows = rowptr.extent(0) - 1;
4196
4197 // merge
4198 // We cannot use KokkosSparse::sort_and_merge_matrix since we
4199 // do not actually want to change the allocations.
4200
4201 Kokkos::parallel_for(
4202 "merge_entries", Kokkos::RangePolicy<execution_space>(0, numRows), KOKKOS_LAMBDA(const LO rlid) {
4203 auto rowNNZ = numRowEntries(rlid);
4204 if (rowNNZ == 0) {
4205 return;
4206 }
4207 auto rowBegin = rowptr(rlid);
4208 auto pos = rowBegin;
4209 for (size_t offset = rowBegin + 1; offset < rowBegin + rowNNZ; ++offset) {
4210 if ((colinds(offset) != colinds(pos))) {
4211 colinds(++pos) = colinds(offset);
4212 }
4213 }
4214 numRowEntries(rlid) = pos + 1 - rowBegin;
4215 });
4216}
4217
4218template <class LocalOrdinal, class GlobalOrdinal, class Node>
4219void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4220 sortAndMergeAllIndices(const bool sorted, const bool merged) {
4221 using std::endl;
4222 const char tfecfFuncName[] = "sortAndMergeAllIndices";
4223 Details::ProfilingRegion regionSortAndMerge("Tpetra::CrsGraph::sortAndMergeAllIndices");
4224
4225 std::unique_ptr<std::string> prefix;
4226 if (verbose_) {
4227 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4228 std::ostringstream os;
4229 os << *prefix << "Start: "
4230 << "sorted=" << (sorted ? "true" : "false")
4231 << ", merged=" << (merged ? "true" : "false") << endl;
4232 std::cerr << os.str();
4233 }
4234 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isGloballyIndexed(), std::logic_error,
4235 "This method may only be called after makeIndicesLocal.");
4236 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!merged && this->isStorageOptimized(), std::logic_error,
4237 "The graph is already storage optimized, so we shouldn't be "
4238 "merging any indices. "
4239 "Please report this bug to the Tpetra developers.");
4240
4241 if (!sorted || !merged) {
4242 if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
4243 // We are sorting & merging the unpacked views.
4244 // This means that not all entries are actually in use. We need to take k_numRowEntries_ into account.
4245 auto rowptr = rowPtrsUnpacked_dev_;
4246 auto colinds = lclIndsUnpacked_wdv.getDeviceView(Access::ReadWrite);
4247
4248 // Create a device copy of k_numRowEntries_.
4249 auto k_numRowEntries_d = Kokkos::create_mirror_view_and_copy(execution_space(), k_numRowEntries_);
4250
4251 // set set unused column entries so they get sorted last
4252 prepareSortMergeUnpackedGraph<execution_space, LocalOrdinal>(rowptr, colinds, k_numRowEntries_d);
4253
4254 if (!sorted) {
4255 KokkosSparse::sort_crs_graph(rowptr, colinds);
4256 this->indicesAreSorted_ = true; // we just sorted every row
4257 }
4258 if (!merged) {
4259 mergeUnpackedGraph<execution_space, LocalOrdinal>(rowptr, colinds, k_numRowEntries_d);
4260 Kokkos::deep_copy(k_numRowEntries_, k_numRowEntries_d);
4261 this->noRedundancies_ = true; // we just merged every row
4262 }
4263 } else {
4264 auto rowptr = rowPtrsPacked_dev_;
4265 auto colinds = lclIndsPacked_wdv.getDeviceView(Access::ReadWrite);
4266 if (!sorted && merged) {
4267 KokkosSparse::sort_crs_graph(rowptr, colinds);
4268 this->indicesAreSorted_ = true; // we just sorted every row
4269 } else {
4270 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
4271 "We should never get here."
4272 "Please report this bug to the Tpetra developers.");
4273 }
4274 }
4275 }
4276
4277 if (verbose_) {
4278 std::ostringstream os;
4279 os << *prefix << "Done" << endl;
4280 std::cerr << os.str();
4281 }
4282}
4283
4284template <class LocalOrdinal, class GlobalOrdinal, class Node>
4285void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4286 makeImportExport(Teuchos::Array<int>& remotePIDs,
4287 const bool useRemotePIDs) {
4288 using Teuchos::ParameterList;
4289 using Teuchos::RCP;
4290 using Teuchos::rcp;
4291 using ::Tpetra::Details::ProfilingRegion;
4292 const char tfecfFuncName[] = "makeImportExport: ";
4293 ProfilingRegion regionMIE("Tpetra::CrsGraph::makeImportExport");
4294
4295 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::logic_error,
4296 "This method may not be called unless the graph has a column Map.");
4297 RCP<ParameterList> params = this->getNonconstParameterList(); // could be null
4298
4299 // Don't do any checks to see if we need to create the Import, if
4300 // it exists already.
4301 //
4302 // FIXME (mfh 25 Mar 2013) This will become incorrect if we
4303 // change CrsGraph in the future to allow changing the column
4304 // Map after fillComplete. For now, the column Map is fixed
4305 // after the first fillComplete call.
4306 if (importer_.is_null()) {
4307 // Create the Import instance if necessary.
4308 if (domainMap_ != colMap_ && (!domainMap_->isSameAs(*colMap_))) {
4309 if (params.is_null() || !params->isSublist("Import")) {
4310 if (useRemotePIDs) {
4311 importer_ = rcp(new import_type(domainMap_, colMap_, remotePIDs));
4312 } else {
4313 importer_ = rcp(new import_type(domainMap_, colMap_));
4314 }
4315 } else {
4316 RCP<ParameterList> importSublist = sublist(params, "Import", true);
4317 if (useRemotePIDs) {
4318 RCP<import_type> newImp =
4319 rcp(new import_type(domainMap_, colMap_, remotePIDs,
4320 importSublist));
4321 importer_ = newImp;
4322 } else {
4323 importer_ = rcp(new import_type(domainMap_, colMap_, importSublist));
4324 }
4325 }
4326 }
4327 }
4328
4329 // Don't do any checks to see if we need to create the Export, if
4330 // it exists already.
4331 if (exporter_.is_null()) {
4332 // Create the Export instance if necessary.
4333 if (rangeMap_ != rowMap_ && !rangeMap_->isSameAs(*rowMap_)) {
4334 if (params.is_null() || !params->isSublist("Export")) {
4335 exporter_ = rcp(new export_type(rowMap_, rangeMap_));
4336 } else {
4337 RCP<ParameterList> exportSublist = sublist(params, "Export", true);
4338 exporter_ = rcp(new export_type(rowMap_, rangeMap_, exportSublist));
4339 }
4340 }
4341 }
4342}
4343
4344template <class LocalOrdinal, class GlobalOrdinal, class Node>
4345std::string
4346CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4347 description() const {
4348 std::ostringstream oss;
4349 oss << dist_object_type::description();
4350 if (isFillComplete()) {
4351 oss << "{status = fill complete"
4352 << ", global rows = " << getGlobalNumRows()
4353 << ", global cols = " << getGlobalNumCols()
4354 << ", global num entries = " << getGlobalNumEntries()
4355 << "}";
4356 } else {
4357 oss << "{status = fill not complete"
4358 << ", global rows = " << getGlobalNumRows()
4359 << "}";
4360 }
4361 return oss.str();
4362}
4363
4364template <class LocalOrdinal, class GlobalOrdinal, class Node>
4365void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4366 describe(Teuchos::FancyOStream& out,
4367 const Teuchos::EVerbosityLevel verbLevel) const {
4368 using std::endl;
4369 using std::setw;
4370 using Teuchos::ArrayView;
4371 using Teuchos::Comm;
4372 using Teuchos::RCP;
4373 using Teuchos::VERB_DEFAULT;
4374 using Teuchos::VERB_EXTREME;
4375 using Teuchos::VERB_HIGH;
4376 using Teuchos::VERB_LOW;
4377 using Teuchos::VERB_MEDIUM;
4378 using Teuchos::VERB_NONE;
4379
4380 Teuchos::EVerbosityLevel vl = verbLevel;
4381 if (vl == VERB_DEFAULT) vl = VERB_LOW;
4382 RCP<const Comm<int>> comm = this->getComm();
4383 const int myImageID = comm->getRank(),
4384 numImages = comm->getSize();
4385 size_t width = 1;
4386 for (size_t dec = 10; dec < getGlobalNumRows(); dec *= 10) {
4387 ++width;
4388 }
4389 width = std::max<size_t>(width, static_cast<size_t>(11)) + 2;
4390 Teuchos::OSTab tab(out);
4391 // none: print nothing
4392 // low: print O(1) info from node 0
4393 // medium: print O(P) info, num entries per node
4394 // high: print O(N) info, num entries per row
4395 // extreme: print O(NNZ) info: print graph indices
4396 //
4397 // for medium and higher, print constituent objects at specified verbLevel
4398 if (vl != VERB_NONE) {
4399 if (myImageID == 0) out << this->description() << std::endl;
4400 // O(1) globals, minus what was already printed by description()
4401 if (isFillComplete() && myImageID == 0) {
4402 out << "Global max number of row entries = " << globalMaxNumRowEntries_ << std::endl;
4403 }
4404 // constituent objects
4405 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4406 if (myImageID == 0) out << "\nRow map: " << std::endl;
4407 rowMap_->describe(out, vl);
4408 if (colMap_ != Teuchos::null) {
4409 if (myImageID == 0) out << "\nColumn map: " << std::endl;
4410 colMap_->describe(out, vl);
4411 }
4412 if (domainMap_ != Teuchos::null) {
4413 if (myImageID == 0) out << "\nDomain map: " << std::endl;
4414 domainMap_->describe(out, vl);
4415 }
4416 if (rangeMap_ != Teuchos::null) {
4417 if (myImageID == 0) out << "\nRange map: " << std::endl;
4418 rangeMap_->describe(out, vl);
4419 }
4420 }
4421 // O(P) data
4422 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4423 for (int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4424 if (myImageID == imageCtr) {
4425 out << "Node ID = " << imageCtr << std::endl
4426 << "Node number of entries = " << this->getLocalNumEntries() << std::endl
4427 << "Node max number of entries = " << nodeMaxNumRowEntries_ << std::endl;
4428 if (!indicesAreAllocated()) {
4429 out << "Indices are not allocated." << std::endl;
4430 }
4431 }
4432 comm->barrier();
4433 comm->barrier();
4434 comm->barrier();
4435 }
4436 }
4437 // O(N) and O(NNZ) data
4438 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
4439 for (int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4440 if (myImageID == imageCtr) {
4441 out << std::setw(width) << "Node ID"
4442 << std::setw(width) << "Global Row"
4443 << std::setw(width) << "Num Entries";
4444 if (vl == VERB_EXTREME) {
4445 out << " Entries";
4446 }
4447 out << std::endl;
4448 const LocalOrdinal lclNumRows =
4449 static_cast<LocalOrdinal>(this->getLocalNumRows());
4450 for (LocalOrdinal r = 0; r < lclNumRows; ++r) {
4451 const RowInfo rowinfo = this->getRowInfo(r);
4452 GlobalOrdinal gid = rowMap_->getGlobalElement(r);
4453 out << std::setw(width) << myImageID
4454 << std::setw(width) << gid
4455 << std::setw(width) << rowinfo.numEntries;
4456 if (vl == VERB_EXTREME) {
4457 out << " ";
4458 if (isGloballyIndexed()) {
4459 auto rowview = gblInds_wdv.getHostView(Access::ReadOnly);
4460 for (size_t j = 0; j < rowinfo.numEntries; ++j) {
4461 GlobalOrdinal colgid = rowview[j + rowinfo.offset1D];
4462 out << colgid << " ";
4463 }
4464 } else if (isLocallyIndexed()) {
4465 auto rowview = lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
4466 for (size_t j = 0; j < rowinfo.numEntries; ++j) {
4467 LocalOrdinal collid = rowview[j + rowinfo.offset1D];
4468 out << colMap_->getGlobalElement(collid) << " ";
4469 }
4470 }
4471 }
4472 out << std::endl;
4473 }
4474 }
4475 comm->barrier();
4476 comm->barrier();
4477 comm->barrier();
4478 }
4479 }
4480 }
4481}
4482
4483template <class LocalOrdinal, class GlobalOrdinal, class Node>
4484bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4485 checkSizes(const SrcDistObject& /* source */) {
4486 // It's not clear what kind of compatibility checks on sizes can
4487 // be performed here. Epetra_CrsGraph doesn't check any sizes for
4488 // compatibility.
4489 return true;
4490}
4491
4492template <class LocalOrdinal, class GlobalOrdinal, class Node>
4493void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4494 copyAndPermute(const SrcDistObject& source,
4495 const size_t numSameIDs,
4496 const Kokkos::DualView<const local_ordinal_type*,
4497 buffer_device_type>& permuteToLIDs,
4498 const Kokkos::DualView<const local_ordinal_type*,
4499 buffer_device_type>& permuteFromLIDs,
4500 const CombineMode /*CM*/) {
4501 using std::endl;
4502 using LO = local_ordinal_type;
4503 using GO = global_ordinal_type;
4504 using this_CRS_type = CrsGraph<LO, GO, node_type>;
4505 const char tfecfFuncName[] = "copyAndPermute: ";
4506 const bool verbose = verbose_;
4507
4508 if (Details::Behavior::useNewCopyAndPermute()) {
4509 const row_graph_type& srcRowGraph = dynamic_cast<const row_graph_type&>(source);
4510 copyAndPermuteNew(srcRowGraph, *this, numSameIDs, permuteToLIDs, permuteFromLIDs, INSERT);
4511 return;
4512 }
4513
4514 Details::ProfilingRegion regionCAP("Tpetra::CrsGraph::copyAndPermute");
4515
4516 std::unique_ptr<std::string> prefix;
4517 if (verbose) {
4518 prefix = this->createPrefix("CrsGraph", "copyAndPermute");
4519 std::ostringstream os;
4520 os << *prefix << endl;
4521 std::cerr << os.str();
4522 }
4523
4524 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(permuteToLIDs.extent(0) != permuteFromLIDs.extent(0),
4525 std::runtime_error, "permuteToLIDs.extent(0) = " << permuteToLIDs.extent(0) << " != permuteFromLIDs.extent(0) = " << permuteFromLIDs.extent(0) << ".");
4526
4527 // We know from checkSizes that the source object is a
4528 // row_graph_type, so we don't need to check again.
4529 const row_graph_type& srcRowGraph =
4530 dynamic_cast<const row_graph_type&>(source);
4531
4532 if (verbose) {
4533 std::ostringstream os;
4534 os << *prefix << "Compute padding" << endl;
4535 std::cerr << os.str();
4536 }
4537 auto padding = computeCrsPadding(srcRowGraph, numSameIDs,
4538 permuteToLIDs, permuteFromLIDs, verbose);
4539 applyCrsPadding(*padding, verbose);
4540
4541 // If the source object is actually a CrsGraph, we can use view
4542 // mode instead of copy mode to access the entries in each row,
4543 // if the graph is not fill complete.
4544 const this_CRS_type* srcCrsGraph =
4545 dynamic_cast<const this_CRS_type*>(&source);
4546
4547 const map_type& srcRowMap = *(srcRowGraph.getRowMap());
4548 const map_type& tgtRowMap = *(getRowMap());
4549 const bool src_filled = srcRowGraph.isFillComplete();
4550 nonconst_global_inds_host_view_type row_copy;
4551 LO myid = 0;
4552
4553 //
4554 // "Copy" part of "copy and permute."
4555 //
4556 if (src_filled || srcCrsGraph == nullptr) {
4557 if (verbose) {
4558 std::ostringstream os;
4559 os << *prefix << "src_filled || srcCrsGraph == nullptr" << endl;
4560 std::cerr << os.str();
4561 }
4562 // If the source graph is fill complete, we can't use view mode,
4563 // because the data might be stored in a different format not
4564 // compatible with the expectations of view mode. Also, if the
4565 // source graph is not a CrsGraph, we can't use view mode,
4566 // because RowGraph only provides copy mode access to the data.
4567 for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
4568 const GO gid = srcRowMap.getGlobalElement(myid);
4569 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow(gid);
4570 Kokkos::resize(row_copy, row_length);
4571 size_t check_row_length = 0;
4572 srcRowGraph.getGlobalRowCopy(gid, row_copy, check_row_length);
4573 this->insertGlobalIndices(gid, row_length, row_copy.data());
4574 }
4575 } else {
4576 if (verbose) {
4577 std::ostringstream os;
4578 os << *prefix << "! src_filled && srcCrsGraph != nullptr" << endl;
4579 std::cerr << os.str();
4580 }
4581 for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
4582 const GO gid = srcRowMap.getGlobalElement(myid);
4583 global_inds_host_view_type row;
4584 srcCrsGraph->getGlobalRowView(gid, row);
4585 this->insertGlobalIndices(gid, row.extent(0), row.data());
4586 }
4587 }
4588
4589 //
4590 // "Permute" part of "copy and permute."
4591 //
4592 auto permuteToLIDs_h = permuteToLIDs.view_host();
4593 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
4594
4595 if (src_filled || srcCrsGraph == nullptr) {
4596 for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
4597 const GO mygid = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
4598 const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
4599 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow(srcgid);
4600 Kokkos::resize(row_copy, row_length);
4601 size_t check_row_length = 0;
4602 srcRowGraph.getGlobalRowCopy(srcgid, row_copy, check_row_length);
4603 this->insertGlobalIndices(mygid, row_length, row_copy.data());
4604 }
4605 } else {
4606 for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
4607 const GO mygid = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
4608 const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
4609 global_inds_host_view_type row;
4610 srcCrsGraph->getGlobalRowView(srcgid, row);
4611 this->insertGlobalIndices(mygid, row.extent(0), row.data());
4612 }
4613 }
4614
4615 if (verbose) {
4616 std::ostringstream os;
4617 os << *prefix << "Done" << endl;
4618 std::cerr << os.str();
4619 }
4620}
4621
4622template <class LocalOrdinal, class GlobalOrdinal, class Node>
4623void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4624 applyCrsPadding(const padding_type& padding,
4625 const bool verbose) {
4626 using Details::padCrsArrays;
4627 using Details::ProfilingRegion;
4628 using std::endl;
4629 using LO = local_ordinal_type;
4630 using row_ptrs_type =
4631 typename local_graph_device_type::row_map_type::non_const_type;
4632 using range_policy =
4633 Kokkos::RangePolicy<execution_space, Kokkos::IndexType<LO>>;
4634 const char tfecfFuncName[] = "applyCrsPadding";
4635 ProfilingRegion regionCAP("Tpetra::CrsGraph::applyCrsPadding");
4636
4637 std::unique_ptr<std::string> prefix;
4638 if (verbose) {
4639 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4640 std::ostringstream os;
4641 os << *prefix << "padding: ";
4642 padding.print(os);
4643 os << endl;
4644 std::cerr << os.str();
4645 }
4646 const int myRank = !verbose ? -1 : [&]() {
4647 auto map = this->getMap();
4648 if (map.is_null()) {
4649 return -1;
4650 }
4651 auto comm = map->getComm();
4652 if (comm.is_null()) {
4653 return -1;
4654 }
4655 return comm->getRank();
4656 }();
4657
4658 // FIXME (mfh 10 Feb 2020) We shouldn't actually reallocate
4659 // row_ptrs_beg or allocate row_ptrs_end unless the allocation
4660 // size needs to increase. That should be the job of
4661 // padCrsArrays.
4662
4663 // Assume global indexing we don't have any indices yet
4664 if (!indicesAreAllocated()) {
4665 if (verbose) {
4666 std::ostringstream os;
4667 os << *prefix << "Call allocateIndices" << endl;
4668 std::cerr << os.str();
4669 }
4670 allocateIndices(GlobalIndices, verbose);
4671 }
4672 TEUCHOS_ASSERT(indicesAreAllocated());
4673
4674 // Making copies here because k_rowPtrs_ has a const type. Otherwise, we
4675 // would use it directly.
4676
4677 auto rowPtrsUnpacked_dev = this->getRowPtrsUnpackedDevice();
4678 if (verbose) {
4679 std::ostringstream os;
4680 os << *prefix << "Allocate row_ptrs_beg: "
4681 << rowPtrsUnpacked_dev.extent(0) << endl;
4682 std::cerr << os.str();
4683 }
4684 using Kokkos::view_alloc;
4685 using Kokkos::WithoutInitializing;
4686 row_ptrs_type row_ptrs_beg(
4687 view_alloc("row_ptrs_beg", WithoutInitializing),
4688 rowPtrsUnpacked_dev.extent(0));
4689 // DEEP_COPY REVIEW - DEVICE-TO-DEVICE
4690 Kokkos::deep_copy(execution_space(), row_ptrs_beg, rowPtrsUnpacked_dev);
4691
4692 const size_t N = row_ptrs_beg.extent(0) == 0 ? size_t(0) : size_t(row_ptrs_beg.extent(0) - 1);
4693 if (verbose) {
4694 std::ostringstream os;
4695 os << *prefix << "Allocate row_ptrs_end: " << N << endl;
4696 std::cerr << os.str();
4697 }
4698 row_ptrs_type row_ptrs_end(
4699 view_alloc("row_ptrs_end", WithoutInitializing), N);
4700 row_ptrs_type num_row_entries;
4701
4702 const bool refill_num_row_entries = k_numRowEntries_.extent(0) != 0;
4703
4704 execution_space().fence(); // we need above deep_copy to be done
4705
4706 if (refill_num_row_entries) { // Case 1: Unpacked storage
4707 // We can't assume correct *this capture until C++17, and it's
4708 // likely more efficient just to capture what we need anyway.
4709 num_row_entries =
4710 row_ptrs_type(view_alloc("num_row_entries", WithoutInitializing), N);
4711 Kokkos::deep_copy(num_row_entries, this->k_numRowEntries_);
4712 Kokkos::parallel_for(
4713 "Fill end row pointers", range_policy(0, N),
4714 KOKKOS_LAMBDA(const size_t i) {
4715 row_ptrs_end(i) = row_ptrs_beg(i) + num_row_entries(i);
4716 });
4717 } else {
4718 // FIXME (mfh 10 Feb 2020) Fix padCrsArrays so that if packed
4719 // storage, we don't need row_ptr_end to be separate allocation;
4720 // could just have it alias row_ptr_beg+1.
4721 Kokkos::parallel_for(
4722 "Fill end row pointers", range_policy(0, N),
4723 KOKKOS_LAMBDA(const size_t i) {
4724 row_ptrs_end(i) = row_ptrs_beg(i + 1);
4725 });
4726 }
4727
4728 if (isGloballyIndexed()) {
4729 padCrsArrays(row_ptrs_beg, row_ptrs_end, gblInds_wdv,
4730 padding, myRank, verbose);
4731 } else {
4732 padCrsArrays(row_ptrs_beg, row_ptrs_end, lclIndsUnpacked_wdv,
4733 padding, myRank, verbose);
4734 }
4735
4736 if (refill_num_row_entries) {
4737 Kokkos::parallel_for(
4738 "Fill num entries", range_policy(0, N),
4739 KOKKOS_LAMBDA(const size_t i) {
4740 num_row_entries(i) = row_ptrs_end(i) - row_ptrs_beg(i);
4741 });
4742 Kokkos::deep_copy(this->k_numRowEntries_, num_row_entries);
4743 }
4744 if (verbose) {
4745 std::ostringstream os;
4746 os << *prefix << "Reassign k_rowPtrs_; old size: "
4747 << rowPtrsUnpacked_dev.extent(0) << ", new size: "
4748 << row_ptrs_beg.extent(0) << endl;
4749 std::cerr << os.str();
4750 TEUCHOS_ASSERT(rowPtrsUnpacked_dev.extent(0) == row_ptrs_beg.extent(0));
4751 }
4752
4753 setRowPtrsUnpacked(row_ptrs_beg);
4754}
4755
4756template <class LocalOrdinal, class GlobalOrdinal, class Node>
4757std::unique_ptr<
4758 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
4759CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4760 computeCrsPadding(
4761 const RowGraph<LocalOrdinal, GlobalOrdinal, Node>& source,
4762 const size_t numSameIDs,
4763 const Kokkos::DualView<const local_ordinal_type*,
4764 buffer_device_type>& permuteToLIDs,
4765 const Kokkos::DualView<const local_ordinal_type*,
4766 buffer_device_type>& permuteFromLIDs,
4767 const bool verbose) const {
4768 using LO = local_ordinal_type;
4769 using std::endl;
4770
4771 std::unique_ptr<std::string> prefix;
4772 if (verbose) {
4773 prefix = this->createPrefix("CrsGraph",
4774 "computeCrsPadding(same & permute)");
4775 std::ostringstream os;
4776 os << *prefix << "{numSameIDs: " << numSameIDs
4777 << ", numPermutes: " << permuteFromLIDs.extent(0) << "}"
4778 << endl;
4779 std::cerr << os.str();
4780 }
4781
4782 const int myRank = [&]() {
4783 auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
4784 return comm.is_null() ? -1 : comm->getRank();
4785 }();
4786 std::unique_ptr<padding_type> padding(
4787 new padding_type(myRank, numSameIDs,
4788 permuteFromLIDs.extent(0)));
4789
4790 computeCrsPaddingForSameIDs(*padding, source,
4791 static_cast<LO>(numSameIDs));
4792 computeCrsPaddingForPermutedIDs(*padding, source, permuteToLIDs,
4793 permuteFromLIDs);
4794 return padding;
4795}
4796
4797template <class LocalOrdinal, class GlobalOrdinal, class Node>
4798void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4799 computeCrsPaddingForSameIDs(
4800 padding_type& padding,
4801 const RowGraph<local_ordinal_type, global_ordinal_type,
4802 node_type>& source,
4803 const local_ordinal_type numSameIDs) const {
4804 using LO = local_ordinal_type;
4805 using GO = global_ordinal_type;
4806 using Details::Impl::getRowGraphGlobalRow;
4807 using std::endl;
4808 const char tfecfFuncName[] = "computeCrsPaddingForSameIds";
4809
4810 std::unique_ptr<std::string> prefix;
4811 const bool verbose = verbose_;
4812 if (verbose) {
4813 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4814 std::ostringstream os;
4815 os << *prefix << "numSameIDs: " << numSameIDs << endl;
4816 std::cerr << os.str();
4817 }
4818
4819 if (numSameIDs == 0) {
4820 return;
4821 }
4822
4823 const map_type& srcRowMap = *(source.getRowMap());
4824 const map_type& tgtRowMap = *rowMap_;
4825 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
4826 const this_CRS_type* srcCrs = dynamic_cast<const this_CRS_type*>(&source);
4827 const bool src_is_unique =
4828 srcCrs == nullptr ? false : srcCrs->isMerged();
4829 const bool tgt_is_unique = this->isMerged();
4830
4831 std::vector<GO> srcGblColIndsScratch;
4832 std::vector<GO> tgtGblColIndsScratch;
4833
4834 execute_sync_host_uvm_access(); // protect host UVM access
4835 for (LO lclRowInd = 0; lclRowInd < numSameIDs; ++lclRowInd) {
4836 const GO srcGblRowInd = srcRowMap.getGlobalElement(lclRowInd);
4837 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(lclRowInd);
4838 auto srcGblColInds = getRowGraphGlobalRow(
4839 srcGblColIndsScratch, source, srcGblRowInd);
4840 auto tgtGblColInds = getRowGraphGlobalRow(
4841 tgtGblColIndsScratch, *this, tgtGblRowInd);
4842 padding.update_same(lclRowInd, tgtGblColInds.getRawPtr(),
4843 tgtGblColInds.size(), tgt_is_unique,
4844 srcGblColInds.getRawPtr(),
4845 srcGblColInds.size(), src_is_unique);
4846 }
4847 if (verbose) {
4848 std::ostringstream os;
4849 os << *prefix << "Done" << endl;
4850 std::cerr << os.str();
4851 }
4852}
4853
4854template <class LocalOrdinal, class GlobalOrdinal, class Node>
4855void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4856 computeCrsPaddingForPermutedIDs(
4857 padding_type& padding,
4858 const RowGraph<local_ordinal_type, global_ordinal_type,
4859 node_type>& source,
4860 const Kokkos::DualView<const local_ordinal_type*,
4861 buffer_device_type>& permuteToLIDs,
4862 const Kokkos::DualView<const local_ordinal_type*,
4863 buffer_device_type>& permuteFromLIDs) const {
4864 using LO = local_ordinal_type;
4865 using GO = global_ordinal_type;
4866 using Details::Impl::getRowGraphGlobalRow;
4867 using std::endl;
4868 const char tfecfFuncName[] = "computeCrsPaddingForPermutedIds";
4869
4870 std::unique_ptr<std::string> prefix;
4871 const bool verbose = verbose_;
4872 if (verbose) {
4873 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4874 std::ostringstream os;
4875 os << *prefix << "permuteToLIDs.extent(0): "
4876 << permuteToLIDs.extent(0)
4877 << ", permuteFromLIDs.extent(0): "
4878 << permuteFromLIDs.extent(0) << endl;
4879 std::cerr << os.str();
4880 }
4881
4882 if (permuteToLIDs.extent(0) == 0) {
4883 return;
4884 }
4885
4886 const map_type& srcRowMap = *(source.getRowMap());
4887 const map_type& tgtRowMap = *rowMap_;
4888 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
4889 const this_CRS_type* srcCrs = dynamic_cast<const this_CRS_type*>(&source);
4890 const bool src_is_unique =
4891 srcCrs == nullptr ? false : srcCrs->isMerged();
4892 const bool tgt_is_unique = this->isMerged();
4893
4894 TEUCHOS_ASSERT(!permuteToLIDs.need_sync_host());
4895 auto permuteToLIDs_h = permuteToLIDs.view_host();
4896 TEUCHOS_ASSERT(!permuteFromLIDs.need_sync_host());
4897 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
4898
4899 std::vector<GO> srcGblColIndsScratch;
4900 std::vector<GO> tgtGblColIndsScratch;
4901 const LO numPermutes = static_cast<LO>(permuteToLIDs_h.extent(0));
4902
4903 execute_sync_host_uvm_access(); // protect host UVM access
4904 for (LO whichPermute = 0; whichPermute < numPermutes; ++whichPermute) {
4905 const LO srcLclRowInd = permuteFromLIDs_h[whichPermute];
4906 const GO srcGblRowInd = srcRowMap.getGlobalElement(srcLclRowInd);
4907 auto srcGblColInds = getRowGraphGlobalRow(
4908 srcGblColIndsScratch, source, srcGblRowInd);
4909 const LO tgtLclRowInd = permuteToLIDs_h[whichPermute];
4910 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(tgtLclRowInd);
4911 auto tgtGblColInds = getRowGraphGlobalRow(
4912 tgtGblColIndsScratch, *this, tgtGblRowInd);
4913 padding.update_permute(whichPermute, tgtLclRowInd,
4914 tgtGblColInds.getRawPtr(),
4915 tgtGblColInds.size(), tgt_is_unique,
4916 srcGblColInds.getRawPtr(),
4917 srcGblColInds.size(), src_is_unique);
4918 }
4919
4920 if (verbose) {
4921 std::ostringstream os;
4922 os << *prefix << "Done" << endl;
4923 std::cerr << os.str();
4924 }
4925}
4926
4927template <class LocalOrdinal, class GlobalOrdinal, class Node>
4928std::unique_ptr<
4929 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
4930CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4931 computeCrsPaddingForImports(
4932 const Kokkos::DualView<const local_ordinal_type*,
4933 buffer_device_type>& importLIDs,
4934 Kokkos::DualView<packet_type*, buffer_device_type> imports,
4935 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
4936 const bool verbose) const {
4937 using Details::Impl::getRowGraphGlobalRow;
4938 using std::endl;
4939 using LO = local_ordinal_type;
4940 using GO = global_ordinal_type;
4941 const char tfecfFuncName[] = "computeCrsPaddingForImports";
4942
4943 std::unique_ptr<std::string> prefix;
4944 if (verbose) {
4945 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4946 std::ostringstream os;
4947 os << *prefix << "importLIDs.extent(0): "
4948 << importLIDs.extent(0)
4949 << ", imports.extent(0): "
4950 << imports.extent(0)
4951 << ", numPacketsPerLID.extent(0): "
4952 << numPacketsPerLID.extent(0) << endl;
4953 std::cerr << os.str();
4954 }
4955
4956 const LO numImports = static_cast<LO>(importLIDs.extent(0));
4957 const int myRank = [&]() {
4958 auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
4959 return comm.is_null() ? -1 : comm->getRank();
4960 }();
4961 std::unique_ptr<padding_type> padding(
4962 new padding_type(myRank, numImports));
4963
4964 if (imports.need_sync_host()) {
4965 imports.sync_host();
4966 }
4967 auto imports_h = imports.view_host();
4968 if (numPacketsPerLID.need_sync_host()) {
4969 numPacketsPerLID.sync_host();
4970 }
4971 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
4972
4973 TEUCHOS_ASSERT(!importLIDs.need_sync_host());
4974 auto importLIDs_h = importLIDs.view_host();
4975
4976 const map_type& tgtRowMap = *rowMap_;
4977 // Always merge source column indices, since isMerged() is
4978 // per-process state, and we don't know its value on other
4979 // processes that sent us data.
4980 constexpr bool src_is_unique = false;
4981 const bool tgt_is_unique = isMerged();
4982
4983 std::vector<GO> tgtGblColIndsScratch;
4984 size_t offset = 0;
4985 execute_sync_host_uvm_access(); // protect host UVM access
4986 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
4987 // CrsGraph packs just global column indices, while CrsMatrix
4988 // packs bytes (first the number of entries in the row, then the
4989 // global column indices, then other stuff like the matrix
4990 // values in that row).
4991 const LO origSrcNumEnt =
4992 static_cast<LO>(numPacketsPerLID_h[whichImport]);
4993 GO* const srcGblColInds = imports_h.data() + offset;
4994
4995 const LO tgtLclRowInd = importLIDs_h[whichImport];
4996 const GO tgtGblRowInd =
4997 tgtRowMap.getGlobalElement(tgtLclRowInd);
4998 auto tgtGblColInds = getRowGraphGlobalRow(
4999 tgtGblColIndsScratch, *this, tgtGblRowInd);
5000 const size_t origTgtNumEnt(tgtGblColInds.size());
5001
5002 padding->update_import(whichImport, tgtLclRowInd,
5003 tgtGblColInds.getRawPtr(),
5004 origTgtNumEnt, tgt_is_unique,
5005 srcGblColInds,
5006 origSrcNumEnt, src_is_unique);
5007 offset += origSrcNumEnt;
5008 }
5009
5010 if (verbose) {
5011 std::ostringstream os;
5012 os << *prefix << "Done" << endl;
5013 std::cerr << os.str();
5014 }
5015 return padding;
5016}
5017
5018template <class LocalOrdinal, class GlobalOrdinal, class Node>
5019std::unique_ptr<
5020 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
5021CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5022 computePaddingForCrsMatrixUnpack(
5023 const Kokkos::DualView<const local_ordinal_type*,
5024 buffer_device_type>& importLIDs,
5025 Kokkos::DualView<char*, buffer_device_type> imports,
5026 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
5027 const bool verbose) const {
5028 using Details::PackTraits;
5029 using Details::Impl::getRowGraphGlobalRow;
5030 using std::endl;
5031 using LO = local_ordinal_type;
5032 using GO = global_ordinal_type;
5033 const char tfecfFuncName[] = "computePaddingForCrsMatrixUnpack";
5034
5035 std::unique_ptr<std::string> prefix;
5036 if (verbose) {
5037 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
5038 std::ostringstream os;
5039 os << *prefix << "importLIDs.extent(0): "
5040 << importLIDs.extent(0)
5041 << ", imports.extent(0): "
5042 << imports.extent(0)
5043 << ", numPacketsPerLID.extent(0): "
5044 << numPacketsPerLID.extent(0) << endl;
5045 std::cerr << os.str();
5046 }
5047 const bool extraVerbose =
5048 verbose && Details::Behavior::verbose("CrsPadding");
5049
5050 const LO numImports = static_cast<LO>(importLIDs.extent(0));
5051 TEUCHOS_ASSERT(LO(numPacketsPerLID.extent(0)) >= numImports);
5052 const int myRank = [&]() {
5053 auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
5054 return comm.is_null() ? -1 : comm->getRank();
5055 }();
5056 std::unique_ptr<padding_type> padding(
5057 new padding_type(myRank, numImports));
5058
5059 if (imports.need_sync_host()) {
5060 imports.sync_host();
5061 }
5062 auto imports_h = imports.view_host();
5063 if (numPacketsPerLID.need_sync_host()) {
5064 numPacketsPerLID.sync_host();
5065 }
5066 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5067
5068 TEUCHOS_ASSERT(!importLIDs.need_sync_host());
5069 auto importLIDs_h = importLIDs.view_host();
5070
5071 const map_type& tgtRowMap = *rowMap_;
5072 // Always merge source column indices, since isMerged() is
5073 // per-process state, and we don't know its value on other
5074 // processes that sent us data.
5075 constexpr bool src_is_unique = false;
5076 const bool tgt_is_unique = isMerged();
5077
5078 std::vector<GO> srcGblColIndsScratch;
5079 std::vector<GO> tgtGblColIndsScratch;
5080 size_t offset = 0;
5081 execute_sync_host_uvm_access(); // protect host UVM access
5082 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
5083 // CrsGraph packs just global column indices, while CrsMatrix
5084 // packs bytes (first the number of entries in the row, then the
5085 // global column indices, then other stuff like the matrix
5086 // values in that row).
5087 const size_t numBytes = numPacketsPerLID_h[whichImport];
5088 if (extraVerbose) {
5089 std::ostringstream os;
5090 os << *prefix << "whichImport=" << whichImport
5091 << ", numImports=" << numImports
5092 << ", numBytes=" << numBytes << endl;
5093 std::cerr << os.str();
5094 }
5095 if (numBytes == 0) {
5096 continue; // special case: no entries to unpack for this row
5097 }
5098 LO origSrcNumEnt = 0;
5099 const size_t numEntBeg = offset;
5100 const size_t numEntLen =
5101 PackTraits<LO>::packValueCount(origSrcNumEnt);
5102 TEUCHOS_ASSERT(numBytes >= numEntLen);
5103 TEUCHOS_ASSERT(imports_h.extent(0) >= numEntBeg + numEntLen);
5104 PackTraits<LO>::unpackValue(origSrcNumEnt,
5105 imports_h.data() + numEntBeg);
5106 if (extraVerbose) {
5107 std::ostringstream os;
5108 os << *prefix << "whichImport=" << whichImport
5109 << ", numImports=" << numImports
5110 << ", origSrcNumEnt=" << origSrcNumEnt << endl;
5111 std::cerr << os.str();
5112 }
5113 TEUCHOS_ASSERT(origSrcNumEnt >= LO(0));
5114 TEUCHOS_ASSERT(numBytes >= size_t(numEntLen + origSrcNumEnt * sizeof(GO)));
5115 const size_t gidsBeg = numEntBeg + numEntLen;
5116 if (srcGblColIndsScratch.size() < size_t(origSrcNumEnt)) {
5117 srcGblColIndsScratch.resize(origSrcNumEnt);
5118 }
5119 GO* const srcGblColInds = srcGblColIndsScratch.data();
5120 PackTraits<GO>::unpackArray(srcGblColInds,
5121 imports_h.data() + gidsBeg,
5122 origSrcNumEnt);
5123 const LO tgtLclRowInd = importLIDs_h[whichImport];
5124 const GO tgtGblRowInd =
5125 tgtRowMap.getGlobalElement(tgtLclRowInd);
5126 auto tgtGblColInds = getRowGraphGlobalRow(
5127 tgtGblColIndsScratch, *this, tgtGblRowInd);
5128 const size_t origNumTgtEnt(tgtGblColInds.size());
5129
5130 if (extraVerbose) {
5131 std::ostringstream os;
5132 os << *prefix << "whichImport=" << whichImport
5133 << ", numImports=" << numImports
5134 << ": Call padding->update_import" << endl;
5135 std::cerr << os.str();
5136 }
5137 padding->update_import(whichImport, tgtLclRowInd,
5138 tgtGblColInds.getRawPtr(),
5139 origNumTgtEnt, tgt_is_unique,
5140 srcGblColInds,
5141 origSrcNumEnt, src_is_unique);
5142 offset += numBytes;
5143 }
5144
5145 if (verbose) {
5146 std::ostringstream os;
5147 os << *prefix << "Done" << endl;
5148 std::cerr << os.str();
5149 }
5150 return padding;
5151}
5152
5153template <class LocalOrdinal, class GlobalOrdinal, class Node>
5154void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5155 packAndPrepare(const SrcDistObject& source,
5156 const Kokkos::DualView<const local_ordinal_type*,
5157 buffer_device_type>& exportLIDs,
5158 Kokkos::DualView<packet_type*,
5159 buffer_device_type>& exports,
5160 Kokkos::DualView<size_t*,
5161 buffer_device_type>
5162 numPacketsPerLID,
5163 size_t& constantNumPackets) {
5164 using Tpetra::Details::ProfilingRegion;
5165 using GO = global_ordinal_type;
5166 using std::endl;
5167 using crs_graph_type =
5168 CrsGraph<local_ordinal_type, global_ordinal_type, node_type>;
5169 const char tfecfFuncName[] = "packAndPrepare: ";
5170 ProfilingRegion region_papn("Tpetra::CrsGraph::packAndPrepare");
5171
5172 const bool verbose = verbose_;
5173 std::unique_ptr<std::string> prefix;
5174 if (verbose) {
5175 prefix = this->createPrefix("CrsGraph", "packAndPrepare");
5176 std::ostringstream os;
5177 os << *prefix << "Start" << endl;
5178 std::cerr << os.str();
5179 }
5180
5181 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(exportLIDs.extent(0) != numPacketsPerLID.extent(0),
5182 std::runtime_error,
5183 "exportLIDs.extent(0) = " << exportLIDs.extent(0)
5184 << " != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent(0)
5185 << ".");
5186 const row_graph_type* srcRowGraphPtr =
5187 dynamic_cast<const row_graph_type*>(&source);
5188 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(srcRowGraphPtr == nullptr, std::invalid_argument,
5189 "Source of an Export "
5190 "or Import operation to a CrsGraph must be a RowGraph with the same "
5191 "template parameters.");
5192 // We don't check whether src_graph has had fillComplete called,
5193 // because it doesn't matter whether the *source* graph has been
5194 // fillComplete'd. The target graph can not be fillComplete'd yet.
5195 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillComplete(), std::runtime_error,
5196 "The target graph of an Import or Export must not be fill complete.");
5197
5198 const crs_graph_type* srcCrsGraphPtr =
5199 dynamic_cast<const crs_graph_type*>(&source);
5200
5201 if (srcCrsGraphPtr == nullptr) {
5202 using Teuchos::ArrayView;
5203 using LO = local_ordinal_type;
5204
5205 if (verbose) {
5206 std::ostringstream os;
5207 os << *prefix << "Source is a RowGraph but not a CrsGraph"
5208 << endl;
5209 std::cerr << os.str();
5210 }
5211 // RowGraph::pack serves the "old" DistObject interface. It
5212 // takes Teuchos::ArrayView and Teuchos::Array&. The latter
5213 // entails deep-copying the exports buffer on output. RowGraph
5214 // is a convenience interface when not a CrsGraph, so we accept
5215 // the performance hit.
5216 TEUCHOS_ASSERT(!exportLIDs.need_sync_host());
5217 auto exportLIDs_h = exportLIDs.view_host();
5218 ArrayView<const LO> exportLIDs_av(exportLIDs_h.data(),
5219 exportLIDs_h.extent(0));
5220 Teuchos::Array<GO> exports_a;
5221
5222 numPacketsPerLID.clear_sync_state();
5223 numPacketsPerLID.modify_host();
5224 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5225 ArrayView<size_t> numPacketsPerLID_av(numPacketsPerLID_h.data(),
5226 numPacketsPerLID_h.extent(0));
5227 srcRowGraphPtr->pack(exportLIDs_av, exports_a, numPacketsPerLID_av,
5228 constantNumPackets);
5229 const size_t newSize = static_cast<size_t>(exports_a.size());
5230 if (static_cast<size_t>(exports.extent(0)) != newSize) {
5231 using exports_dv_type = Kokkos::DualView<packet_type*, buffer_device_type>;
5232 exports = exports_dv_type("exports", newSize);
5233 }
5234 Kokkos::View<const packet_type*, Kokkos::HostSpace,
5235 Kokkos::MemoryUnmanaged>
5236 exports_a_h(exports_a.getRawPtr(), newSize);
5237 exports.clear_sync_state();
5238 exports.modify_host();
5239 // DEEP_COPY REVIEW - NOT TESTED
5240 Kokkos::deep_copy(exports.view_host(), exports_a_h);
5241 }
5242 // packCrsGraphNew requires k_rowPtrsPacked_ to be set
5243 else if (!getColMap().is_null() &&
5244 (this->getRowPtrsPackedDevice().extent(0) != 0 ||
5245 getRowMap()->getLocalNumElements() == 0)) {
5246 if (verbose) {
5247 std::ostringstream os;
5248 os << *prefix << "packCrsGraphNew path" << endl;
5249 std::cerr << os.str();
5250 }
5251 using export_pids_type =
5252 Kokkos::DualView<const int*, buffer_device_type>;
5253 export_pids_type exportPIDs; // not filling it; needed for syntax
5254 using LO = local_ordinal_type;
5255 using NT = node_type;
5256 using Tpetra::Details::packCrsGraphNew;
5257 packCrsGraphNew<LO, GO, NT>(*srcCrsGraphPtr, exportLIDs, exportPIDs,
5258 exports, numPacketsPerLID,
5259 constantNumPackets, false);
5260 } else {
5261 srcCrsGraphPtr->packFillActiveNew(exportLIDs, exports, numPacketsPerLID,
5262 constantNumPackets);
5263 }
5264
5265 if (verbose) {
5266 std::ostringstream os;
5267 os << *prefix << "Done" << endl;
5268 std::cerr << os.str();
5269 }
5270}
5271
5272template <class LocalOrdinal, class GlobalOrdinal, class Node>
5273void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5274 pack(const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5275 Teuchos::Array<GlobalOrdinal>& exports,
5276 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5277 size_t& constantNumPackets) const {
5278 auto col_map = this->getColMap();
5279 // packCrsGraph requires k_rowPtrsPacked to be set
5280 if (!col_map.is_null() && (this->getRowPtrsPackedDevice().extent(0) != 0 || getRowMap()->getLocalNumElements() == 0)) {
5281 using Tpetra::Details::packCrsGraph;
5282 packCrsGraph<LocalOrdinal, GlobalOrdinal, Node>(*this, exports, numPacketsPerLID,
5283 exportLIDs, constantNumPackets);
5284 } else {
5285 this->packFillActive(exportLIDs, exports, numPacketsPerLID,
5286 constantNumPackets);
5287 }
5288}
5289
5290template <class LocalOrdinal, class GlobalOrdinal, class Node>
5291void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5292 packFillActive(const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5293 Teuchos::Array<GlobalOrdinal>& exports,
5294 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5295 size_t& constantNumPackets) const {
5296 using std::endl;
5297 using LO = LocalOrdinal;
5298 using GO = GlobalOrdinal;
5299 using host_execution_space =
5300 typename Kokkos::View<size_t*, device_type>::
5301 host_mirror_type::execution_space;
5302 const char tfecfFuncName[] = "packFillActive: ";
5303 const bool verbose = verbose_;
5304
5305 const auto numExportLIDs = exportLIDs.size();
5306 std::unique_ptr<std::string> prefix;
5307 if (verbose) {
5308 prefix = this->createPrefix("CrsGraph", "allocateIndices");
5309 std::ostringstream os;
5310 os << *prefix << "numExportLIDs=" << numExportLIDs << endl;
5311 std::cerr << os.str();
5312 }
5313 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numExportLIDs != numPacketsPerLID.size(), std::runtime_error,
5314 "exportLIDs.size() = " << numExportLIDs << " != numPacketsPerLID.size()"
5315 " = "
5316 << numPacketsPerLID.size() << ".");
5317
5318 const map_type& rowMap = *(this->getRowMap());
5319 const map_type* const colMapPtr = this->colMap_.getRawPtr();
5320 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() && colMapPtr == nullptr, std::logic_error,
5321 "This graph claims to be locally indexed, but its column Map is nullptr. "
5322 "This should never happen. Please report this bug to the Tpetra "
5323 "developers.");
5324
5325 // We may pack different amounts of data for different rows.
5326 constantNumPackets = 0;
5327
5328 // mfh 20 Sep 2017: Teuchos::ArrayView isn't thread safe (well,
5329 // it might be now, but we might as well be safe).
5330 size_t* const numPacketsPerLID_raw = numPacketsPerLID.getRawPtr();
5331 const LO* const exportLIDs_raw = exportLIDs.getRawPtr();
5332
5333 // Count the total number of packets (column indices, in the case
5334 // of a CrsGraph) to pack. While doing so, set
5335 // numPacketsPerLID[i] to the number of entries owned by the
5336 // calling process in (local) row exportLIDs[i] of the graph, that
5337 // the caller wants us to send out.
5338 Kokkos::RangePolicy<host_execution_space, LO> inputRange(0, numExportLIDs);
5339 size_t totalNumPackets = 0;
5340 size_t errCount = 0;
5341 // lambdas turn what they capture const, so we can't
5342 // atomic_add(&errCount,1). Instead, we need a View to modify.
5343 typedef Kokkos::Device<host_execution_space, Kokkos::HostSpace>
5344 host_device_type;
5345 Kokkos::View<size_t, host_device_type> errCountView(&errCount);
5346 constexpr size_t ONE = 1;
5347
5348 execute_sync_host_uvm_access(); // protect host UVM access
5349 Kokkos::parallel_reduce(
5350 "Tpetra::CrsGraph::pack: totalNumPackets",
5351 inputRange,
5352 [=, *this](const LO& i, size_t& curTotalNumPackets) {
5353 const GO gblRow = rowMap.getGlobalElement(exportLIDs_raw[i]);
5354 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid()) {
5355 Kokkos::atomic_add(&errCountView(), ONE);
5356 numPacketsPerLID_raw[i] = 0;
5357 } else {
5358 const size_t numEnt = this->getNumEntriesInGlobalRow(gblRow);
5359 numPacketsPerLID_raw[i] = numEnt;
5360 curTotalNumPackets += numEnt;
5361 }
5362 },
5363 totalNumPackets);
5364
5365 if (verbose) {
5366 std::ostringstream os;
5367 os << *prefix << "totalNumPackets=" << totalNumPackets << endl;
5368 std::cerr << os.str();
5369 }
5370 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
5371 "totalNumPackets count encountered "
5372 "one or more errors! errCount = "
5373 << errCount
5374 << ", totalNumPackets = " << totalNumPackets << ".");
5375 errCount = 0;
5376
5377 // Allocate space for all the column indices to pack.
5378 exports.resize(totalNumPackets);
5379
5380 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->supportsRowViews(), std::logic_error,
5381 "this->supportsRowViews() returns false; this should never happen. "
5382 "Please report this bug to the Tpetra developers.");
5383
5384 // Loop again over the rows to export, and pack rows of indices
5385 // into the output buffer.
5386
5387 if (verbose) {
5388 std::ostringstream os;
5389 os << *prefix << "Pack into exports" << endl;
5390 std::cerr << os.str();
5391 }
5392
5393 // Teuchos::ArrayView may not be thread safe, or may not be
5394 // efficiently thread safe. Better to use the raw pointer.
5395 GO* const exports_raw = exports.getRawPtr();
5396 errCount = 0;
5397 Kokkos::parallel_scan("Tpetra::CrsGraph::pack: pack from views",
5398 inputRange, [=, &prefix, *this](const LO i, size_t& exportsOffset, const bool final) {
5399 const size_t curOffset = exportsOffset;
5400 const GO gblRow = rowMap.getGlobalElement(exportLIDs_raw[i]);
5401 const RowInfo rowInfo =
5402 this->getRowInfoFromGlobalRowIndex(gblRow);
5403
5404 using TDO = Tpetra::Details::OrdinalTraits<size_t>;
5405 if (rowInfo.localRow == TDO::invalid()) {
5406 if (verbose) {
5407 std::ostringstream os;
5408 os << *prefix << ": INVALID rowInfo: i=" << i
5409 << ", lclRow=" << exportLIDs_raw[i] << endl;
5410 std::cerr << os.str();
5411 }
5412 Kokkos::atomic_add(&errCountView(), ONE);
5413 } else if (curOffset + rowInfo.numEntries > totalNumPackets) {
5414 if (verbose) {
5415 std::ostringstream os;
5416 os << *prefix << ": UH OH! For i=" << i << ", lclRow="
5417 << exportLIDs_raw[i] << ", gblRow=" << gblRow << ", curOffset "
5418 "(= "
5419 << curOffset << ") + numEnt (= " << rowInfo.numEntries
5420 << ") > totalNumPackets (= " << totalNumPackets << ")."
5421 << endl;
5422 std::cerr << os.str();
5423 }
5424 Kokkos::atomic_add(&errCountView(), ONE);
5425 } else {
5426 const LO numEnt = static_cast<LO>(rowInfo.numEntries);
5427 if (this->isLocallyIndexed()) {
5428 auto lclColInds = getLocalIndsViewHost(rowInfo);
5429 if (final) {
5430 for (LO k = 0; k < numEnt; ++k) {
5431 const LO lclColInd = lclColInds(k);
5432 const GO gblColInd = colMapPtr->getGlobalElement(lclColInd);
5433 // Pack it, even if it's wrong. Let the receiving
5434 // process deal with it. Otherwise, we'll miss out
5435 // on any correct data.
5436 exports_raw[curOffset + k] = gblColInd;
5437 } // for each entry in the row
5438 } // final pass?
5439 exportsOffset = curOffset + numEnt;
5440 } else if (this->isGloballyIndexed()) {
5441 auto gblColInds = getGlobalIndsViewHost(rowInfo);
5442 if (final) {
5443 for (LO k = 0; k < numEnt; ++k) {
5444 const GO gblColInd = gblColInds(k);
5445 // Pack it, even if it's wrong. Let the receiving
5446 // process deal with it. Otherwise, we'll miss out
5447 // on any correct data.
5448 exports_raw[curOffset + k] = gblColInd;
5449 } // for each entry in the row
5450 } // final pass?
5451 exportsOffset = curOffset + numEnt;
5452 }
5453 // If neither globally nor locally indexed, then the graph
5454 // has no entries in this row (or indeed, in any row on this
5455 // process) to pack.
5456 }
5457 });
5458
5459 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
5460 "Packing encountered "
5461 "one or more errors! errCount = "
5462 << errCount
5463 << ", totalNumPackets = " << totalNumPackets << ".");
5464
5465 if (verbose) {
5466 std::ostringstream os;
5467 os << *prefix << "Done" << endl;
5468 std::cerr << os.str();
5469 }
5470}
5471
5472template <class LocalOrdinal, class GlobalOrdinal, class Node>
5473void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5474 packFillActiveNew(const Kokkos::DualView<const local_ordinal_type*,
5475 buffer_device_type>& exportLIDs,
5476 Kokkos::DualView<packet_type*,
5477 buffer_device_type>& exports,
5478 Kokkos::DualView<size_t*,
5479 buffer_device_type>
5480 numPacketsPerLID,
5481 size_t& constantNumPackets) const {
5482 using std::endl;
5483 using LO = local_ordinal_type;
5484 using GO = global_ordinal_type;
5485 using host_execution_space = typename Kokkos::View<size_t*,
5486 device_type>::host_mirror_type::execution_space;
5487 using host_device_type =
5488 Kokkos::Device<host_execution_space, Kokkos::HostSpace>;
5489 using exports_dv_type =
5490 Kokkos::DualView<packet_type*, buffer_device_type>;
5491 const char tfecfFuncName[] = "packFillActiveNew: ";
5492 const bool verbose = verbose_;
5493
5494 const auto numExportLIDs = exportLIDs.extent(0);
5495 std::unique_ptr<std::string> prefix;
5496 if (verbose) {
5497 prefix = this->createPrefix("CrsGraph", "packFillActiveNew");
5498 std::ostringstream os;
5499 os << *prefix << "numExportLIDs: " << numExportLIDs
5500 << ", numPacketsPerLID.extent(0): "
5501 << numPacketsPerLID.extent(0) << endl;
5502 std::cerr << os.str();
5503 }
5504 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numExportLIDs != numPacketsPerLID.extent(0), std::runtime_error,
5505 "exportLIDs.extent(0) = " << numExportLIDs
5506 << " != numPacketsPerLID.extent(0) = "
5507 << numPacketsPerLID.extent(0) << ".");
5508 TEUCHOS_ASSERT(!exportLIDs.need_sync_host());
5509 auto exportLIDs_h = exportLIDs.view_host();
5510
5511 const map_type& rowMap = *(this->getRowMap());
5512 const map_type* const colMapPtr = this->colMap_.getRawPtr();
5513 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() && colMapPtr == nullptr, std::logic_error,
5514 "This graph claims to be locally indexed, but its column Map is nullptr. "
5515 "This should never happen. Please report this bug to the Tpetra "
5516 "developers.");
5517
5518 // We may pack different amounts of data for different rows.
5519 constantNumPackets = 0;
5520
5521 numPacketsPerLID.clear_sync_state();
5522 numPacketsPerLID.modify_host();
5523 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5524
5525 // Count the total number of packets (column indices, in the case
5526 // of a CrsGraph) to pack. While doing so, set
5527 // numPacketsPerLID[i] to the number of entries owned by the
5528 // calling process in (local) row exportLIDs[i] of the graph, that
5529 // the caller wants us to send out.
5530 using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
5531 range_type inputRange(0, numExportLIDs);
5532 size_t totalNumPackets = 0;
5533 size_t errCount = 0;
5534 // lambdas turn what they capture const, so we can't
5535 // atomic_add(&errCount,1). Instead, we need a View to modify.
5536 Kokkos::View<size_t, host_device_type> errCountView(&errCount);
5537 constexpr size_t ONE = 1;
5538
5539 if (verbose) {
5540 std::ostringstream os;
5541 os << *prefix << "Compute totalNumPackets" << endl;
5542 std::cerr << os.str();
5543 }
5544
5545 execute_sync_host_uvm_access(); // protect host UVM access
5546 totalNumPackets = 0;
5547 for (size_t i = 0; i < numExportLIDs; ++i) {
5548 const LO lclRow = exportLIDs_h[i];
5549 const GO gblRow = rowMap.getGlobalElement(lclRow);
5550 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid()) {
5551 if (verbose) {
5552 std::ostringstream os;
5553 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5554 << " not in row Map on this process" << endl;
5555 std::cerr << os.str();
5556 }
5557 Kokkos::atomic_add(&errCountView(), ONE);
5558 numPacketsPerLID_h(i) = 0;
5559 } else {
5560 const size_t numEnt = this->getNumEntriesInGlobalRow(gblRow);
5561 numPacketsPerLID_h(i) = numEnt;
5562 totalNumPackets += numEnt;
5563 }
5564 }
5565
5566 if (verbose) {
5567 std::ostringstream os;
5568 os << *prefix << "totalNumPackets: " << totalNumPackets
5569 << ", errCount: " << errCount << endl;
5570 std::cerr << os.str();
5571 }
5572 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
5573 "totalNumPackets count encountered "
5574 "one or more errors! totalNumPackets: "
5575 << totalNumPackets
5576 << ", errCount: " << errCount << ".");
5577
5578 // Allocate space for all the column indices to pack.
5579 if (size_t(exports.extent(0)) < totalNumPackets) {
5580 // FIXME (mfh 09 Apr 2019) Create without initializing.
5581 exports = exports_dv_type("exports", totalNumPackets);
5582 }
5583
5584 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->supportsRowViews(), std::logic_error,
5585 "this->supportsRowViews() returns false; this should never happen. "
5586 "Please report this bug to the Tpetra developers.");
5587
5588 // Loop again over the rows to export, and pack rows of indices
5589 // into the output buffer.
5590
5591 if (verbose) {
5592 std::ostringstream os;
5593 os << *prefix << "Pack into exports buffer" << endl;
5594 std::cerr << os.str();
5595 }
5596
5597 exports.clear_sync_state();
5598 exports.modify_host();
5599 auto exports_h = exports.view_host();
5600
5601 errCount = 0;
5602
5603 // The following parallel_scan needs const host access to lclIndsUnpacked_wdv
5604 // (if locally indexed) or gblInds_wdv (if globally indexed).
5605 if (isLocallyIndexed())
5606 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
5607 else if (isGloballyIndexed())
5608 gblInds_wdv.getHostView(Access::ReadOnly);
5609
5610 Details::disableWDVTracking();
5611 Kokkos::parallel_scan("Tpetra::CrsGraph::packFillActiveNew: Pack exports",
5612 inputRange, [=, &prefix, *this](const LO i, size_t& exportsOffset, const bool final) {
5613 const size_t curOffset = exportsOffset;
5614 const LO lclRow = exportLIDs_h(i);
5615 const GO gblRow = rowMap.getGlobalElement(lclRow);
5616 if (gblRow == Details::OrdinalTraits<GO>::invalid()) {
5617 if (verbose) {
5618 std::ostringstream os;
5619 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5620 << " not in row Map on this process" << endl;
5621 std::cerr << os.str();
5622 }
5623 Kokkos::atomic_add(&errCountView(), ONE);
5624 return;
5625 }
5626
5627 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(gblRow);
5628 if (rowInfo.localRow == Details::OrdinalTraits<size_t>::invalid()) {
5629 if (verbose) {
5630 std::ostringstream os;
5631 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5632 << ", gblRow=" << gblRow << ": invalid rowInfo"
5633 << endl;
5634 std::cerr << os.str();
5635 }
5636 Kokkos::atomic_add(&errCountView(), ONE);
5637 return;
5638 }
5639
5640 if (curOffset + rowInfo.numEntries > totalNumPackets) {
5641 if (verbose) {
5642 std::ostringstream os;
5643 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5644 << ", gblRow=" << gblRow << ", curOffset (= "
5645 << curOffset << ") + numEnt (= " << rowInfo.numEntries
5646 << ") > totalNumPackets (= " << totalNumPackets
5647 << ")." << endl;
5648 std::cerr << os.str();
5649 }
5650 Kokkos::atomic_add(&errCountView(), ONE);
5651 return;
5652 }
5653
5654 const LO numEnt = static_cast<LO>(rowInfo.numEntries);
5655 if (this->isLocallyIndexed()) {
5656 auto lclColInds = getLocalIndsViewHost(rowInfo);
5657 if (final) {
5658 for (LO k = 0; k < numEnt; ++k) {
5659 const LO lclColInd = lclColInds(k);
5660 const GO gblColInd = colMapPtr->getGlobalElement(lclColInd);
5661 // Pack it, even if it's wrong. Let the receiving
5662 // process deal with it. Otherwise, we'll miss out
5663 // on any correct data.
5664 exports_h(curOffset + k) = gblColInd;
5665 } // for each entry in the row
5666 } // final pass?
5667 exportsOffset = curOffset + numEnt;
5668 } else if (this->isGloballyIndexed()) {
5669 auto gblColInds = getGlobalIndsViewHost(rowInfo);
5670 if (final) {
5671 for (LO k = 0; k < numEnt; ++k) {
5672 const GO gblColInd = gblColInds(k);
5673 // Pack it, even if it's wrong. Let the receiving
5674 // process deal with it. Otherwise, we'll miss out
5675 // on any correct data.
5676 exports_h(curOffset + k) = gblColInd;
5677 } // for each entry in the row
5678 } // final pass?
5679 exportsOffset = curOffset + numEnt;
5680 }
5681 // If neither globally nor locally indexed, then the graph
5682 // has no entries in this row (or indeed, in any row on this
5683 // process) to pack.
5684 });
5685 Details::enableWDVTracking();
5686
5687 // TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5688 // (errCount != 0, std::logic_error, "Packing encountered "
5689 // "one or more errors! errCount = " << errCount
5690 // << ", totalNumPackets = " << totalNumPackets << ".");
5691
5692 if (verbose) {
5693 std::ostringstream os;
5694 os << *prefix << "errCount=" << errCount << "; Done" << endl;
5695 std::cerr << os.str();
5696 }
5697}
5698
5699template <class LocalOrdinal, class GlobalOrdinal, class Node>
5700void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5701 unpackAndCombine(const Kokkos::DualView<const local_ordinal_type*,
5702 buffer_device_type>& importLIDs,
5703 Kokkos::DualView<packet_type*,
5704 buffer_device_type>
5705 imports,
5706 Kokkos::DualView<size_t*,
5707 buffer_device_type>
5708 numPacketsPerLID,
5709 const size_t /* constantNumPackets */,
5710 const CombineMode /* combineMode */) {
5711 using Details::ProfilingRegion;
5712 using std::endl;
5713 using LO = local_ordinal_type;
5714 using GO = global_ordinal_type;
5715 const char tfecfFuncName[] = "unpackAndCombine";
5716
5717 ProfilingRegion regionCGC("Tpetra::CrsGraph::unpackAndCombine");
5718 const bool verbose = verbose_;
5719
5720 std::unique_ptr<std::string> prefix;
5721 if (verbose) {
5722 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
5723 std::ostringstream os;
5724 os << *prefix << "Start" << endl;
5725 std::cerr << os.str();
5726 }
5727 {
5728 auto padding = computeCrsPaddingForImports(
5729 importLIDs, imports, numPacketsPerLID, verbose);
5730 applyCrsPadding(*padding, verbose);
5731 if (verbose) {
5732 std::ostringstream os;
5733 os << *prefix << "Done computing & applying padding" << endl;
5734 std::cerr << os.str();
5735 }
5736 }
5737
5738 // FIXME (mfh 02 Apr 2012) REPLACE combine mode has a perfectly
5739 // reasonable meaning, whether or not the matrix is fill complete.
5740 // It's just more work to implement.
5741
5742 // We are not checking the value of the CombineMode input
5743 // argument. For CrsGraph, we only support import/export
5744 // operations if fillComplete has not yet been called. Any
5745 // incoming column-indices are inserted into the target graph. In
5746 // this context, CombineMode values of ADD vs INSERT are
5747 // equivalent. What is the meaning of REPLACE for CrsGraph? If a
5748 // duplicate column-index is inserted, it will be compressed out
5749 // when fillComplete is called.
5750 //
5751 // Note: I think REPLACE means that an existing row is replaced by
5752 // the imported row, i.e., the existing indices are cleared. CGB,
5753 // 6/17/2010
5754
5755 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(importLIDs.extent(0) != numPacketsPerLID.extent(0),
5756 std::runtime_error, ": importLIDs.extent(0) = " << importLIDs.extent(0) << " != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent(0) << ".");
5757 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isFillComplete(), std::runtime_error,
5758 ": Import or Export operations are not allowed on a target "
5759 "CrsGraph that is fillComplete.");
5760
5761 const size_t numImportLIDs(importLIDs.extent(0));
5762 if (numPacketsPerLID.need_sync_host()) {
5763 numPacketsPerLID.sync_host();
5764 }
5765 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5766 if (imports.need_sync_host()) {
5767 imports.sync_host();
5768 }
5769 auto imports_h = imports.view_host();
5770 TEUCHOS_ASSERT(!importLIDs.need_sync_host());
5771 auto importLIDs_h = importLIDs.view_host();
5772
5773 // If we're inserting in local indices, let's pre-allocate
5774 Teuchos::Array<LO> lclColInds;
5775 if (isLocallyIndexed()) {
5776 if (verbose) {
5777 std::ostringstream os;
5778 os << *prefix << "Preallocate local indices scratch" << endl;
5779 std::cerr << os.str();
5780 }
5781 size_t maxNumInserts = 0;
5782 for (size_t i = 0; i < numImportLIDs; ++i) {
5783 maxNumInserts = std::max(maxNumInserts, numPacketsPerLID_h[i]);
5784 }
5785 if (verbose) {
5786 std::ostringstream os;
5787 os << *prefix << "Local indices scratch size: "
5788 << maxNumInserts << endl;
5789 std::cerr << os.str();
5790 }
5791 lclColInds.resize(maxNumInserts);
5792 } else {
5793 if (verbose) {
5794 std::ostringstream os;
5795 os << *prefix;
5796 if (isGloballyIndexed()) {
5797 os << "Graph is globally indexed";
5798 } else {
5799 os << "Graph is neither locally nor globally indexed";
5800 }
5801 os << endl;
5802 std::cerr << os.str();
5803 }
5804 }
5805
5806 TEUCHOS_ASSERT(!rowMap_.is_null());
5807 const map_type& rowMap = *rowMap_;
5808
5809 try {
5810 size_t importsOffset = 0;
5811 for (size_t i = 0; i < numImportLIDs; ++i) {
5812 if (verbose) {
5813 std::ostringstream os;
5814 os << *prefix << "i=" << i << ", numImportLIDs="
5815 << numImportLIDs << endl;
5816 std::cerr << os.str();
5817 }
5818 // We can only unpack into owned rows, since we only have
5819 // local row indices.
5820 const LO lclRow = importLIDs_h[i];
5821 const GO gblRow = rowMap.getGlobalElement(lclRow);
5822 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(gblRow == Teuchos::OrdinalTraits<GO>::invalid(),
5823 std::logic_error, "importLIDs[i=" << i << "]=" << lclRow << " is not in the row Map on the calling "
5824 "process.");
5825 const LO numEnt = numPacketsPerLID_h[i];
5826 const GO* const gblColInds = (numEnt == 0) ? nullptr : imports_h.data() + importsOffset;
5827 if (!isLocallyIndexed()) {
5828 insertGlobalIndicesFiltered(lclRow, gblColInds, numEnt);
5829 } else {
5830 // FIXME (mfh 09 Feb 2020) Now would be a good time to do
5831 // column Map filtering.
5832 for (LO j = 0; j < numEnt; j++) {
5833 lclColInds[j] = colMap_->getLocalElement(gblColInds[j]);
5834 }
5835 insertLocalIndices(lclRow, numEnt, lclColInds.data());
5836 }
5837 importsOffset += numEnt;
5838 }
5839 } catch (std::exception& e) {
5840 TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
5841 "Tpetra::CrsGraph::unpackAndCombine: Insert loop threw an "
5842 "exception: "
5843 << endl
5844 << e.what());
5845 }
5846
5847 if (verbose) {
5848 std::ostringstream os;
5849 os << *prefix << "Done" << endl;
5850 std::cerr << os.str();
5851 }
5852}
5853
5854template <class LocalOrdinal, class GlobalOrdinal, class Node>
5855void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5856 removeEmptyProcessesInPlace(const Teuchos::RCP<const map_type>& newMap) {
5857 using Teuchos::Comm;
5858 using Teuchos::null;
5859 using Teuchos::ParameterList;
5860 using Teuchos::RCP;
5861
5862 // We'll set all the state "transactionally," so that this method
5863 // satisfies the strong exception guarantee. This object's state
5864 // won't be modified until the end of this method.
5865 RCP<const map_type> rowMap, domainMap, rangeMap, colMap;
5866 RCP<import_type> importer;
5867 RCP<export_type> exporter;
5868
5869 rowMap = newMap;
5870 RCP<const Comm<int>> newComm =
5871 (newMap.is_null()) ? null : newMap->getComm();
5872
5873 if (!domainMap_.is_null()) {
5874 if (domainMap_.getRawPtr() == rowMap_.getRawPtr()) {
5875 // Common case: original domain and row Maps are identical.
5876 // In that case, we need only replace the original domain Map
5877 // with the new Map. This ensures that the new domain and row
5878 // Maps _stay_ identical.
5879 domainMap = newMap;
5880 } else {
5881 domainMap = domainMap_->replaceCommWithSubset(newComm);
5882 }
5883 }
5884 if (!rangeMap_.is_null()) {
5885 if (rangeMap_.getRawPtr() == rowMap_.getRawPtr()) {
5886 // Common case: original range and row Maps are identical. In
5887 // that case, we need only replace the original range Map with
5888 // the new Map. This ensures that the new range and row Maps
5889 // _stay_ identical.
5890 rangeMap = newMap;
5891 } else {
5892 rangeMap = rangeMap_->replaceCommWithSubset(newComm);
5893 }
5894 }
5895 if (!colMap_.is_null()) {
5896 colMap = colMap_->replaceCommWithSubset(newComm);
5897 }
5898
5899 // (Re)create the Export and / or Import if necessary.
5900 if (!newComm.is_null()) {
5901 RCP<ParameterList> params = this->getNonconstParameterList(); // could be null
5902 //
5903 // The operations below are collective on the new communicator.
5904 //
5905 // (Re)create the Export object if necessary. If I haven't
5906 // called fillComplete yet, I don't have a rangeMap, so I must
5907 // first check if the _original_ rangeMap is not null. Ditto
5908 // for the Import object and the domain Map.
5909 if (!rangeMap_.is_null() &&
5910 rangeMap != rowMap &&
5911 !rangeMap->isSameAs(*rowMap)) {
5912 if (params.is_null() || !params->isSublist("Export")) {
5913 exporter = rcp(new export_type(rowMap, rangeMap));
5914 } else {
5915 RCP<ParameterList> exportSublist = sublist(params, "Export", true);
5916 exporter = rcp(new export_type(rowMap, rangeMap, exportSublist));
5917 }
5918 }
5919 // (Re)create the Import object if necessary.
5920 if (!domainMap_.is_null() &&
5921 domainMap != colMap &&
5922 !domainMap->isSameAs(*colMap)) {
5923 if (params.is_null() || !params->isSublist("Import")) {
5924 importer = rcp(new import_type(domainMap, colMap));
5925 } else {
5926 RCP<ParameterList> importSublist = sublist(params, "Import", true);
5927 importer = rcp(new import_type(domainMap, colMap, importSublist));
5928 }
5929 }
5930 } // if newComm is not null
5931
5932 // Defer side effects until the end. If no destructors throw
5933 // exceptions (they shouldn't anyway), then this method satisfies
5934 // the strong exception guarantee.
5935 exporter_ = exporter;
5936 importer_ = importer;
5937 rowMap_ = rowMap;
5938 // mfh 31 Mar 2013: DistObject's map_ is the row Map of a CrsGraph
5939 // or CrsMatrix. CrsGraph keeps a redundant pointer (rowMap_) to
5940 // the same object. We might want to get rid of this redundant
5941 // pointer sometime, but for now, we'll leave it alone and just
5942 // set map_ to the same object.
5943 this->map_ = rowMap;
5944 domainMap_ = domainMap;
5945 rangeMap_ = rangeMap;
5946 colMap_ = colMap;
5947}
5948
5949template <class LocalOrdinal, class GlobalOrdinal, class Node>
5950void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5951 getLocalDiagOffsets(const Kokkos::View<size_t*, device_type, Kokkos::MemoryUnmanaged>& offsets) const {
5952 using std::endl;
5953 using LO = LocalOrdinal;
5954 using GO = GlobalOrdinal;
5955 const char tfecfFuncName[] = "getLocalDiagOffsets: ";
5956 const bool verbose = verbose_;
5957
5958 std::unique_ptr<std::string> prefix;
5959 if (verbose) {
5960 prefix = this->createPrefix("CrsGraph", "getLocalDiagOffsets");
5961 std::ostringstream os;
5962 os << *prefix << "offsets.extent(0)=" << offsets.extent(0)
5963 << endl;
5964 std::cerr << os.str();
5965 }
5966
5967 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error, "The graph must have a column Map.");
5968 const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
5969 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<LO>(offsets.extent(0)) < lclNumRows,
5970 std::invalid_argument, "offsets.extent(0) = " << offsets.extent(0) << " < getLocalNumRows() = " << lclNumRows << ".");
5971
5972 const map_type& rowMap = *(this->getRowMap());
5973 const map_type& colMap = *(this->getColMap());
5974
5975 // We only use these in debug mode, but since debug mode is a
5976 // run-time option, they need to exist here. That's why we create
5977 // the vector with explicit size zero, to avoid overhead if debug
5978 // mode is off.
5979 bool allRowMapDiagEntriesInColMap = true;
5980 bool allDiagEntriesFound = true;
5981 bool allOffsetsCorrect = true;
5982 bool noOtherWeirdness = true;
5983 using wrong_offsets_type = std::vector<std::pair<LO, size_t>>;
5984 wrong_offsets_type wrongOffsets(0);
5985
5986 // mfh 12 Mar 2016: LocalMap works on (CUDA) device. It has just
5987 // the subset of Map functionality that we need below.
5988 auto lclRowMap = rowMap.getLocalMap();
5989 auto lclColMap = colMap.getLocalMap();
5990
5991 // FIXME (mfh 16 Dec 2015) It's easy to thread-parallelize this
5992 // setup, at least on the host. For CUDA, we have to use LocalMap
5993 // (that comes from each of the two Maps).
5994
5995 const bool sorted = this->isSorted();
5996 if (isFillComplete()) {
5997 auto lclGraph = this->getLocalGraphDevice();
5998 ::Tpetra::Details::getGraphDiagOffsets(offsets, lclRowMap, lclColMap,
5999 lclGraph.row_map,
6000 lclGraph.entries, sorted);
6001 } else {
6002 // NOTE (mfh 22 Feb 2017): We have to run this code on host,
6003 // since the graph is not fill complete. The previous version
6004 // of this code assumed UVM; this version does not.
6005 auto offsets_h = Kokkos::create_mirror_view(offsets);
6006
6007 for (LO lclRowInd = 0; lclRowInd < lclNumRows; ++lclRowInd) {
6008 // Find the diagonal entry. Since the row Map and column Map
6009 // may differ, we have to compare global row and column
6010 // indices, not local.
6011 const GO gblRowInd = lclRowMap.getGlobalElement(lclRowInd);
6012 const GO gblColInd = gblRowInd;
6013 const LO lclColInd = lclColMap.getLocalElement(gblColInd);
6014
6015 if (lclColInd == Tpetra::Details::OrdinalTraits<LO>::invalid()) {
6016 allRowMapDiagEntriesInColMap = false;
6017 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid();
6018 } else {
6019 const RowInfo rowInfo = this->getRowInfo(lclRowInd);
6020 if (static_cast<LO>(rowInfo.localRow) == lclRowInd &&
6021 rowInfo.numEntries > 0) {
6022 auto colInds = this->getLocalIndsViewHost(rowInfo);
6023 const size_t hint = 0; // not needed for this algorithm
6024 const size_t offset =
6025 KokkosSparse::findRelOffset(colInds, rowInfo.numEntries,
6026 lclColInd, hint, sorted);
6027 offsets_h(lclRowInd) = offset;
6028
6029 if (debug_) {
6030 // Now that we have what we think is an offset, make sure
6031 // that it really does point to the diagonal entry. Offsets
6032 // are _relative_ to each row, not absolute (for the whole
6033 // (local) graph).
6034 typename local_inds_dualv_type::t_host::const_type lclColInds;
6035 try {
6036 lclColInds = this->getLocalIndsViewHost(rowInfo);
6037 } catch (...) {
6038 noOtherWeirdness = false;
6039 }
6040 // Don't continue with error checking if the above failed.
6041 if (noOtherWeirdness) {
6042 const size_t numEnt = lclColInds.extent(0);
6043 if (offset >= numEnt) {
6044 // Offsets are relative to each row, so this means that
6045 // the offset is out of bounds.
6046 allOffsetsCorrect = false;
6047 wrongOffsets.push_back(std::make_pair(lclRowInd, offset));
6048 } else {
6049 const LO actualLclColInd = lclColInds(offset);
6050 const GO actualGblColInd = lclColMap.getGlobalElement(actualLclColInd);
6051 if (actualGblColInd != gblColInd) {
6052 allOffsetsCorrect = false;
6053 wrongOffsets.push_back(std::make_pair(lclRowInd, offset));
6054 }
6055 }
6056 }
6057 } // debug_
6058 } else { // either row is empty, or something went wrong w/ getRowInfo()
6059 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid();
6060 allDiagEntriesFound = false;
6061 }
6062 } // whether lclColInd is a valid local column index
6063 } // for each local row
6064 // DEEP_COPY REVIEW - NOT TESTED
6065 Kokkos::deep_copy(offsets, offsets_h);
6066 } // whether the graph is fill complete
6067
6068 if (verbose && wrongOffsets.size() != 0) {
6069 std::ostringstream os;
6070 os << *prefix << "Wrong offsets: [";
6071 for (size_t k = 0; k < wrongOffsets.size(); ++k) {
6072 os << "(" << wrongOffsets[k].first << ","
6073 << wrongOffsets[k].second << ")";
6074 if (k + 1 < wrongOffsets.size()) {
6075 os << ", ";
6076 }
6077 }
6078 os << "]" << endl;
6079 std::cerr << os.str();
6080 }
6081
6082 if (debug_) {
6083 using std::endl;
6084 using Teuchos::reduceAll;
6085 Teuchos::RCP<const Teuchos::Comm<int>> comm = this->getComm();
6086 const bool localSuccess =
6087 allRowMapDiagEntriesInColMap && allDiagEntriesFound && allOffsetsCorrect;
6088 const int numResults = 5;
6089 int lclResults[5];
6090 lclResults[0] = allRowMapDiagEntriesInColMap ? 1 : 0;
6091 lclResults[1] = allDiagEntriesFound ? 1 : 0;
6092 lclResults[2] = allOffsetsCorrect ? 1 : 0;
6093 lclResults[3] = noOtherWeirdness ? 1 : 0;
6094 // min-all-reduce will compute least rank of all the processes
6095 // that didn't succeed.
6096 lclResults[4] = !localSuccess ? comm->getRank() : comm->getSize();
6097
6098 int gblResults[5];
6099 gblResults[0] = 0;
6100 gblResults[1] = 0;
6101 gblResults[2] = 0;
6102 gblResults[3] = 0;
6103 gblResults[4] = 0;
6104 reduceAll<int, int>(*comm, Teuchos::REDUCE_MIN,
6105 numResults, lclResults, gblResults);
6106
6107 if (gblResults[0] != 1 || gblResults[1] != 1 || gblResults[2] != 1 || gblResults[3] != 1) {
6108 std::ostringstream os; // build error message
6109 os << "Issue(s) that we noticed (on Process " << gblResults[4] << ", "
6110 "possibly among others): "
6111 << endl;
6112 if (gblResults[0] == 0) {
6113 os << " - The column Map does not contain at least one diagonal entry "
6114 "of the graph."
6115 << endl;
6116 }
6117 if (gblResults[1] == 0) {
6118 os << " - On one or more processes, some row does not contain a "
6119 "diagonal entry."
6120 << endl;
6121 }
6122 if (gblResults[2] == 0) {
6123 os << " - On one or more processes, some offsets are incorrect."
6124 << endl;
6125 }
6126 if (gblResults[3] == 0) {
6127 os << " - One or more processes had some other error."
6128 << endl;
6129 }
6130 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
6131 }
6132 } // debug_
6133}
6134
6135template <class LocalOrdinal, class GlobalOrdinal, class Node>
6136void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6137 getLocalOffRankOffsets(offset_device_view_type& offsets) const {
6138 using std::endl;
6139 const char tfecfFuncName[] = "getLocalOffRankOffsets: ";
6140 const bool verbose = verbose_;
6141
6142 std::unique_ptr<std::string> prefix;
6143 if (verbose) {
6144 prefix = this->createPrefix("CrsGraph", "getLocalOffRankOffsets");
6145 std::ostringstream os;
6146 os << *prefix << "offsets.extent(0)=" << offsets.extent(0)
6147 << endl;
6148 std::cerr << os.str();
6149 }
6150
6151 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error, "The graph must have a column Map.");
6152 // Instead of throwing, we could also copy the rowPtr to k_offRankOffsets_.
6153
6154 const size_t lclNumRows = this->getLocalNumRows();
6155
6156 if (haveLocalOffRankOffsets_ && k_offRankOffsets_.extent(0) == lclNumRows + 1) {
6157 offsets = k_offRankOffsets_;
6158 return;
6159 }
6160 haveLocalOffRankOffsets_ = false;
6161
6162 const map_type& colMap = *(this->getColMap());
6163 const map_type& domMap = *(this->getDomainMap());
6164
6165 // mfh 12 Mar 2016: LocalMap works on (CUDA) device. It has just
6166 // the subset of Map functionality that we need below.
6167 auto lclColMap = colMap.getLocalMap();
6168 auto lclDomMap = domMap.getLocalMap();
6169
6170 // FIXME (mfh 16 Dec 2015) It's easy to thread-parallelize this
6171 // setup, at least on the host. For CUDA, we have to use LocalMap
6172 // (that comes from each of the two Maps).
6173
6174 TEUCHOS_ASSERT(this->isSorted());
6175 if (isFillComplete()) {
6176 k_offRankOffsets_ = offset_device_view_type(Kokkos::ViewAllocateWithoutInitializing("offRankOffset"), lclNumRows + 1);
6177 auto lclGraph = this->getLocalGraphDevice();
6178 ::Tpetra::Details::getGraphOffRankOffsets(k_offRankOffsets_,
6179 lclColMap, lclDomMap,
6180 lclGraph);
6181 offsets = k_offRankOffsets_;
6182 haveLocalOffRankOffsets_ = true;
6183 } else {
6184 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error, "Can't get off-rank offsets for non-fill-complete graph");
6185 }
6186}
6187
6188namespace { // (anonymous)
6189
6190// mfh 21 Jan 2016: This is useful for getLocalDiagOffsets (see
6191// below). The point is to avoid the deep copy between the input
6192// Teuchos::ArrayRCP and the internally used Kokkos::View. We
6193// can't use UVM to avoid the deep copy with CUDA, because the
6194// ArrayRCP is a host pointer, while the input to the graph's
6195// getLocalDiagOffsets method is a device pointer. Assigning a
6196// host pointer to a device pointer is incorrect unless the host
6197// pointer points to host pinned memory. The goal is to get rid
6198// of the Teuchos::ArrayRCP overload anyway, so we accept the deep
6199// copy for backwards compatibility.
6200//
6201// We have to use template magic because
6202// "staticGraph_->getLocalDiagOffsets(offsetsHosts)" won't compile
6203// if device_type::memory_space is not Kokkos::HostSpace (as is
6204// the case with CUDA).
6205
6206template <class DeviceType,
6207 const bool memSpaceIsHostSpace =
6208 std::is_same<typename DeviceType::memory_space,
6209 Kokkos::HostSpace>::value>
6210struct HelpGetLocalDiagOffsets {};
6211
6212template <class DeviceType>
6213struct HelpGetLocalDiagOffsets<DeviceType, true> {
6214 typedef DeviceType device_type;
6215 typedef Kokkos::View<size_t*, Kokkos::HostSpace,
6216 Kokkos::MemoryUnmanaged>
6217 device_offsets_type;
6218 typedef Kokkos::View<size_t*, Kokkos::HostSpace,
6219 Kokkos::MemoryUnmanaged>
6220 host_offsets_type;
6221
6222 static device_offsets_type
6223 getDeviceOffsets(const host_offsets_type& hostOffsets) {
6224 // Host and device are the same; no need to allocate a
6225 // temporary device View.
6226 return hostOffsets;
6227 }
6228
6229 static void
6230 copyBackIfNeeded(const host_offsets_type& /* hostOffsets */,
6231 const device_offsets_type& /* deviceOffsets */) { /* copy back not needed; host and device are the same */
6232 }
6233};
6234
6235template <class DeviceType>
6236struct HelpGetLocalDiagOffsets<DeviceType, false> {
6237 typedef DeviceType device_type;
6238 // We have to do a deep copy, since host memory space != device
6239 // memory space. Thus, the device View is managed (we need to
6240 // allocate a temporary device View).
6241 typedef Kokkos::View<size_t*, device_type> device_offsets_type;
6242 typedef Kokkos::View<size_t*, Kokkos::HostSpace,
6243 Kokkos::MemoryUnmanaged>
6244 host_offsets_type;
6245
6246 static device_offsets_type
6247 getDeviceOffsets(const host_offsets_type& hostOffsets) {
6248 // Host memory space != device memory space, so we must
6249 // allocate a temporary device View for the graph.
6250 return device_offsets_type("offsets", hostOffsets.extent(0));
6251 }
6252
6253 static void
6254 copyBackIfNeeded(const host_offsets_type& hostOffsets,
6255 const device_offsets_type& deviceOffsets) {
6256 // DEEP_COPY REVIEW - NOT TESTED
6257 Kokkos::deep_copy(hostOffsets, deviceOffsets);
6258 }
6259};
6260} // namespace
6261
6262template <class LocalOrdinal, class GlobalOrdinal, class Node>
6263void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6264 getLocalDiagOffsets(Teuchos::ArrayRCP<size_t>& offsets) const {
6265 typedef LocalOrdinal LO;
6266 const char tfecfFuncName[] = "getLocalDiagOffsets: ";
6267 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::runtime_error,
6268 "The graph does not yet have a column Map.");
6269 const LO myNumRows = static_cast<LO>(this->getLocalNumRows());
6270 if (static_cast<LO>(offsets.size()) != myNumRows) {
6271 // NOTE (mfh 21 Jan 2016) This means that the method does not
6272 // satisfy the strong exception guarantee (no side effects
6273 // unless successful).
6274 offsets.resize(myNumRows);
6275 }
6276
6277 // mfh 21 Jan 2016: This method unfortunately takes a
6278 // Teuchos::ArrayRCP, which is host memory. The graph wants a
6279 // device pointer. We can't access host memory from the device;
6280 // that's the wrong direction for UVM. (It's the right direction
6281 // for inefficient host pinned memory, but we don't want to use
6282 // that here.) Thus, if device memory space != host memory space,
6283 // we allocate and use a temporary device View to get the offsets.
6284 // If the two spaces are equal, the template magic makes the deep
6285 // copy go away.
6286 typedef HelpGetLocalDiagOffsets<device_type> helper_type;
6287 typedef typename helper_type::host_offsets_type host_offsets_type;
6288 // Unmanaged host View that views the output array.
6289 host_offsets_type hostOffsets(offsets.getRawPtr(), myNumRows);
6290 // Allocate temp device View if host != device, else reuse host array.
6291 auto deviceOffsets = helper_type::getDeviceOffsets(hostOffsets);
6292 // NOT recursion; this calls the overload that takes a device View.
6293 this->getLocalDiagOffsets(deviceOffsets);
6294 helper_type::copyBackIfNeeded(hostOffsets, deviceOffsets);
6295}
6296
6297template <class LocalOrdinal, class GlobalOrdinal, class Node>
6298bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6299 supportsRowViews() const {
6300 return true;
6301}
6302
6303template <class LocalOrdinal, class GlobalOrdinal, class Node>
6304void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6305 transferAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6306 const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>& rowTransfer,
6307 const Teuchos::RCP<const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>>& domainTransfer,
6308 const Teuchos::RCP<const map_type>& domainMap,
6309 const Teuchos::RCP<const map_type>& rangeMap,
6310 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6311 using Teuchos::ArrayRCP;
6312 using Teuchos::ArrayView;
6313 using Teuchos::Comm;
6314 using Teuchos::ParameterList;
6315 using Teuchos::rcp;
6316 using Teuchos::RCP;
6317 using Tpetra::Details::getArrayViewFromDualView;
6318 using Tpetra::Details::packCrsGraphWithOwningPIDs;
6319 using Tpetra::Details::unpackAndCombineIntoCrsArrays;
6320 using Tpetra::Details::unpackAndCombineWithOwningPIDsCount;
6321#ifdef HAVE_TPETRA_MMM_TIMINGS
6322 using std::string;
6323 using Teuchos::TimeMonitor;
6324#endif
6325
6326 using LO = LocalOrdinal;
6327 using GO = GlobalOrdinal;
6328 using NT = node_type;
6329 using this_CRS_type = CrsGraph<LO, GO, NT>;
6330 using ivector_type = Vector<int, LO, GO, NT>;
6331
6332 const char* prefix = "Tpetra::CrsGraph::transferAndFillComplete: ";
6333
6334#ifdef HAVE_TPETRA_MMM_TIMINGS
6335 string label;
6336 if (!params.is_null()) label = params->get("Timer Label", label);
6337 string prefix2 = string("Tpetra ") + label + std::string(": CrsGraph TAFC ");
6338 RCP<TimeMonitor> MM =
6339 rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2 + string("Pack-1"))));
6340#endif
6341
6342 // Make sure that the input argument rowTransfer is either an
6343 // Import or an Export. Import and Export are the only two
6344 // subclasses of Transfer that we defined, but users might
6345 // (unwisely, for now at least) decide to implement their own
6346 // subclasses. Exclude this possibility.
6347 const import_type* xferAsImport = dynamic_cast<const import_type*>(&rowTransfer);
6348 const export_type* xferAsExport = dynamic_cast<const export_type*>(&rowTransfer);
6349 TEUCHOS_TEST_FOR_EXCEPTION(
6350 xferAsImport == nullptr && xferAsExport == nullptr, std::invalid_argument,
6351 prefix << "The 'rowTransfer' input argument must be either an Import or "
6352 "an Export, and its template parameters must match the corresponding "
6353 "template parameters of the CrsGraph.");
6354
6355 // Make sure that the input argument domainTransfer is either an
6356 // Import or an Export. Import and Export are the only two
6357 // subclasses of Transfer that we defined, but users might
6358 // (unwisely, for now at least) decide to implement their own
6359 // subclasses. Exclude this possibility.
6360 Teuchos::RCP<const import_type> xferDomainAsImport =
6361 Teuchos::rcp_dynamic_cast<const import_type>(domainTransfer);
6362 Teuchos::RCP<const export_type> xferDomainAsExport =
6363 Teuchos::rcp_dynamic_cast<const export_type>(domainTransfer);
6364
6365 if (!domainTransfer.is_null()) {
6366 TEUCHOS_TEST_FOR_EXCEPTION(
6367 (xferDomainAsImport.is_null() && xferDomainAsExport.is_null()), std::invalid_argument,
6368 prefix << "The 'domainTransfer' input argument must be either an "
6369 "Import or an Export, and its template parameters must match the "
6370 "corresponding template parameters of the CrsGraph.");
6371
6372 TEUCHOS_TEST_FOR_EXCEPTION(
6373 (xferAsImport != nullptr || !xferDomainAsImport.is_null()) &&
6374 ((xferAsImport != nullptr && xferDomainAsImport.is_null()) ||
6375 (xferAsImport == nullptr && !xferDomainAsImport.is_null())),
6376 std::invalid_argument,
6377 prefix << "The 'rowTransfer' and 'domainTransfer' input arguments "
6378 "must be of the same type (either Import or Export).");
6379
6380 TEUCHOS_TEST_FOR_EXCEPTION(
6381 (xferAsExport != nullptr || !xferDomainAsExport.is_null()) &&
6382 ((xferAsExport != nullptr && xferDomainAsExport.is_null()) ||
6383 (xferAsExport == nullptr && !xferDomainAsExport.is_null())),
6384 std::invalid_argument,
6385 prefix << "The 'rowTransfer' and 'domainTransfer' input arguments "
6386 "must be of the same type (either Import or Export).");
6387
6388 } // domainTransfer != null
6389
6390 // FIXME (mfh 15 May 2014) Wouldn't communication still be needed,
6391 // if the source Map is not distributed but the target Map is?
6392 const bool communication_needed = rowTransfer.getSourceMap()->isDistributed();
6393
6394 //
6395 // Get the caller's parameters
6396 //
6397
6398 bool reverseMode = false; // Are we in reverse mode?
6399 bool restrictComm = false; // Do we need to restrict the communicator?
6400 RCP<ParameterList> graphparams; // parameters for the destination graph
6401 if (!params.is_null()) {
6402 reverseMode = params->get("Reverse Mode", reverseMode);
6403 restrictComm = params->get("Restrict Communicator", restrictComm);
6404 graphparams = sublist(params, "CrsGraph");
6405 }
6406
6407 // Get the new domain and range Maps. We need some of them for error
6408 // checking, now that we have the reverseMode parameter.
6409 RCP<const map_type> MyRowMap = reverseMode ? rowTransfer.getSourceMap() : rowTransfer.getTargetMap();
6410 RCP<const map_type> MyColMap; // create this below
6411 RCP<const map_type> MyDomainMap = !domainMap.is_null() ? domainMap : getDomainMap();
6412 RCP<const map_type> MyRangeMap = !rangeMap.is_null() ? rangeMap : getRangeMap();
6413 RCP<const map_type> BaseRowMap = MyRowMap;
6414 RCP<const map_type> BaseDomainMap = MyDomainMap;
6415
6416 // If the user gave us a nonnull destGraph, then check whether it's
6417 // "pristine." That means that it has no entries.
6418 //
6419 // FIXME (mfh 15 May 2014) If this is not true on all processes,
6420 // then this exception test may hang. It would be better to
6421 // forward an error flag to the next communication phase.
6422 if (!destGraph.is_null()) {
6423 // FIXME (mfh 15 May 2014): The Epetra idiom for checking
6424 // whether a graph or matrix has no entries on the calling
6425 // process, is that it is neither locally nor globally indexed.
6426 // This may change eventually with the Kokkos refactor version
6427 // of Tpetra, so it would be better just to check the quantity
6428 // of interest directly. Note that with the Kokkos refactor
6429 // version of Tpetra, asking for the total number of entries in
6430 // a graph or matrix that is not fill complete might require
6431 // computation (kernel launch), since it is not thread scalable
6432 // to update a count every time an entry is inserted.
6433 const bool NewFlag =
6434 !destGraph->isLocallyIndexed() && !destGraph->isGloballyIndexed();
6435 TEUCHOS_TEST_FOR_EXCEPTION(!NewFlag, std::invalid_argument,
6436 prefix << "The input argument 'destGraph' is only allowed to be nonnull, "
6437 "if its graph is empty (neither locally nor globally indexed).");
6438
6439 // FIXME (mfh 15 May 2014) At some point, we want to change
6440 // graphs and matrices so that their DistObject Map
6441 // (this->getMap()) may differ from their row Map. This will
6442 // make redistribution for 2-D distributions more efficient. I
6443 // hesitate to change this check, because I'm not sure how much
6444 // the code here depends on getMap() and getRowMap() being the
6445 // same.
6446 TEUCHOS_TEST_FOR_EXCEPTION(
6447 !destGraph->getRowMap()->isSameAs(*MyRowMap), std::invalid_argument,
6448 prefix << "The (row) Map of the input argument 'destGraph' is not the "
6449 "same as the (row) Map specified by the input argument 'rowTransfer'.");
6450
6451 TEUCHOS_TEST_FOR_EXCEPTION(
6452 !destGraph->checkSizes(*this), std::invalid_argument,
6453 prefix << "You provided a nonnull destination graph, but checkSizes() "
6454 "indicates that it is not a legal legal target for redistribution from "
6455 "the source graph (*this). This may mean that they do not have the "
6456 "same dimensions.");
6457 }
6458
6459 // If forward mode (the default), then *this's (row) Map must be
6460 // the same as the source Map of the Transfer. If reverse mode,
6461 // then *this's (row) Map must be the same as the target Map of
6462 // the Transfer.
6463 //
6464 // FIXME (mfh 15 May 2014) At some point, we want to change graphs
6465 // and matrices so that their DistObject Map (this->getMap()) may
6466 // differ from their row Map. This will make redistribution for
6467 // 2-D distributions more efficient. I hesitate to change this
6468 // check, because I'm not sure how much the code here depends on
6469 // getMap() and getRowMap() being the same.
6470 TEUCHOS_TEST_FOR_EXCEPTION(
6471 !(reverseMode || getRowMap()->isSameAs(*rowTransfer.getSourceMap())),
6472 std::invalid_argument, prefix << "rowTransfer->getSourceMap() must match this->getRowMap() in forward mode.");
6473
6474 TEUCHOS_TEST_FOR_EXCEPTION(
6475 !(!reverseMode || getRowMap()->isSameAs(*rowTransfer.getTargetMap())),
6476 std::invalid_argument, prefix << "rowTransfer->getTargetMap() must match this->getRowMap() in reverse mode.");
6477
6478 // checks for domainTransfer
6479 TEUCHOS_TEST_FOR_EXCEPTION(
6480 !xferDomainAsImport.is_null() && !xferDomainAsImport->getTargetMap()->isSameAs(*domainMap),
6481 std::invalid_argument,
6482 prefix << "The target map of the 'domainTransfer' input argument must be "
6483 "the same as the rebalanced domain map 'domainMap'");
6484
6485 TEUCHOS_TEST_FOR_EXCEPTION(
6486 !xferDomainAsExport.is_null() && !xferDomainAsExport->getSourceMap()->isSameAs(*domainMap),
6487 std::invalid_argument,
6488 prefix << "The source map of the 'domainTransfer' input argument must be "
6489 "the same as the rebalanced domain map 'domainMap'");
6490
6491 // The basic algorithm here is:
6492 //
6493 // 1. Call the moral equivalent of "Distor.do" to handle the import.
6494 // 2. Copy all the Imported and Copy/Permuted data into the raw
6495 // CrsGraph pointers, still using GIDs.
6496 // 3. Call an optimized version of MakeColMap that avoids the
6497 // Directory lookups (since the importer knows who owns all the
6498 // GIDs) AND reindexes to LIDs.
6499 // 4. Call expertStaticFillComplete()
6500
6501 // Get information from the Importer
6502 const size_t NumSameIDs = rowTransfer.getNumSameIDs();
6503 ArrayView<const LO> ExportLIDs = reverseMode ? rowTransfer.getRemoteLIDs() : rowTransfer.getExportLIDs();
6504 ArrayView<const LO> RemoteLIDs = reverseMode ? rowTransfer.getExportLIDs() : rowTransfer.getRemoteLIDs();
6505 ArrayView<const LO> PermuteToLIDs = reverseMode ? rowTransfer.getPermuteFromLIDs() : rowTransfer.getPermuteToLIDs();
6506 ArrayView<const LO> PermuteFromLIDs = reverseMode ? rowTransfer.getPermuteToLIDs() : rowTransfer.getPermuteFromLIDs();
6507 Distributor& Distor = rowTransfer.getDistributor();
6508
6509 // Owning PIDs
6510 Teuchos::Array<int> SourcePids;
6511 Teuchos::Array<int> TargetPids;
6512 int MyPID = getComm()->getRank();
6513
6514 // Temp variables for sub-communicators
6515 RCP<const map_type> ReducedRowMap, ReducedColMap,
6516 ReducedDomainMap, ReducedRangeMap;
6517 RCP<const Comm<int>> ReducedComm;
6518
6519 // If the user gave us a null destGraph, then construct the new
6520 // destination graph. We will replace its column Map later.
6521 if (destGraph.is_null()) {
6522 destGraph = rcp(new this_CRS_type(MyRowMap, 0, graphparams));
6523 }
6524
6525 /***************************************************/
6526 /***** 1) First communicator restriction phase ****/
6527 /***************************************************/
6528 if (restrictComm) {
6529 ReducedRowMap = MyRowMap->removeEmptyProcesses();
6530 ReducedComm = ReducedRowMap.is_null() ? Teuchos::null : ReducedRowMap->getComm();
6531 destGraph->removeEmptyProcessesInPlace(ReducedRowMap);
6532
6533 ReducedDomainMap = MyRowMap.getRawPtr() == MyDomainMap.getRawPtr() ? ReducedRowMap : MyDomainMap->replaceCommWithSubset(ReducedComm);
6534 ReducedRangeMap = MyRowMap.getRawPtr() == MyRangeMap.getRawPtr() ? ReducedRowMap : MyRangeMap->replaceCommWithSubset(ReducedComm);
6535
6536 // Reset the "my" maps
6537 MyRowMap = ReducedRowMap;
6538 MyDomainMap = ReducedDomainMap;
6539 MyRangeMap = ReducedRangeMap;
6540
6541 // Update my PID, if we've restricted the communicator
6542 if (!ReducedComm.is_null()) {
6543 MyPID = ReducedComm->getRank();
6544 } else {
6545 MyPID = -2; // For debugging
6546 }
6547 } else {
6548 ReducedComm = MyRowMap->getComm();
6549 }
6550
6551 /***************************************************/
6552 /***** 2) From Tpera::DistObject::doTransfer() ****/
6553 /***************************************************/
6554#ifdef HAVE_TPETRA_MMM_TIMINGS
6555 MM = Teuchos::null;
6556 MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2 + string("ImportSetup"))));
6557#endif
6558 // Get the owning PIDs
6559 RCP<const import_type> MyImporter = getImporter();
6560
6561 // check whether domain maps of source graph and base domain map is the same
6562 bool bSameDomainMap = BaseDomainMap->isSameAs(*getDomainMap());
6563
6564 if (!restrictComm && !MyImporter.is_null() && bSameDomainMap) {
6565 // Same domain map as source graph
6566 //
6567 // NOTE: This won't work for restrictComm (because the Import
6568 // doesn't know the restricted PIDs), though writing an
6569 // optimized version for that case would be easy (Import an
6570 // IntVector of the new PIDs). Might want to add this later.
6571 Import_Util::getPids(*MyImporter, SourcePids, false);
6572 } else if (restrictComm && !MyImporter.is_null() && bSameDomainMap) {
6573 // Same domain map as source graph (restricted communicator)
6574 // We need one import from the domain to the column map
6575 ivector_type SourceDomain_pids(getDomainMap(), true);
6576 ivector_type SourceCol_pids(getColMap());
6577 // SourceDomain_pids contains the restricted pids
6578 SourceDomain_pids.putScalar(MyPID);
6579
6580 SourceCol_pids.doImport(SourceDomain_pids, *MyImporter, INSERT);
6581 SourcePids.resize(getColMap()->getLocalNumElements());
6582 SourceCol_pids.get1dCopy(SourcePids());
6583 } else if (MyImporter.is_null() && bSameDomainMap) {
6584 // Graph has no off-process entries
6585 SourcePids.resize(getColMap()->getLocalNumElements());
6586 SourcePids.assign(getColMap()->getLocalNumElements(), MyPID);
6587 } else if (!MyImporter.is_null() &&
6588 !domainTransfer.is_null()) {
6589 // general implementation for rectangular matrices with
6590 // domain map different than SourceGraph domain map.
6591 // User has to provide a DomainTransfer object. We need
6592 // to communications (import/export)
6593
6594 // TargetDomain_pids lives on the rebalanced new domain map
6595 ivector_type TargetDomain_pids(domainMap);
6596 TargetDomain_pids.putScalar(MyPID);
6597
6598 // SourceDomain_pids lives on the non-rebalanced old domain map
6599 ivector_type SourceDomain_pids(getDomainMap());
6600
6601 // SourceCol_pids lives on the non-rebalanced old column map
6602 ivector_type SourceCol_pids(getColMap());
6603
6604 if (!reverseMode && !xferDomainAsImport.is_null()) {
6605 SourceDomain_pids.doExport(TargetDomain_pids, *xferDomainAsImport, INSERT);
6606 } else if (reverseMode && !xferDomainAsExport.is_null()) {
6607 SourceDomain_pids.doExport(TargetDomain_pids, *xferDomainAsExport, INSERT);
6608 } else if (!reverseMode && !xferDomainAsExport.is_null()) {
6609 SourceDomain_pids.doImport(TargetDomain_pids, *xferDomainAsExport, INSERT);
6610 } else if (reverseMode && !xferDomainAsImport.is_null()) {
6611 SourceDomain_pids.doImport(TargetDomain_pids, *xferDomainAsImport, INSERT);
6612 } else {
6613 TEUCHOS_TEST_FOR_EXCEPTION(
6614 true, std::logic_error,
6615 prefix << "Should never get here! Please report this bug to a Tpetra developer.");
6616 }
6617 SourceCol_pids.doImport(SourceDomain_pids, *MyImporter, INSERT);
6618 SourcePids.resize(getColMap()->getLocalNumElements());
6619 SourceCol_pids.get1dCopy(SourcePids());
6620 } else if (BaseDomainMap->isSameAs(*BaseRowMap) &&
6621 getDomainMap()->isSameAs(*getRowMap())) {
6622 // We can use the rowTransfer + SourceGraph's Import to find out who owns what.
6623 ivector_type TargetRow_pids(domainMap);
6624 ivector_type SourceRow_pids(getRowMap());
6625 ivector_type SourceCol_pids(getColMap());
6626
6627 TargetRow_pids.putScalar(MyPID);
6628 if (!reverseMode && xferAsImport != nullptr) {
6629 SourceRow_pids.doExport(TargetRow_pids, *xferAsImport, INSERT);
6630 } else if (reverseMode && xferAsExport != nullptr) {
6631 SourceRow_pids.doExport(TargetRow_pids, *xferAsExport, INSERT);
6632 } else if (!reverseMode && xferAsExport != nullptr) {
6633 SourceRow_pids.doImport(TargetRow_pids, *xferAsExport, INSERT);
6634 } else if (reverseMode && xferAsImport != nullptr) {
6635 SourceRow_pids.doImport(TargetRow_pids, *xferAsImport, INSERT);
6636 } else {
6637 TEUCHOS_TEST_FOR_EXCEPTION(
6638 true, std::logic_error,
6639 prefix << "Should never get here! Please report this bug to a Tpetra developer.");
6640 }
6641 SourceCol_pids.doImport(SourceRow_pids, *MyImporter, INSERT);
6642 SourcePids.resize(getColMap()->getLocalNumElements());
6643 SourceCol_pids.get1dCopy(SourcePids());
6644 } else {
6645 TEUCHOS_TEST_FOR_EXCEPTION(
6646 true, std::invalid_argument,
6647 prefix << "This method only allows either domainMap == getDomainMap(), "
6648 "or (domainMap == rowTransfer.getTargetMap() and getDomainMap() == getRowMap()).");
6649 }
6650
6651 // Tpetra-specific stuff
6652 size_t constantNumPackets = destGraph->constantNumberOfPackets();
6653 if (constantNumPackets == 0) {
6654 destGraph->reallocArraysForNumPacketsPerLid(ExportLIDs.size(),
6655 RemoteLIDs.size());
6656 } else {
6657 // There are a constant number of packets per element. We
6658 // already know (from the number of "remote" (incoming)
6659 // elements) how many incoming elements we expect, so we can
6660 // resize the buffer accordingly.
6661 const size_t rbufLen = RemoteLIDs.size() * constantNumPackets;
6662 destGraph->reallocImportsIfNeeded(rbufLen, false, nullptr);
6663 }
6664
6665 {
6666 // packAndPrepare* methods modify numExportPacketsPerLID_.
6667 destGraph->numExportPacketsPerLID_.modify_host();
6668 Teuchos::ArrayView<size_t> numExportPacketsPerLID =
6669 getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
6670
6671 // Pack & Prepare w/ owning PIDs
6672 packCrsGraphWithOwningPIDs(*this, destGraph->exports_,
6673 numExportPacketsPerLID, ExportLIDs,
6674 SourcePids, constantNumPackets);
6675 }
6676
6677 // Do the exchange of remote data.
6678#ifdef HAVE_TPETRA_MMM_TIMINGS
6679 MM = Teuchos::null;
6680 MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2 + string("Transfer"))));
6681#endif
6682
6683 if (communication_needed) {
6684 if (reverseMode) {
6685 if (constantNumPackets == 0) { // variable number of packets per LID
6686 // Make sure that host has the latest version, since we're
6687 // using the version on host. If host has the latest
6688 // version, syncing to host does nothing.
6689 destGraph->numExportPacketsPerLID_.sync_host();
6690 Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
6691 getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
6692 destGraph->numImportPacketsPerLID_.sync_host();
6693 Teuchos::ArrayView<size_t> numImportPacketsPerLID =
6694 getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
6695
6696 Distor.doReversePostsAndWaits(destGraph->numExportPacketsPerLID_.view_host(), 1,
6697 destGraph->numImportPacketsPerLID_.view_host());
6698 size_t totalImportPackets = 0;
6699 for (Array_size_type i = 0; i < numImportPacketsPerLID.size(); ++i) {
6700 totalImportPackets += numImportPacketsPerLID[i];
6701 }
6702
6703 // Reallocation MUST go before setting the modified flag,
6704 // because it may clear out the flags.
6705 destGraph->reallocImportsIfNeeded(totalImportPackets, false, nullptr);
6706 destGraph->imports_.modify_host();
6707 auto hostImports = destGraph->imports_.view_host();
6708 // This is a legacy host pack/unpack path, so use the host
6709 // version of exports_.
6710 destGraph->exports_.sync_host();
6711 auto hostExports = destGraph->exports_.view_host();
6712 Distor.doReversePostsAndWaits(hostExports,
6713 numExportPacketsPerLID,
6714 hostImports,
6715 numImportPacketsPerLID);
6716 } else { // constant number of packets per LI
6717 destGraph->imports_.modify_host();
6718 auto hostImports = destGraph->imports_.view_host();
6719 // This is a legacy host pack/unpack path, so use the host
6720 // version of exports_.
6721 destGraph->exports_.sync_host();
6722 auto hostExports = destGraph->exports_.view_host();
6723 Distor.doReversePostsAndWaits(hostExports,
6724 constantNumPackets,
6725 hostImports);
6726 }
6727 } else { // forward mode (the default)
6728 if (constantNumPackets == 0) { // variable number of packets per LID
6729 // Make sure that host has the latest version, since we're
6730 // using the version on host. If host has the latest
6731 // version, syncing to host does nothing.
6732 destGraph->numExportPacketsPerLID_.sync_host();
6733 destGraph->numImportPacketsPerLID_.sync_host();
6734 Distor.doPostsAndWaits(destGraph->numExportPacketsPerLID_.view_host(), 1,
6735 destGraph->numImportPacketsPerLID_.view_host());
6736
6737 Teuchos::ArrayView<const size_t> numImportPacketsPerLID =
6738 getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
6739 size_t totalImportPackets = 0;
6740 for (Array_size_type i = 0; i < numImportPacketsPerLID.size(); ++i) {
6741 totalImportPackets += numImportPacketsPerLID[i];
6742 }
6743
6744 // Reallocation MUST go before setting the modified flag,
6745 // because it may clear out the flags.
6746 destGraph->reallocImportsIfNeeded(totalImportPackets, false, nullptr);
6747 destGraph->imports_.modify_host();
6748 auto hostImports = destGraph->imports_.view_host();
6749 // This is a legacy host pack/unpack path, so use the host
6750 // version of exports_.
6751 destGraph->exports_.sync_host();
6752 auto hostExports = destGraph->exports_.view_host();
6753 Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
6754 getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
6755 Distor.doPostsAndWaits(hostExports, numExportPacketsPerLID, hostImports, numImportPacketsPerLID);
6756 } else { // constant number of packets per LID
6757 destGraph->imports_.modify_host();
6758 auto hostImports = destGraph->imports_.view_host();
6759 // This is a legacy host pack/unpack path, so use the host
6760 // version of exports_.
6761 destGraph->exports_.sync_host();
6762 auto hostExports = destGraph->exports_.view_host();
6763 Distor.doPostsAndWaits(hostExports, constantNumPackets, hostImports);
6764 }
6765 }
6766 }
6767
6768 /*********************************************************************/
6769 /**** 3) Copy all of the Same/Permute/Remote data into CSR_arrays ****/
6770 /*********************************************************************/
6771
6772#ifdef HAVE_TPETRA_MMM_TIMINGS
6773 MM = Teuchos::null;
6774 MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2 + string("Unpack-1"))));
6775#endif
6776
6777 // Backwards compatibility measure. We'll use this again below.
6778 destGraph->numImportPacketsPerLID_.sync_host();
6779 Teuchos::ArrayView<const size_t> numImportPacketsPerLID =
6780 getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
6781 destGraph->imports_.sync_host();
6782 Teuchos::ArrayView<const packet_type> hostImports =
6783 getArrayViewFromDualView(destGraph->imports_);
6784 size_t mynnz =
6785 unpackAndCombineWithOwningPIDsCount(*this, RemoteLIDs, hostImports,
6786 numImportPacketsPerLID,
6787 constantNumPackets, INSERT,
6788 NumSameIDs, PermuteToLIDs, PermuteFromLIDs);
6789 size_t N = BaseRowMap->getLocalNumElements();
6790
6791 // Allocations
6792 ArrayRCP<size_t> CSR_rowptr(N + 1);
6793 ArrayRCP<GO> CSR_colind_GID;
6794 ArrayRCP<LO> CSR_colind_LID;
6795 CSR_colind_GID.resize(mynnz);
6796
6797 // If LO and GO are the same, we can reuse memory when
6798 // converting the column indices from global to local indices.
6799 if (typeid(LO) == typeid(GO)) {
6800 CSR_colind_LID = Teuchos::arcp_reinterpret_cast<LO>(CSR_colind_GID);
6801 } else {
6802 CSR_colind_LID.resize(mynnz);
6803 }
6804
6805 // FIXME (mfh 15 May 2014) Why can't we abstract this out as an
6806 // unpackAndCombine method on a "CrsArrays" object? This passing
6807 // in a huge list of arrays is icky. Can't we have a bit of an
6808 // abstraction? Implementing a concrete DistObject subclass only
6809 // takes five methods.
6810 unpackAndCombineIntoCrsArrays(*this, RemoteLIDs, hostImports,
6811 numImportPacketsPerLID, constantNumPackets,
6812 INSERT, NumSameIDs, PermuteToLIDs,
6813 PermuteFromLIDs, N, mynnz, MyPID,
6814 CSR_rowptr(), CSR_colind_GID(),
6815 SourcePids(), TargetPids);
6816
6817 /**************************************************************/
6818 /**** 4) Call Optimized MakeColMap w/ no Directory Lookups ****/
6819 /**************************************************************/
6820#ifdef HAVE_TPETRA_MMM_TIMINGS
6821 MM = Teuchos::null;
6822 MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2 + string("Unpack-2"))));
6823#endif
6824 // Call an optimized version of makeColMap that avoids the
6825 // Directory lookups (since the Import object knows who owns all
6826 // the GIDs).
6827 Teuchos::Array<int> RemotePids;
6828 Import_Util::lowCommunicationMakeColMapAndReindex(CSR_rowptr(),
6829 CSR_colind_LID(),
6830 CSR_colind_GID(),
6831 BaseDomainMap,
6832 TargetPids, RemotePids,
6833 MyColMap);
6834
6835 /*******************************************************/
6836 /**** 4) Second communicator restriction phase ****/
6837 /*******************************************************/
6838 if (restrictComm) {
6839 ReducedColMap = (MyRowMap.getRawPtr() == MyColMap.getRawPtr()) ? ReducedRowMap : MyColMap->replaceCommWithSubset(ReducedComm);
6840 MyColMap = ReducedColMap; // Reset the "my" maps
6841 }
6842
6843 // Replace the col map
6844 destGraph->replaceColMap(MyColMap);
6845
6846 // Short circuit if the processor is no longer in the communicator
6847 //
6848 // NOTE: Epetra replaces modifies all "removed" processes so they
6849 // have a dummy (serial) Map that doesn't touch the original
6850 // communicator. Duplicating that here might be a good idea.
6851 if (ReducedComm.is_null()) {
6852 return;
6853 }
6854
6855 /***************************************************/
6856 /**** 5) Sort ****/
6857 /***************************************************/
6858 if ((!reverseMode && xferAsImport != nullptr) ||
6859 (reverseMode && xferAsExport != nullptr)) {
6860 Import_Util::sortCrsEntries(CSR_rowptr(),
6861 CSR_colind_LID());
6862 } else if ((!reverseMode && xferAsExport != nullptr) ||
6863 (reverseMode && xferAsImport != nullptr)) {
6864 Import_Util::sortAndMergeCrsEntries(CSR_rowptr(),
6865 CSR_colind_LID());
6866 if (CSR_rowptr[N] != mynnz) {
6867 CSR_colind_LID.resize(CSR_rowptr[N]);
6868 }
6869 } else {
6870 TEUCHOS_TEST_FOR_EXCEPTION(
6871 true, std::logic_error,
6872 prefix << "Should never get here! Please report this bug to a Tpetra developer.");
6873 }
6874 /***************************************************/
6875 /**** 6) Reset the colmap and the arrays ****/
6876 /***************************************************/
6877
6878 // Call constructor for the new graph (restricted as needed)
6879 //
6880 destGraph->setAllIndices(CSR_rowptr, CSR_colind_LID);
6881
6882 /***************************************************/
6883 /**** 7) Build Importer & Call ESFC ****/
6884 /***************************************************/
6885 // Pre-build the importer using the existing PIDs
6886 Teuchos::ParameterList esfc_params;
6887#ifdef HAVE_TPETRA_MMM_TIMINGS
6888 MM = Teuchos::null;
6889 MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2 + string("CreateImporter"))));
6890#endif
6891 RCP<import_type> MyImport = rcp(new import_type(MyDomainMap, MyColMap, RemotePids));
6892#ifdef HAVE_TPETRA_MMM_TIMINGS
6893 MM = Teuchos::null;
6894 MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2 + string("ESFC"))));
6895
6896 esfc_params.set("Timer Label", prefix + std::string("TAFC"));
6897#endif
6898 if (!params.is_null())
6899 esfc_params.set("compute global constants", params->get("compute global constants", true));
6900
6901 destGraph->expertStaticFillComplete(MyDomainMap, MyRangeMap,
6902 MyImport, Teuchos::null, rcp(&esfc_params, false));
6903}
6904
6905template <class LocalOrdinal, class GlobalOrdinal, class Node>
6906void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6907 importAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6908 const import_type& importer,
6909 const Teuchos::RCP<const map_type>& domainMap,
6910 const Teuchos::RCP<const map_type>& rangeMap,
6911 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6912 transferAndFillComplete(destGraph, importer, Teuchos::null, domainMap, rangeMap, params);
6913}
6914
6915template <class LocalOrdinal, class GlobalOrdinal, class Node>
6916void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6917 importAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6918 const import_type& rowImporter,
6919 const import_type& domainImporter,
6920 const Teuchos::RCP<const map_type>& domainMap,
6921 const Teuchos::RCP<const map_type>& rangeMap,
6922 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6923 transferAndFillComplete(destGraph, rowImporter, Teuchos::rcpFromRef(domainImporter), domainMap, rangeMap, params);
6924}
6925
6926template <class LocalOrdinal, class GlobalOrdinal, class Node>
6927void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6928 exportAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6929 const export_type& exporter,
6930 const Teuchos::RCP<const map_type>& domainMap,
6931 const Teuchos::RCP<const map_type>& rangeMap,
6932 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6933 transferAndFillComplete(destGraph, exporter, Teuchos::null, domainMap, rangeMap, params);
6934}
6935
6936template <class LocalOrdinal, class GlobalOrdinal, class Node>
6937void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6938 exportAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6939 const export_type& rowExporter,
6940 const export_type& domainExporter,
6941 const Teuchos::RCP<const map_type>& domainMap,
6942 const Teuchos::RCP<const map_type>& rangeMap,
6943 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6944 transferAndFillComplete(destGraph, rowExporter, Teuchos::rcpFromRef(domainExporter), domainMap, rangeMap, params);
6945}
6946
6947template <class LocalOrdinal, class GlobalOrdinal, class Node>
6948void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6949 swap(CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& graph) {
6950 std::swap(graph.need_sync_host_uvm_access, this->need_sync_host_uvm_access);
6951
6952 std::swap(graph.rowMap_, this->rowMap_);
6953 std::swap(graph.colMap_, this->colMap_);
6954 std::swap(graph.rangeMap_, this->rangeMap_);
6955 std::swap(graph.domainMap_, this->domainMap_);
6956
6957 std::swap(graph.importer_, this->importer_);
6958 std::swap(graph.exporter_, this->exporter_);
6959
6960 std::swap(graph.nodeMaxNumRowEntries_, this->nodeMaxNumRowEntries_);
6961
6962 std::swap(graph.globalNumEntries_, this->globalNumEntries_);
6963 std::swap(graph.globalMaxNumRowEntries_, this->globalMaxNumRowEntries_);
6964
6965 std::swap(graph.numAllocForAllRows_, this->numAllocForAllRows_);
6966
6967 std::swap(graph.rowPtrsPacked_dev_, this->rowPtrsPacked_dev_);
6968 std::swap(graph.rowPtrsPacked_host_, this->rowPtrsPacked_host_);
6969
6970 std::swap(graph.rowPtrsUnpacked_dev_, this->rowPtrsUnpacked_dev_);
6971 std::swap(graph.rowPtrsUnpacked_host_, this->rowPtrsUnpacked_host_);
6972 std::swap(graph.packedUnpackedRowPtrsMatch_, this->packedUnpackedRowPtrsMatch_);
6973
6974 std::swap(graph.k_offRankOffsets_, this->k_offRankOffsets_);
6975
6976 std::swap(graph.lclIndsUnpacked_wdv, this->lclIndsUnpacked_wdv);
6977 std::swap(graph.gblInds_wdv, this->gblInds_wdv);
6978 std::swap(graph.lclIndsPacked_wdv, this->lclIndsPacked_wdv);
6979
6980 std::swap(graph.storageStatus_, this->storageStatus_);
6981
6982 std::swap(graph.indicesAreAllocated_, this->indicesAreAllocated_);
6983 std::swap(graph.indicesAreLocal_, this->indicesAreLocal_);
6984 std::swap(graph.indicesAreGlobal_, this->indicesAreGlobal_);
6985 std::swap(graph.fillComplete_, this->fillComplete_);
6986 std::swap(graph.indicesAreSorted_, this->indicesAreSorted_);
6987 std::swap(graph.noRedundancies_, this->noRedundancies_);
6988 std::swap(graph.haveLocalConstants_, this->haveLocalConstants_);
6989 std::swap(graph.haveGlobalConstants_, this->haveGlobalConstants_);
6990 std::swap(graph.haveLocalOffRankOffsets_, this->haveLocalOffRankOffsets_);
6991
6992 std::swap(graph.sortGhostsAssociatedWithEachProcessor_, this->sortGhostsAssociatedWithEachProcessor_);
6993
6994 std::swap(graph.k_numAllocPerRow_, this->k_numAllocPerRow_); // View
6995 std::swap(graph.k_numRowEntries_, this->k_numRowEntries_); // View
6996 std::swap(graph.nonlocals_, this->nonlocals_); // std::map
6997}
6998
6999template <class LocalOrdinal, class GlobalOrdinal, class Node>
7000bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
7001 isIdenticalTo(const CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& graph) const {
7002 auto compare_nonlocals = [&](const nonlocals_type& m1, const nonlocals_type& m2) {
7003 bool output = true;
7004 output = m1.size() == m2.size() ? output : false;
7005 for (auto& it_m : m1) {
7006 size_t key = it_m.first;
7007 output = m2.find(key) != m2.end() ? output : false;
7008 if (output) {
7009 auto v1 = m1.find(key)->second;
7010 auto v2 = m2.find(key)->second;
7011 std::sort(v1.begin(), v1.end());
7012 std::sort(v2.begin(), v2.end());
7013
7014 output = v1.size() == v2.size() ? output : false;
7015 for (size_t i = 0; output && i < v1.size(); i++) {
7016 output = v1[i] == v2[i] ? output : false;
7017 }
7018 }
7019 }
7020 return output;
7021 };
7022
7023 bool output = true;
7024
7025 output = this->rowMap_->isSameAs(*(graph.rowMap_)) ? output : false;
7026 output = this->colMap_->isSameAs(*(graph.colMap_)) ? output : false;
7027 output = this->rangeMap_->isSameAs(*(graph.rangeMap_)) ? output : false;
7028 output = this->domainMap_->isSameAs(*(graph.domainMap_)) ? output : false;
7029
7030 output = this->nodeMaxNumRowEntries_ == graph.nodeMaxNumRowEntries_ ? output : false;
7031
7032 output = this->globalNumEntries_ == graph.globalNumEntries_ ? output : false;
7033 output = this->globalMaxNumRowEntries_ == graph.globalMaxNumRowEntries_ ? output : false;
7034
7035 output = this->numAllocForAllRows_ == graph.numAllocForAllRows_ ? output : false;
7036
7037 output = this->storageStatus_ == graph.storageStatus_ ? output : false; // EStorageStatus is an enum
7038
7039 output = this->indicesAreAllocated_ == graph.indicesAreAllocated_ ? output : false;
7040 output = this->indicesAreLocal_ == graph.indicesAreLocal_ ? output : false;
7041 output = this->indicesAreGlobal_ == graph.indicesAreGlobal_ ? output : false;
7042 output = this->fillComplete_ == graph.fillComplete_ ? output : false;
7043 output = this->indicesAreSorted_ == graph.indicesAreSorted_ ? output : false;
7044 output = this->noRedundancies_ == graph.noRedundancies_ ? output : false;
7045 output = this->haveLocalConstants_ == graph.haveLocalConstants_ ? output : false;
7046 output = this->haveGlobalConstants_ == graph.haveGlobalConstants_ ? output : false;
7047 output = this->haveLocalOffRankOffsets_ == graph.haveLocalOffRankOffsets_ ? output : false;
7048 output = this->sortGhostsAssociatedWithEachProcessor_ == graph.sortGhostsAssociatedWithEachProcessor_ ? output : false;
7049
7050 // Compare nonlocals_ -- std::map<GlobalOrdinal, std::vector<GlobalOrdinal> >
7051 // nonlocals_ isa std::map<GO, std::vector<GO> >
7052 output = compare_nonlocals(this->nonlocals_, graph.nonlocals_) ? output : false;
7053
7054 // Compare k_numAllocPerRow_ isa Kokkos::View::host_mirror_type
7055 // - since this is a host_mirror_type type, it should be in host memory already
7056 output = this->k_numAllocPerRow_.extent(0) == graph.k_numAllocPerRow_.extent(0) ? output : false;
7057 if (output && this->k_numAllocPerRow_.extent(0) > 0) {
7058 for (size_t i = 0; output && i < this->k_numAllocPerRow_.extent(0); i++)
7059 output = this->k_numAllocPerRow_(i) == graph.k_numAllocPerRow_(i) ? output : false;
7060 }
7061
7062 // Compare k_numRowEntries_ isa Kokkos::View::host_mirror_type
7063 // - since this is a host_mirror_type type, it should be in host memory already
7064 output = this->k_numRowEntries_.extent(0) == graph.k_numRowEntries_.extent(0) ? output : false;
7065 if (output && this->k_numRowEntries_.extent(0) > 0) {
7066 for (size_t i = 0; output && i < this->k_numRowEntries_.extent(0); i++)
7067 output = this->k_numRowEntries_(i) == graph.k_numRowEntries_(i) ? output : false;
7068 }
7069
7070 // Compare this->k_rowPtrs_ isa Kokkos::View<LocalOrdinal*, ...>
7071 {
7072 auto rowPtrsThis = this->getRowPtrsUnpackedHost();
7073 auto rowPtrsGraph = graph.getRowPtrsUnpackedHost();
7074 output = rowPtrsThis.extent(0) == rowPtrsGraph.extent(0) ? output : false;
7075 for (size_t i = 0; output && i < rowPtrsThis.extent(0); i++)
7076 output = rowPtrsThis(i) == rowPtrsGraph(i) ? output : false;
7077 }
7078
7079 // Compare lclIndsUnpacked_wdv isa Kokkos::View<LocalOrdinal*, ...>
7080 output = this->lclIndsUnpacked_wdv.extent(0) == graph.lclIndsUnpacked_wdv.extent(0) ? output : false;
7081 if (output && this->lclIndsUnpacked_wdv.extent(0) > 0) {
7082 auto indThis = this->lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
7083 auto indGraph = graph.lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
7084 for (size_t i = 0; output && i < indThis.extent(0); i++)
7085 output = indThis(i) == indGraph(i) ? output : false;
7086 }
7087
7088 // Compare gblInds_wdv isa Kokkos::View<GlobalOrdinal*, ...>
7089 output = this->gblInds_wdv.extent(0) == graph.gblInds_wdv.extent(0) ? output : false;
7090 if (output && this->gblInds_wdv.extent(0) > 0) {
7091 auto indtThis = this->gblInds_wdv.getHostView(Access::ReadOnly);
7092 auto indtGraph = graph.gblInds_wdv.getHostView(Access::ReadOnly);
7093 for (size_t i = 0; output && i < indtThis.extent(0); i++)
7094 output = indtThis(i) == indtGraph(i) ? output : false;
7095 }
7096
7097 // Check lclGraph_ isa
7098 // KokkosSparse::StaticCrsGraph<LocalOrdinal, Kokkos::LayoutLeft, execution_space>
7099 // KokkosSparse::StaticCrsGraph has 3 data members in it:
7100 // Kokkos::View<size_type*, ...> row_map
7101 // (local_graph_device_type::row_map_type)
7102 // Kokkos::View<data_type*, ...> entries
7103 // (local_graph_device_type::entries_type)
7104 // Kokkos::View<size_type*, ...> row_block_offsets
7105 // (local_graph_device_type::row_block_type)
7106 // There is currently no KokkosSparse::StaticCrsGraph comparison function
7107 // that's built-in, so we will just compare
7108 // the three data items here. This can be replaced if Kokkos ever
7109 // puts in its own comparison routine.
7110 local_graph_host_type thisLclGraph = this->getLocalGraphHost();
7111 local_graph_host_type graphLclGraph = graph.getLocalGraphHost();
7112
7113 output = thisLclGraph.row_map.extent(0) == graphLclGraph.row_map.extent(0)
7114 ? output
7115 : false;
7116 if (output && thisLclGraph.row_map.extent(0) > 0) {
7117 auto lclGraph_rowmap_host_this = thisLclGraph.row_map;
7118 auto lclGraph_rowmap_host_graph = graphLclGraph.row_map;
7119 for (size_t i = 0; output && i < lclGraph_rowmap_host_this.extent(0); i++)
7120 output = lclGraph_rowmap_host_this(i) == lclGraph_rowmap_host_graph(i)
7121 ? output
7122 : false;
7123 }
7124
7125 output = thisLclGraph.entries.extent(0) == graphLclGraph.entries.extent(0)
7126 ? output
7127 : false;
7128 if (output && thisLclGraph.entries.extent(0) > 0) {
7129 auto lclGraph_entries_host_this = thisLclGraph.entries;
7130 auto lclGraph_entries_host_graph = graphLclGraph.entries;
7131 for (size_t i = 0; output && i < lclGraph_entries_host_this.extent(0); i++)
7132 output = lclGraph_entries_host_this(i) == lclGraph_entries_host_graph(i)
7133 ? output
7134 : false;
7135 }
7136
7137 output =
7138 thisLclGraph.row_block_offsets.extent(0) ==
7139 graphLclGraph.row_block_offsets.extent(0)
7140 ? output
7141 : false;
7142 if (output && thisLclGraph.row_block_offsets.extent(0) > 0) {
7143 auto lclGraph_rbo_host_this = thisLclGraph.row_block_offsets;
7144 auto lclGraph_rbo_host_graph = graphLclGraph.row_block_offsets;
7145 for (size_t i = 0; output && i < lclGraph_rbo_host_this.extent(0); i++)
7146 output = lclGraph_rbo_host_this(i) == lclGraph_rbo_host_graph(i)
7147 ? output
7148 : false;
7149 }
7150
7151 // For Importer and Exporter, we don't need to explicitly check them since
7152 // they will be consistent with the maps.
7153 // Note: importer_ isa Teuchos::RCP<const import_type>
7154 // exporter_ isa Teuchos::RCP<const export_type>
7155
7156 return output;
7157}
7158
7159template <class LocalOrdinal, class GlobalOrdinal, class Node>
7160void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::insertGlobalIndicesDevice(
7161 const CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& srcCrsGraph,
7162 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& tgtCrsGraph,
7163 const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteToLIDs,
7164 const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteFromLIDs,
7165 LocalOrdinal loopEnd) {
7166 using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
7167 using LO = LocalOrdinal;
7168 using GO = GlobalOrdinal;
7169 typedef typename crs_graph_type::global_inds_device_view_type::non_const_value_type global_inds_device_value_t;
7170 typedef typename Node::execution_space exec_space;
7171 typedef Kokkos::RangePolicy<exec_space, LO> range_type;
7172
7173 const LocalOrdinal LINV = Teuchos::OrdinalTraits<LocalOrdinal>::invalid();
7174 const GlobalOrdinal GINV = Teuchos::OrdinalTraits<GlobalOrdinal>::invalid();
7175
7176 using local_map_type = typename crs_graph_type::map_type::local_map_type;
7177 local_map_type srcRowMapLocal = srcCrsGraph.getRowMap()->getLocalMap();
7178 local_map_type srcColMapLocal = srcCrsGraph.getColMap()->getLocalMap();
7179 local_map_type tgtRowMapLocal = tgtCrsGraph.getRowMap()->getLocalMap();
7180
7181 auto tgtLocalRowPtrsDevice = tgtCrsGraph.getRowPtrsUnpackedDevice();
7182 auto tgtGlobalColInds = tgtCrsGraph.gblInds_wdv.getDeviceView(Access::ReadWrite);
7183 auto srcLocalRowPtrsDevice = srcCrsGraph.getLocalRowPtrsDevice();
7184 auto srcLocalColIndsDevice = srcCrsGraph.lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly);
7185
7186 typename crs_graph_type::num_row_entries_type::non_const_type h_numRowEnt = tgtCrsGraph.k_numRowEntries_;
7187
7188 auto k_numRowEnt = Kokkos::create_mirror_view_and_copy(device_type(), h_numRowEnt);
7189
7190 const bool sorted = false;
7191
7192 bool hasMap = permuteFromLIDs.extent(0) > 0;
7193 auto permuteToLIDs_d = permuteToLIDs.view_device();
7194 auto permuteFromLIDs_d = permuteFromLIDs.view_device();
7195
7196#ifdef CRSGRAPH_INNER_ABORT
7197#undef CRSGRAPH_INNER_ABORT
7198#endif
7199
7200#ifdef KOKKOS_ENABLE_SYCL
7201#define CRSGRAPH_INNER_ABORT(lin) \
7202 do { \
7203 sycl::ext::oneapi::experimental::printf("ERROR: Tpetra_CrsGraph_def.hpp:%d", lin); \
7204 Kokkos::abort("error"); \
7205 } while (0)
7206#else
7207#define CRSGRAPH_INNER_ABORT(lin) \
7208 do { \
7209 printf("ERROR: Tpetra_CrsGraph_def.hpp:%d", lin); \
7210 Kokkos::abort("error"); \
7211 } while (0)
7212#endif
7213
7214 Kokkos::parallel_for(
7215 "Tpetra_CrsGraph::copyAndPermuteNew",
7216 range_type(0, loopEnd),
7217 KOKKOS_LAMBDA(const LO sourceLID) {
7218 auto srcLid = sourceLID;
7219 auto tgtLid = sourceLID;
7220 if (hasMap) {
7221 srcLid = permuteFromLIDs_d(srcLid);
7222 tgtLid = permuteToLIDs_d(tgtLid);
7223 }
7224 auto srcGid = srcRowMapLocal.getGlobalElement(srcLid);
7225 if (srcGid == GINV) CRSGRAPH_INNER_ABORT(__LINE__);
7226 auto tgtGid = tgtRowMapLocal.getGlobalElement(tgtLid);
7227 auto tgtLocalRow = tgtRowMapLocal.getLocalElement(tgtGid);
7228 if (tgtLocalRow == LINV) CRSGRAPH_INNER_ABORT(__LINE__);
7229 if (tgtLocalRow != tgtLid) CRSGRAPH_INNER_ABORT(__LINE__);
7230 auto tgtNumEntries = k_numRowEnt(tgtLocalRow);
7231
7232 // FIXME no auto use
7233 auto start = srcLocalRowPtrsDevice(srcLid);
7234 auto end = srcLocalRowPtrsDevice(srcLid + 1);
7235 auto rowLength = (end - start);
7236
7237 auto tstart = tgtLocalRowPtrsDevice(tgtLocalRow);
7238 auto tend = tstart + tgtNumEntries;
7239 auto tend1 = tgtLocalRowPtrsDevice(tgtLocalRow + 1);
7240
7241 const size_t num_avail = (tend1 < tend) ? size_t(0) : tend1 - tend;
7242 size_t num_inserted = 0;
7243
7244 global_inds_device_value_t* tgtGlobalColIndsPtr = tgtGlobalColInds.data();
7245
7246 size_t hint = 0;
7247 for (size_t j = 0; j < rowLength; j++) {
7248 auto ci = srcLocalColIndsDevice(start + j);
7249 GO gi = srcColMapLocal.getGlobalElement(ci);
7250 if (gi == GINV) CRSGRAPH_INNER_ABORT(__LINE__);
7251 auto numInTgtRow = (tend - tstart);
7252
7253 const size_t offset = KokkosSparse::findRelOffset(
7254 tgtGlobalColIndsPtr + tstart, numInTgtRow, gi, hint, sorted);
7255
7256 if (offset == numInTgtRow) {
7257 if (num_inserted >= num_avail) { // not enough room
7258 Kokkos::abort("num_avail");
7259 }
7260 tgtGlobalColIndsPtr[tstart + offset] = gi;
7261 ++tend;
7262 hint = offset + 1;
7263 ++num_inserted;
7264 }
7265 }
7266 k_numRowEnt(tgtLocalRow) += num_inserted;
7267 return size_t(0);
7268 });
7269 Kokkos::deep_copy(tgtCrsGraph.k_numRowEntries_, k_numRowEnt);
7270 tgtCrsGraph.setLocallyModified();
7271}
7272
7273template <class LocalOrdinal, class GlobalOrdinal, class Node>
7274void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::copyAndPermuteNew(
7275 const row_graph_type& srcRowGraph,
7276 row_graph_type& tgtRowGraph,
7277 const size_t numSameIDs,
7278 const Kokkos::DualView<const local_ordinal_type*,
7279 buffer_device_type>& permuteToLIDs,
7280 const Kokkos::DualView<const local_ordinal_type*,
7281 buffer_device_type>& permuteFromLIDs,
7282 const CombineMode CM) {
7283 using std::endl;
7284 using LO = local_ordinal_type;
7285 using GO = global_ordinal_type;
7286 const char tfecfFuncName[] = "copyAndPermuteNew: ";
7287 const bool verbose = verbose_;
7288
7289 Details::ProfilingRegion regionCAP("Tpetra::CrsGraph::copyAndPermuteNew");
7290 std::unique_ptr<std::string> prefix;
7291 if (verbose) {
7292 prefix = this->createPrefix("CrsGraph", "copyAndPermuteNew");
7293 std::ostringstream os;
7294 os << *prefix << endl;
7295 std::cerr << os.str();
7296 }
7297
7298 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
7299 permuteToLIDs.extent(0) != permuteFromLIDs.extent(0),
7300 std::runtime_error,
7301 "permuteToLIDs.extent(0) = " << permuteToLIDs.extent(0) << " != permuteFromLIDs.extent(0) = " << permuteFromLIDs.extent(0) << ".");
7302
7303 if (verbose) {
7304 std::ostringstream os;
7305 os << *prefix << "Compute padding" << endl;
7306 std::cerr << os.str();
7307 }
7308
7309 using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
7310 const crs_graph_type* srcCrsGraphPtr = dynamic_cast<const crs_graph_type*>(&srcRowGraph);
7311 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
7312 !srcCrsGraphPtr, std::runtime_error, "error srcGraph type= " << typeid(srcRowGraph).name());
7313 const crs_graph_type& srcCrsGraph = *srcCrsGraphPtr;
7314
7315 crs_graph_type* tgtCrsGraphPtr = dynamic_cast<crs_graph_type*>(&tgtRowGraph);
7316 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
7317 !tgtCrsGraphPtr, std::runtime_error, "error tgtGraph type= " << typeid(tgtRowGraph).name());
7318
7319 crs_graph_type& tgtCrsGraph = *tgtCrsGraphPtr;
7320 auto padding = tgtCrsGraph.computeCrsPadding(
7321 srcRowGraph, numSameIDs, permuteToLIDs, permuteFromLIDs, verbose);
7322 tgtCrsGraph.applyCrsPadding(*padding, verbose);
7323
7324 const map_type& srcRowMap = *(srcRowGraph.getRowMap());
7325 const map_type& tgtRowMap = *(tgtRowGraph.getRowMap());
7326 const bool src_filled = srcRowGraph.isFillComplete();
7327 nonconst_global_inds_host_view_type row_copy;
7328 LO myid = 0;
7329
7330 //
7331 // "Copy" part of "copy and permute."
7332 //
7333 LO numSameIDs_as_LID = static_cast<LO>(numSameIDs);
7334
7335 if (src_filled || srcCrsGraphPtr == nullptr) {
7336 if (verbose) {
7337 std::ostringstream os;
7338 os << *prefix << "src_filled || srcCrsGraph == nullptr" << endl;
7339 std::cerr << os.str();
7340 }
7341 // If the source graph is fill complete, we can't use view mode,
7342 // because the data might be stored in a different format not
7343 // compatible with the expectations of view mode. Also, if the
7344 // source graph is not a CrsGraph, we can't use view mode,
7345 // because RowGraph only provides copy mode access to the data.
7346 Kokkos::DualView<const local_ordinal_type*, buffer_device_type> noPermute;
7347 insertGlobalIndicesDevice(srcCrsGraph, tgtCrsGraph,
7348 noPermute, noPermute,
7349 numSameIDs_as_LID);
7350 } else {
7351 if (verbose) {
7352 std::ostringstream os;
7353 os << *prefix << "! src_filled && srcCrsGraph != nullptr" << endl;
7354 std::cerr << os.str();
7355 }
7356 for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
7357 const GO gid = srcRowMap.getGlobalElement(myid);
7358 global_inds_host_view_type row;
7359 srcCrsGraph.getGlobalRowView(gid, row);
7360 tgtCrsGraph.insertGlobalIndices(gid, row.extent(0), row.data());
7361 }
7362 }
7363
7364 //
7365 // "Permute" part of "copy and permute."
7366 //
7367 auto permuteToLIDs_h = permuteToLIDs.view_host();
7368 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
7369 auto permuteToLIDs_d = permuteToLIDs.view_device();
7370 auto permuteFromLIDs_d = permuteFromLIDs.view_device();
7371
7372 if (src_filled || srcCrsGraphPtr == nullptr) {
7373 insertGlobalIndicesDevice(
7374 srcCrsGraph,
7375 tgtCrsGraph,
7376 permuteToLIDs,
7377 permuteFromLIDs, // note reversed arg order, tgt, then src
7378 static_cast<LO>(permuteToLIDs_h.extent(0)));
7379 } else {
7380 for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
7381 const GO mygid = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
7382 const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
7383 global_inds_host_view_type row;
7384 srcCrsGraph.getGlobalRowView(srcgid, row);
7385 tgtCrsGraph.insertGlobalIndices(mygid, row.extent(0), row.data());
7386 }
7387 }
7388
7389 if (verbose) {
7390 std::ostringstream os;
7391 os << *prefix << "Done" << endl;
7392 std::cerr << os.str();
7393 }
7394}
7395
7396} // namespace Tpetra
7397
7398//
7399// Explicit instantiation macros
7400//
7401// Must be expanded from within the Tpetra namespace!
7402//
7403
7404#define TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7405 template <> \
7406 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7407 importAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7408 const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7409 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7410 CrsGraph<LO, GO, NODE>::node_type>& importer, \
7411 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7412 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7413 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7414 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7415 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7416 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7417 const Teuchos::RCP<Teuchos::ParameterList>& params);
7418
7419#define TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE) \
7420 template <> \
7421 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7422 importAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7423 const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7424 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7425 CrsGraph<LO, GO, NODE>::node_type>& rowImporter, \
7426 const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7427 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7428 CrsGraph<LO, GO, NODE>::node_type>& domainImporter, \
7429 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7430 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7431 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7432 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7433 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7434 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7435 const Teuchos::RCP<Teuchos::ParameterList>& params);
7436
7437#define TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7438 template <> \
7439 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7440 exportAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7441 const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7442 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7443 CrsGraph<LO, GO, NODE>::node_type>& exporter, \
7444 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7445 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7446 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7447 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7448 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7449 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7450 const Teuchos::RCP<Teuchos::ParameterList>& params);
7451
7452#define TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE) \
7453 template <> \
7454 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7455 exportAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7456 const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7457 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7458 CrsGraph<LO, GO, NODE>::node_type>& rowExporter, \
7459 const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7460 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7461 CrsGraph<LO, GO, NODE>::node_type>& domainExporter, \
7462 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7463 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7464 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7465 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7466 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7467 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7468 const Teuchos::RCP<Teuchos::ParameterList>& params);
7469
7470#define TPETRA_CRSGRAPH_INSTANT(LO, GO, NODE) \
7471 template class CrsGraph<LO, GO, NODE>; \
7472 TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7473 TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7474 TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE) \
7475 TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE)
7476
7477#endif // TPETRA_CRSGRAPH_DEF_HPP
Tpetra_Details_Behavior.hpp
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Tpetra_Details_Profiling.hpp
Declaration of Tpetra::Details::Profiling, a scope guard for Kokkos Profiling.
Tpetra_Details_computeOffsets.hpp
Declare and define the functions Tpetra::Details::computeOffsetsFromCounts and Tpetra::computeOffsets...
Tpetra_Details_copyOffsets.hpp
Declare and define Tpetra::Details::copyOffsets, an implementation detail of Tpetra (in particular,...
Tpetra_Details_crsUtils.hpp
Functions for manipulating CRS arrays.
Tpetra_Details_gathervPrint.hpp
Declaration of a function that prints strings from each process.
Tpetra_Details_getEntryOnHost.hpp
Declaration and definition of Tpetra::Details::getEntryOnHost.
Tpetra_Import_Util2.hpp
Utility functions for packing and unpacking sparse matrix entries.
Tpetra::Import_Util::lowCommunicationMakeColMapAndReindex
void lowCommunicationMakeColMapAndReindex(const Teuchos::ArrayView< const size_t > &rowptr, const Teuchos::ArrayView< LocalOrdinal > &colind_LID, const Teuchos::ArrayView< GlobalOrdinal > &colind_GID, const Teuchos::RCP< const Tpetra::Map< LocalOrdinal, GlobalOrdinal, Node > > &domainMapRCP, const Teuchos::ArrayView< const int > &owningPIDs, Teuchos::Array< int > &remotePIDs, Teuchos::RCP< const Tpetra::Map< LocalOrdinal, GlobalOrdinal, Node > > &colMap)
lowCommunicationMakeColMapAndReindex
Definition Tpetra_Import_Util2.hpp:785
Tpetra::Import_Util::sortAndMergeCrsEntries
void sortAndMergeCrsEntries(const Teuchos::ArrayView< size_t > &CRS_rowptr, const Teuchos::ArrayView< Ordinal > &CRS_colind, const Teuchos::ArrayView< Scalar > &CRS_vals)
Sort and merge the entries of the (raw CSR) matrix by column index within each row.
Definition Tpetra_Import_Util2.hpp:464
Tpetra::Import_Util::sortCrsEntries
void sortCrsEntries(const Teuchos::ArrayView< size_t > &CRS_rowptr, const Teuchos::ArrayView< Ordinal > &CRS_colind, const Teuchos::ArrayView< Scalar > &CRS_vals)
Sort the entries of the (raw CSR) matrix by column index within each row.
Definition Tpetra_Import_Util2.hpp:432
Tpetra_Import_Util.hpp
Internal functions and macros designed for use with Tpetra::Import and Tpetra::Export objects.
Tpetra::Import_Util::getPids
void getPids(const Tpetra::Import< LocalOrdinal, GlobalOrdinal, Node > &Importer, Teuchos::Array< int > &pids, bool use_minus_one_for_local)
Like getPidGidPairs, but just gets the PIDs, ordered by the column Map.
Definition Tpetra_Import_Util.hpp:110
Tpetra_Util.hpp
Stand-alone utility functions and macros.
Tpetra::CrsGraph
A distributed graph accessed by rows (adjacency lists) and stored sparsely.
Definition Tpetra_CrsGraph_decl.hpp:195
Tpetra::CrsGraph::isMerged
bool isMerged() const
Whether duplicate column indices in each row have been merged.
Definition Tpetra_CrsGraph_def.hpp:1058
Tpetra::CrsGraph::unpackAndCombine
virtual void unpackAndCombine(const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &importLIDs, Kokkos::DualView< packet_type *, buffer_device_type > imports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, const size_t constantNumPackets, const CombineMode combineMode) override
Definition Tpetra_CrsGraph_def.hpp:5701
Tpetra::CrsGraph::getLocalIndsViewDevice
local_inds_dualv_type::t_dev::const_type getLocalIndsViewDevice(const RowInfo &rowinfo) const
Get a const, locally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myRo...
Definition Tpetra_CrsGraph_def.hpp:1296
Tpetra::CrsGraph::globalMaxNumRowEntries_
global_size_t globalMaxNumRowEntries_
Global maximum of the number of entries in each row.
Definition Tpetra_CrsGraph_decl.hpp:2141
Tpetra::CrsGraph::reindexColumns
void reindexColumns(const Teuchos::RCP< const map_type > &newColMap, const Teuchos::RCP< const import_type > &newImport=Teuchos::null, const bool sortIndicesInEachRow=true)
Reindex the column indices in place, and replace the column Map. Optionally, replace the Import objec...
Definition Tpetra_CrsGraph_def.hpp:3511
Tpetra::CrsGraph::getGlobalIndsViewHost
global_inds_dualv_type::t_host::const_type getGlobalIndsViewHost(const RowInfo &rowinfo) const
Get a const, globally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myR...
Definition Tpetra_CrsGraph_def.hpp:1283
Tpetra::CrsGraph::getNumEntriesInLocalRow
size_t getNumEntriesInLocalRow(local_ordinal_type localRow) const override
Get the number of entries in the given row (local index).
Definition Tpetra_CrsGraph_def.hpp:2017
Tpetra::CrsGraph::getColMap
Teuchos::RCP< const map_type > getColMap() const override
Returns the Map that describes the column distribution in this graph.
Definition Tpetra_CrsGraph_def.hpp:827
Tpetra::CrsGraph::getValidParameters
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const override
Default parameter list suitable for validation.
Definition Tpetra_CrsGraph_def.hpp:736
Tpetra::CrsGraph::storageStatus_
Details::EStorageStatus storageStatus_
Status of the graph's storage, when not in a fill-complete state.
Definition Tpetra_CrsGraph_decl.hpp:2444
Tpetra::CrsGraph::import_type
::Tpetra::Import< LocalOrdinal, GlobalOrdinal, Node > import_type
The Import specialization used by this class.
Definition Tpetra_CrsGraph_decl.hpp:237
Tpetra::CrsGraph::packet_type
global_ordinal_type packet_type
Type of each entry of the DistObject communication buffer.
Definition Tpetra_CrsGraph_decl.hpp:1171
Tpetra::CrsGraph::global_ordinal_type
GlobalOrdinal global_ordinal_type
The type of the graph's global indices.
Definition Tpetra_CrsGraph_decl.hpp:210
Tpetra::CrsGraph::insertGlobalIndicesIntoNonownedRows
void insertGlobalIndicesIntoNonownedRows(const global_ordinal_type gblRow, const global_ordinal_type gblColInds[], const local_ordinal_type numGblColInds)
Implementation of insertGlobalIndices for nonowned rows.
Definition Tpetra_CrsGraph_def.hpp:2435
Tpetra::CrsGraph::rangeMap_
Teuchos::RCP< const map_type > rangeMap_
The Map describing the range of the (matrix corresponding to the) graph.
Definition Tpetra_CrsGraph_decl.hpp:2105
Tpetra::CrsGraph::makeIndicesLocal
std::pair< size_t, std::string > makeIndicesLocal(const bool verbose=false)
Convert column indices from global to local.
Definition Tpetra_CrsGraph_def.hpp:3945
Tpetra::CrsGraph::getLocalIndicesDevice
local_inds_device_view_type getLocalIndicesDevice() const
Get a device view of the packed column indicies.
Definition Tpetra_CrsGraph_def.hpp:2074
Tpetra::CrsGraph::getGlobalNumEntries
global_size_t getGlobalNumEntries() const override
Returns the global number of entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:881
Tpetra::CrsGraph::isIdenticalTo
bool isIdenticalTo(const CrsGraph< LocalOrdinal, GlobalOrdinal, Node > &graph) const
Create a cloned CrsGraph for a different Node type.
Definition Tpetra_CrsGraph_def.hpp:7001
Tpetra::CrsGraph::getComm
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const override
Returns the communicator.
Definition Tpetra_CrsGraph_def.hpp:1033
Tpetra::CrsGraph::lclIndsUnpacked_wdv
local_inds_wdv_type lclIndsUnpacked_wdv
Local ordinals of column indices for all rows Valid when isLocallyIndexed is true If OptimizedStorage...
Definition Tpetra_CrsGraph_decl.hpp:2269
Tpetra::CrsGraph::globalAssemble
void globalAssemble()
Communicate nonlocal contributions to other processes.
Definition Tpetra_CrsGraph_def.hpp:2644
Tpetra::CrsGraph::getRowInfoFromGlobalRowIndex
RowInfo getRowInfoFromGlobalRowIndex(const global_ordinal_type gblRow) const
Get information about the locally owned row with global index gblRow.
Definition Tpetra_CrsGraph_def.hpp:1361
Tpetra::CrsGraph::getLocalDiagOffsets
void getLocalDiagOffsets(const Kokkos::View< size_t *, device_type, Kokkos::MemoryUnmanaged > &offsets) const
Get offsets of the diagonal entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:5951
Tpetra::CrsGraph::findGlobalIndices
size_t findGlobalIndices(const RowInfo &rowInfo, const Teuchos::ArrayView< const global_ordinal_type > &indices, std::function< void(const size_t, const size_t, const size_t)> fun) const
Finds indices in the given row.
Definition Tpetra_CrsGraph_def.hpp:1690
Tpetra::CrsGraph::fillComplete
void fillComplete(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Tell the graph that you are done changing its structure.
Definition Tpetra_CrsGraph_def.hpp:2895
Tpetra::CrsGraph::gblInds_wdv
global_inds_wdv_type gblInds_wdv
Global ordinals of column indices for all rows.
Definition Tpetra_CrsGraph_decl.hpp:2299
Tpetra::CrsGraph::nodeMaxNumRowEntries_
size_t nodeMaxNumRowEntries_
Local maximum of the number of entries in each row.
Definition Tpetra_CrsGraph_decl.hpp:2128
Tpetra::CrsGraph::local_graph_device_type
KokkosSparse::StaticCrsGraph< local_ordinal_type, Kokkos::LayoutLeft, device_type, void, size_t > local_graph_device_type
The type of the part of the sparse graph on each MPI process.
Definition Tpetra_CrsGraph_decl.hpp:223
Tpetra::CrsGraph::importer_
Teuchos::RCP< const import_type > importer_
The Import from the domain Map to the column Map.
Definition Tpetra_CrsGraph_decl.hpp:2115
Tpetra::CrsGraph::k_numRowEntries_
num_row_entries_type k_numRowEntries_
The number of local entries in each locally owned row.
Definition Tpetra_CrsGraph_decl.hpp:2424
Tpetra::CrsGraph::getRowPtrsUnpackedDevice
const row_ptrs_device_view_type & getRowPtrsUnpackedDevice() const
Get the unpacked row pointers on device.
Definition Tpetra_CrsGraph_decl.hpp:2177
Tpetra::CrsGraph::numAllocForAllRows_
size_t numAllocForAllRows_
The maximum number of entries to allow in each locally owned row.
Definition Tpetra_CrsGraph_decl.hpp:2372
Tpetra::CrsGraph::hasColMap
bool hasColMap() const override
Whether the graph has a column Map.
Definition Tpetra_CrsGraph_def.hpp:861
Tpetra::CrsGraph::local_ordinal_type
LocalOrdinal local_ordinal_type
The type of the graph's local indices.
Definition Tpetra_CrsGraph_decl.hpp:208
Tpetra::CrsGraph::description
std::string description() const override
Return a one-line human-readable description of this object.
Definition Tpetra_CrsGraph_def.hpp:4347
Tpetra::CrsGraph::isStorageOptimized
bool isStorageOptimized() const
Returns true if storage has been optimized.
Definition Tpetra_CrsGraph_def.hpp:867
Tpetra::CrsGraph::getGlobalRowCopy
void getGlobalRowCopy(global_ordinal_type gblRow, nonconst_global_inds_host_view_type &gblColInds, size_t &numColInds) const override
Get a copy of the given row, using global indices.
Definition Tpetra_CrsGraph_def.hpp:2122
Tpetra::CrsGraph::removeLocalIndices
void removeLocalIndices(local_ordinal_type localRow)
Remove all graph indices from the specified local row.
Definition Tpetra_CrsGraph_def.hpp:2452
Tpetra::CrsGraph::importAndFillComplete
void importAndFillComplete(Teuchos::RCP< CrsGraph< local_ordinal_type, global_ordinal_type, Node > > &destGraph, const import_type &importer, const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null) const
Import from this to the given destination graph, and make the result fill complete.
Definition Tpetra_CrsGraph_def.hpp:6907
Tpetra::CrsGraph::getGlobalNumRows
global_size_t getGlobalNumRows() const override
Returns the number of global rows in the graph.
Definition Tpetra_CrsGraph_def.hpp:780
Tpetra::CrsGraph::getDomainMap
Teuchos::RCP< const map_type > getDomainMap() const override
Returns the Map associated with the domain of this graph.
Definition Tpetra_CrsGraph_def.hpp:834
Tpetra::CrsGraph::replaceRangeMapAndExporter
void replaceRangeMapAndExporter(const Teuchos::RCP< const map_type > &newRangeMap, const Teuchos::RCP< const export_type > &newExporter)
Replace the current Range Map and Export with the given parameters.
Definition Tpetra_CrsGraph_def.hpp:3818
Tpetra::CrsGraph::computeLocalConstants
void computeLocalConstants()
Compute local constants, if they have not yet been computed.
Definition Tpetra_CrsGraph_def.hpp:3918
Tpetra::CrsGraph::describe
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const override
Print this object to the given output stream with the given verbosity level.
Definition Tpetra_CrsGraph_def.hpp:4366
Tpetra::CrsGraph::setParameterList
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &params) override
Set the given list of parameters (must be nonnull).
Definition Tpetra_CrsGraph_def.hpp:770
Tpetra::CrsGraph::resumeFill
void resumeFill(const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Resume fill operations.
Definition Tpetra_CrsGraph_def.hpp:2860
Tpetra::CrsGraph::insertIndices
size_t insertIndices(RowInfo &rowInfo, const SLocalGlobalViews &newInds, const ELocalGlobal lg, const ELocalGlobal I)
Insert indices into the given row.
Definition Tpetra_CrsGraph_def.hpp:1454
Tpetra::CrsGraph::device_type
typename Node::device_type device_type
This class' Kokkos device type.
Definition Tpetra_CrsGraph_decl.hpp:212
Tpetra::CrsGraph::insertGlobalIndicesFiltered
void insertGlobalIndicesFiltered(const local_ordinal_type lclRow, const global_ordinal_type gblColInds[], const local_ordinal_type numGblColInds)
Like insertGlobalIndices(), but with column Map filtering.
Definition Tpetra_CrsGraph_def.hpp:2376
Tpetra::CrsGraph::copyAndPermute
virtual void copyAndPermute(const SrcDistObject &source, const size_t numSameIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteToLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteFromLIDs, const CombineMode CM) override
Definition Tpetra_CrsGraph_def.hpp:4494
Tpetra::CrsGraph::getRowInfo
RowInfo getRowInfo(const local_ordinal_type myRow) const
Get information about the locally owned row with local index myRow.
Definition Tpetra_CrsGraph_def.hpp:1321
Tpetra::CrsGraph::getGlobalIndsViewDevice
global_inds_dualv_type::t_dev::const_type getGlobalIndsViewDevice(const RowInfo &rowinfo) const
Get a const, globally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myR...
Definition Tpetra_CrsGraph_def.hpp:1309
Tpetra::CrsGraph::local_graph_host_type
typename local_graph_device_type::HostMirror local_graph_host_type
The type of the part of the sparse graph on each MPI process.
Definition Tpetra_CrsGraph_decl.hpp:231
Tpetra::CrsGraph::colMap_
Teuchos::RCP< const map_type > colMap_
The Map describing the distribution of columns of the graph.
Definition Tpetra_CrsGraph_decl.hpp:2103
Tpetra::CrsGraph::noRedundancies_
bool noRedundancies_
Whether the graph's indices are non-redundant (merged) in each row, on this process.
Definition Tpetra_CrsGraph_decl.hpp:2456
Tpetra::CrsGraph::getLocalRowPtrsHost
row_ptrs_host_view_type getLocalRowPtrsHost() const
Get a host view of the packed row offsets.
Definition Tpetra_CrsGraph_def.hpp:2053
Tpetra::CrsGraph::isSorted
bool isSorted() const
Whether graph indices in all rows are known to be sorted.
Definition Tpetra_CrsGraph_def.hpp:1052
Tpetra::CrsGraph::buffer_device_type
typename dist_object_type::buffer_device_type buffer_device_type
Kokkos::Device specialization for communication buffers.
Definition Tpetra_CrsGraph_decl.hpp:1169
Tpetra::CrsGraph::setAllIndices
void setAllIndices(const typename local_graph_device_type::row_map_type &rowPointers, const typename local_graph_device_type::entries_type::non_const_type &columnIndices)
Set the graph's data directly, using 1-D storage.
Definition Tpetra_CrsGraph_def.hpp:2483
Tpetra::CrsGraph::insertLocalIndices
void insertLocalIndices(const local_ordinal_type localRow, const Teuchos::ArrayView< const local_ordinal_type > &indices)
Insert local indices into the graph.
Definition Tpetra_CrsGraph_def.hpp:2221
Tpetra::CrsGraph::getLocalIndicesHost
local_inds_host_view_type getLocalIndicesHost() const
Get a host view of the packed column indicies.
Definition Tpetra_CrsGraph_def.hpp:2067
Tpetra::CrsGraph::supportsRowViews
bool supportsRowViews() const override
Whether this class implements getLocalRowView() and getGlobalRowView() (it does).
Definition Tpetra_CrsGraph_def.hpp:6299
Tpetra::CrsGraph::getNumEntriesInGlobalRow
size_t getNumEntriesInGlobalRow(global_ordinal_type globalRow) const override
Returns the current number of entries on this node in the specified global row.
Definition Tpetra_CrsGraph_def.hpp:2005
Tpetra::CrsGraph::isFillComplete
bool isFillComplete() const override
Whether fillComplete() has been called and the graph is in compute mode.
Definition Tpetra_CrsGraph_def.hpp:970
Tpetra::CrsGraph::setDomainRangeMaps
void setDomainRangeMaps(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap)
Definition Tpetra_CrsGraph_def.hpp:1733
Tpetra::CrsGraph::swap
void swap(CrsGraph< local_ordinal_type, global_ordinal_type, Node > &graph)
Swaps the data from *this with the data and maps from graph.
Definition Tpetra_CrsGraph_def.hpp:6949
Tpetra::CrsGraph::map_type
::Tpetra::Map< LocalOrdinal, GlobalOrdinal, Node > map_type
The Map specialization used by this class.
Definition Tpetra_CrsGraph_decl.hpp:235
Tpetra::CrsGraph::getGlobalRowView
void getGlobalRowView(const global_ordinal_type gblRow, global_inds_host_view_type &gblColInds) const override
Get a const view of the given global row's global column indices.
Definition Tpetra_CrsGraph_def.hpp:2191
Tpetra::CrsGraph::getRowPtrsUnpackedHost
const row_ptrs_host_view_type & getRowPtrsUnpackedHost() const
Get the unpacked row pointers on host. Lazily make a copy from device.
Definition Tpetra_CrsGraph_decl.hpp:2182
Tpetra::CrsGraph::exportAndFillComplete
void exportAndFillComplete(Teuchos::RCP< CrsGraph< local_ordinal_type, global_ordinal_type, Node > > &destGraph, const export_type &exporter, const Teuchos::RCP< const map_type > &domainMap=Teuchos::null, const Teuchos::RCP< const map_type > &rangeMap=Teuchos::null, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null) const
Export from this to the given destination graph, and make the result fill complete.
Definition Tpetra_CrsGraph_def.hpp:6928
Tpetra::CrsGraph::makeColMap
void makeColMap(Teuchos::Array< int > &remotePIDs)
Make and set the graph's column Map.
Definition Tpetra_CrsGraph_def.hpp:4101
Tpetra::CrsGraph::haveGlobalConstants_
bool haveGlobalConstants_
Whether all processes have computed global constants.
Definition Tpetra_CrsGraph_decl.hpp:2460
Tpetra::CrsGraph::getGlobalMaxNumRowEntries
size_t getGlobalMaxNumRowEntries() const override
Maximum number of entries in any row of the graph, over all processes in the graph's communicator.
Definition Tpetra_CrsGraph_def.hpp:952
Tpetra::CrsGraph::checkInternalState
void checkInternalState() const
Throw an exception if the internal state is not consistent.
Definition Tpetra_CrsGraph_def.hpp:1758
Tpetra::CrsGraph::getRangeMap
Teuchos::RCP< const map_type > getRangeMap() const override
Returns the Map associated with the domain of this graph.
Definition Tpetra_CrsGraph_def.hpp:841
Tpetra::CrsGraph::expertStaticFillComplete
void expertStaticFillComplete(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< const import_type > &importer=Teuchos::null, const Teuchos::RCP< const export_type > &exporter=Teuchos::null, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Perform a fillComplete on a graph that already has data, via setAllIndices().
Definition Tpetra_CrsGraph_def.hpp:3106
Tpetra::CrsGraph::sortGhostsAssociatedWithEachProcessor_
bool sortGhostsAssociatedWithEachProcessor_
Whether to require makeColMap() (and therefore fillComplete()) to order column Map GIDs associated wi...
Definition Tpetra_CrsGraph_decl.hpp:2483
Tpetra::CrsGraph::getNumAllocatedEntriesInGlobalRow
size_t getNumAllocatedEntriesInGlobalRow(global_ordinal_type globalRow) const
Current number of allocated entries in the given row on the calling (MPI) process,...
Definition Tpetra_CrsGraph_def.hpp:2029
Tpetra::CrsGraph::getExporter
Teuchos::RCP< const export_type > getExporter() const override
Returns the exporter associated with this graph.
Definition Tpetra_CrsGraph_def.hpp:855
Tpetra::CrsGraph::makeImportExport
void makeImportExport(Teuchos::Array< int > &remotePIDs, const bool useRemotePIDs)
Make the Import and Export objects, if needed.
Definition Tpetra_CrsGraph_def.hpp:4286
Tpetra::CrsGraph::getIndexBase
global_ordinal_type getIndexBase() const override
Returns the index base for global indices for this graph.
Definition Tpetra_CrsGraph_def.hpp:1040
Tpetra::CrsGraph::getLocalRowPtrsDevice
row_ptrs_device_view_type getLocalRowPtrsDevice() const
Get a device view of the packed row offsets.
Definition Tpetra_CrsGraph_def.hpp:2060
Tpetra::CrsGraph::getLocalRowCopy
void getLocalRowCopy(local_ordinal_type gblRow, nonconst_local_inds_host_view_type &gblColInds, size_t &numColInds) const override
Get a copy of the given row, using local indices.
Definition Tpetra_CrsGraph_def.hpp:2080
Tpetra::CrsGraph::getLocalIndsViewHost
local_inds_dualv_type::t_host::const_type getLocalIndsViewHost(const RowInfo &rowinfo) const
Get a const, locally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myRo...
Definition Tpetra_CrsGraph_def.hpp:1257
Tpetra::CrsGraph::isFillActive
bool isFillActive() const
Whether resumeFill() has been called and the graph is in edit mode.
Definition Tpetra_CrsGraph_def.hpp:976
Tpetra::CrsGraph::getRowMap
Teuchos::RCP< const map_type > getRowMap() const override
Returns the Map that describes the row distribution in this graph.
Definition Tpetra_CrsGraph_def.hpp:820
Tpetra::CrsGraph::globalNumEntries_
global_size_t globalNumEntries_
Global number of entries in the graph.
Definition Tpetra_CrsGraph_decl.hpp:2134
Tpetra::CrsGraph::insertGlobalIndicesImpl
size_t insertGlobalIndicesImpl(const local_ordinal_type lclRow, const global_ordinal_type inputGblColInds[], const size_t numInputInds)
Insert global indices, using an input local row index.
Definition Tpetra_CrsGraph_def.hpp:1551
Tpetra::CrsGraph::export_type
::Tpetra::Export< LocalOrdinal, GlobalOrdinal, Node > export_type
The Export specialization used by this class.
Definition Tpetra_CrsGraph_decl.hpp:239
Tpetra::CrsGraph::getLocalNumEntries
size_t getLocalNumEntries() const override
The local number of entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:893
Tpetra::CrsGraph::getImporter
Teuchos::RCP< const import_type > getImporter() const override
Returns the importer associated with this graph.
Definition Tpetra_CrsGraph_def.hpp:848
Tpetra::CrsGraph::lclIndsPacked_wdv
local_inds_wdv_type lclIndsPacked_wdv
Local ordinals of column indices for all rows Valid when isLocallyIndexed is true Built during fillCo...
Definition Tpetra_CrsGraph_decl.hpp:2284
Tpetra::CrsGraph::domainMap_
Teuchos::RCP< const map_type > domainMap_
The Map describing the domain of the (matrix corresponding to the) graph.
Definition Tpetra_CrsGraph_decl.hpp:2107
Tpetra::CrsGraph::getRowPtrsPackedHost
const row_ptrs_host_view_type & getRowPtrsPackedHost() const
Get the packed row pointers on host. Lazily make a copy from device.
Definition Tpetra_CrsGraph_decl.hpp:2218
Tpetra::CrsGraph::getLocalNumCols
size_t getLocalNumCols() const override
Returns the number of columns connected to the locally owned rows of this graph.
Definition Tpetra_CrsGraph_def.hpp:806
Tpetra::CrsGraph::nonlocals_
nonlocals_type nonlocals_
Nonlocal data given to insertGlobalIndices.
Definition Tpetra_CrsGraph_decl.hpp:2467
Tpetra::CrsGraph::pack
virtual void pack(const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Teuchos::Array< global_ordinal_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t &constantNumPackets) const override
Definition Tpetra_CrsGraph_def.hpp:5274
Tpetra::CrsGraph::getLocalOffRankOffsets
void getLocalOffRankOffsets(offset_device_view_type &offsets) const
Get offsets of the off-rank entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:6137
Tpetra::CrsGraph::getGlobalNumCols
global_size_t getGlobalNumCols() const override
Returns the number of global columns in the graph.
Definition Tpetra_CrsGraph_def.hpp:787
Tpetra::CrsGraph::k_numAllocPerRow_
Kokkos::View< constsize_t *, device_type >::host_mirror_type k_numAllocPerRow_
The maximum number of entries to allow in each locally owned row, per row.
Definition Tpetra_CrsGraph_decl.hpp:2361
Tpetra::CrsGraph::indicesAreSorted_
bool indicesAreSorted_
Whether the graph's indices are sorted in each row, on this process.
Definition Tpetra_CrsGraph_decl.hpp:2453
Tpetra::CrsGraph::node_type
Node node_type
This class' Kokkos Node type.
Definition Tpetra_CrsGraph_decl.hpp:220
Tpetra::CrsGraph::exporter_
Teuchos::RCP< const export_type > exporter_
The Export from the row Map to the range Map.
Definition Tpetra_CrsGraph_decl.hpp:2122
Tpetra::CrsGraph::insertGlobalIndices
void insertGlobalIndices(const global_ordinal_type globalRow, const Teuchos::ArrayView< const global_ordinal_type > &indices)
Insert global indices into the graph.
Definition Tpetra_CrsGraph_def.hpp:2368
Tpetra::CrsGraph::getLocalIndsViewHostNonConst
local_inds_dualv_type::t_host getLocalIndsViewHostNonConst(const RowInfo &rowinfo)
Get a ReadWrite locally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(m...
Definition Tpetra_CrsGraph_def.hpp:1270
Tpetra::CrsGraph::replaceDomainMap
void replaceDomainMap(const Teuchos::RCP< const map_type > &newDomainMap)
Replace the current domain Map with the given objects.
Definition Tpetra_CrsGraph_def.hpp:3741
Tpetra::CrsGraph::computeGlobalConstants
void computeGlobalConstants()
Compute global constants, if they have not yet been computed.
Definition Tpetra_CrsGraph_def.hpp:3875
Tpetra::CrsGraph::getNumAllocatedEntriesInLocalRow
size_t getNumAllocatedEntriesInLocalRow(local_ordinal_type localRow) const
Current number of allocated entries in the given row on the calling (MPI) process,...
Definition Tpetra_CrsGraph_def.hpp:2041
Tpetra::CrsGraph::local_inds_device_view_type
typename row_graph_type::local_inds_device_view_type local_inds_device_view_type
The Kokkos::View type for views of local ordinals on device and host.
Definition Tpetra_CrsGraph_decl.hpp:262
Tpetra::CrsGraph::k_offRankOffsets_
offset_device_view_type k_offRankOffsets_
The offsets for off-rank entries.
Definition Tpetra_CrsGraph_decl.hpp:2431
Tpetra::CrsGraph::replaceDomainMapAndImporter
void replaceDomainMapAndImporter(const Teuchos::RCP< const map_type > &newDomainMap, const Teuchos::RCP< const import_type > &newImporter)
Replace the current domain Map and Import with the given parameters.
Definition Tpetra_CrsGraph_def.hpp:3760
Tpetra::CrsGraph::setLocallyModified
void setLocallyModified()
Report that we made a local modification to its structure.
Definition Tpetra_CrsGraph_def.hpp:1064
Tpetra::CrsGraph::getLocalAllocationSize
size_t getLocalAllocationSize() const
The local number of indices allocated for the graph, over all rows on the calling (MPI) process.
Definition Tpetra_CrsGraph_def.hpp:995
Tpetra::CrsGraph::replaceRangeMap
void replaceRangeMap(const Teuchos::RCP< const map_type > &newRangeMap)
Replace the current Range Map with the given objects.
Definition Tpetra_CrsGraph_def.hpp:3799
Tpetra::CrsGraph::rowMap_
Teuchos::RCP< const map_type > rowMap_
The Map describing the distribution of rows of the graph.
Definition Tpetra_CrsGraph_decl.hpp:2101
Tpetra::CrsGraph::getRowPtrsPackedDevice
const row_ptrs_device_view_type & getRowPtrsPackedDevice() const
Get the packed row pointers on device.
Definition Tpetra_CrsGraph_decl.hpp:2213
Tpetra::CrsGraph::removeEmptyProcessesInPlace
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap) override
Remove processes owning zero rows from the Maps and their communicator.
Definition Tpetra_CrsGraph_def.hpp:5856
Tpetra::CrsGraph::getLocalRowView
void getLocalRowView(const LocalOrdinal lclRow, local_inds_host_view_type &lclColInds) const override
Get a const view of the given local row's local column indices.
Definition Tpetra_CrsGraph_def.hpp:2156
Tpetra::CrsGraph::isGloballyIndexed
bool isGloballyIndexed() const override
Whether the graph's column indices are stored as global indices.
Definition Tpetra_CrsGraph_def.hpp:988
Tpetra::CrsGraph::isLocallyIndexed
bool isLocallyIndexed() const override
Whether the graph's column indices are stored as local indices.
Definition Tpetra_CrsGraph_def.hpp:982
Tpetra::CrsGraph::getLocalMaxNumRowEntries
size_t getLocalMaxNumRowEntries() const override
Maximum number of entries in any row of the graph, on this process.
Definition Tpetra_CrsGraph_def.hpp:964
Tpetra::CrsGraph::checkSizes
virtual bool checkSizes(const SrcDistObject &source) override
Compare the source and target (this) objects for compatibility.
Definition Tpetra_CrsGraph_def.hpp:4485
Tpetra::CrsGraph::getLocalGraphDevice
local_graph_device_type getLocalGraphDevice() const
Get the local graph.
Definition Tpetra_CrsGraph_def.hpp:3858
Tpetra::CrsGraph::getLocalNumRows
size_t getLocalNumRows() const override
Returns the number of graph rows owned on the calling node.
Definition Tpetra_CrsGraph_def.hpp:799
Tpetra::CrsGraph::replaceColMap
void replaceColMap(const Teuchos::RCP< const map_type > &newColMap)
Replace the graph's current column Map with the given Map.
Definition Tpetra_CrsGraph_def.hpp:3494
Tpetra::CrsGraph::haveLocalConstants_
bool haveLocalConstants_
Whether this process has computed local constants.
Definition Tpetra_CrsGraph_decl.hpp:2458
Tpetra::CrsMatrix::getGlobalRowView
void getGlobalRowView(GlobalOrdinal GlobalRow, global_inds_host_view_type &indices, values_host_view_type &values) const override
Get a constant, nonpersisting view of a row of this matrix, using global row and column indices.
Definition Tpetra_CrsMatrix_def.hpp:3226
Tpetra::CrsMatrix::isFillComplete
bool isFillComplete() const override
Whether the matrix is fill complete.
Definition Tpetra_CrsMatrix_def.hpp:781
Tpetra::Details::Behavior::useNewCopyAndPermute
static bool useNewCopyAndPermute()
Use new implementation of copyAndPermute.
Definition Tpetra_Details_Behavior.cpp:549
Tpetra::Details::Behavior::debug
static bool debug()
Whether Tpetra is in debug mode.
Definition Tpetra_Details_Behavior.cpp:238
Tpetra::Details::Behavior::verbose
static bool verbose()
Whether Tpetra is in verbose mode.
Definition Tpetra_Details_Behavior.cpp:247
Tpetra::Details::Behavior::verbosePrintCountThreshold
static size_t verbosePrintCountThreshold()
Number of entries below which arrays, lists, etc. will be printed in debug mode.
Definition Tpetra_Details_Behavior.cpp:300
Tpetra::Details::LocalMap
"Local" part of Map suitable for Kokkos kernels.
Definition Tpetra_Details_LocalMap.hpp:39
Tpetra::DistObject< GlobalOrdinal, LocalOrdinal, GlobalOrdinal, Node >::doImport
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, const CombineMode CM, const bool restrictedMode=false)
Tpetra::DistObject< GlobalOrdinal, LocalOrdinal, GlobalOrdinal, Node >::doExport
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, const CombineMode CM, const bool restrictedMode=false)
Tpetra::Distributor
Sets up and executes a communication plan for a Tpetra DistObject.
Definition Tpetra_Distributor.hpp:95
Tpetra::Map::getGlobalElement
global_ordinal_type getGlobalElement(local_ordinal_type localIndex) const
The global index corresponding to the given local index.
Definition Tpetra_Map_def.hpp:1209
Tpetra::Map::isNodeLocalElement
bool isNodeLocalElement(local_ordinal_type localIndex) const
Whether the given local index is valid for this Map on the calling process.
Definition Tpetra_Map_def.hpp:1258
Tpetra::Map::getLocalElement
local_ordinal_type getLocalElement(global_ordinal_type globalIndex) const
The local index corresponding to the given global index.
Definition Tpetra_Map_def.hpp:1187
Tpetra::Map::isNodeGlobalElement
bool isNodeGlobalElement(global_ordinal_type globalIndex) const
Whether the given global index is owned by this Map on the calling process.
Definition Tpetra_Map_def.hpp:1268
Tpetra::Map::getLocalMap
local_map_type getLocalMap() const
Get the LocalMap for Kokkos-Kernels.
Definition Tpetra_Map_def.hpp:1286
Tpetra::RowGraph
An abstract interface for graphs accessed by rows.
Definition Tpetra_RowGraph_decl.hpp:38
Tpetra::RowGraph::isFillComplete
virtual bool isFillComplete() const =0
Whether fillComplete() has been called (without an intervening resumeFill()).
Tpetra::RowGraph::getRowMap
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getRowMap() const =0
The Map that describes this graph's distribution of rows over processes.
Tpetra::RowGraph::getGlobalRowCopy
virtual void getGlobalRowCopy(const GlobalOrdinal gblRow, nonconst_global_inds_host_view_type &gblColInds, size_t &numColInds) const =0
Get a copy of the global column indices in a given row of the graph.
Tpetra::RowGraph::getNumEntriesInGlobalRow
virtual size_t getNumEntriesInGlobalRow(GlobalOrdinal globalRow) const =0
Returns the current number of entries on this node in the specified global row.
Tpetra::SrcDistObject
Abstract base class for objects that can be the source of an Import or Export operation.
Definition Tpetra_SrcDistObject.hpp:57
Tpetra::Vector
A distributed dense vector.
Definition Tpetra_Vector_decl.hpp:44
Details
Implementation details of Tpetra.
Tpetra::Details
Nonmember function that computes a residual Computes R = B - A * X.
Definition Tpetra_KokkosRefactor_Details_MultiVectorLocalDeepCopy.hpp:17
Tpetra::Details::padCrsArrays
void padCrsArrays(const RowPtr &rowPtrBeg, const RowPtr &rowPtrEnd, Indices &indices_wdv, const Padding &padding, const int my_rank, const bool verbose)
Determine if the row pointers and indices arrays need to be resized to accommodate new entries....
Definition Tpetra_Details_crsUtils.hpp:558
Tpetra::Details::verbosePrintArray
void verbosePrintArray(std::ostream &out, const ArrayType &x, const char name[], const size_t maxNumToPrint)
Print min(x.size(), maxNumToPrint) entries of x.
Definition Tpetra_Util.hpp:1022
Tpetra::Details::copyOffsets
void copyOffsets(const OutputViewType &dst, const InputViewType &src)
Copy row offsets (in a sparse graph or matrix) from src to dst. The offsets may have different types.
Definition Tpetra_Details_copyOffsets.hpp:490
Tpetra::Details::unpackAndCombineIntoCrsArrays
void unpackAndCombineIntoCrsArrays(const CrsGraph< LO, GO, NT > &sourceGraph, const Teuchos::ArrayView< const LO > &importLIDs, const Teuchos::ArrayView< const typename CrsGraph< LO, GO, NT >::packet_type > &imports, const Teuchos::ArrayView< const size_t > &numPacketsPerLID, const size_t constantNumPackets, const CombineMode combineMode, const size_t numSameIDs, const Teuchos::ArrayView< const LO > &permuteToLIDs, const Teuchos::ArrayView< const LO > &permuteFromLIDs, size_t TargetNumRows, size_t TargetNumNonzeros, const int MyTargetPID, const Teuchos::ArrayView< size_t > &CRS_rowptr, const Teuchos::ArrayView< GO > &CRS_colind, const Teuchos::ArrayView< const int > &SourcePids, Teuchos::Array< int > &TargetPids)
unpackAndCombineIntoCrsArrays
Tpetra::Details::disableWDVTracking
void disableWDVTracking()
Disable WrappedDualView reference-count tracking and syncing. Call this before entering a host-parall...
Definition Tpetra_Details_WrappedDualView.cpp:23
Tpetra::Details::packCrsGraph
void packCrsGraph(const CrsGraph< LO, GO, NT > &sourceGraph, Teuchos::Array< typename CrsGraph< LO, GO, NT >::packet_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, const Teuchos::ArrayView< const LO > &exportLIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse graph for communication.
Definition Tpetra_Details_packCrsGraph_def.hpp:691
Tpetra::Details::unpackAndCombineWithOwningPIDsCount
size_t unpackAndCombineWithOwningPIDsCount(const CrsGraph< LO, GO, NT > &sourceGraph, const Teuchos::ArrayView< const LO > &importLIDs, const Teuchos::ArrayView< const typename CrsGraph< LO, GO, NT >::packet_type > &imports, const Teuchos::ArrayView< const size_t > &numPacketsPerLID, size_t constantNumPackets, CombineMode combineMode, size_t numSameIDs, const Teuchos::ArrayView< const LO > &permuteToLIDs, const Teuchos::ArrayView< const LO > &permuteFromLIDs)
Special version of Tpetra::Details::unpackCrsGraphAndCombine that also unpacks owning process ranks.
Tpetra::Details::getArrayViewFromDualView
Teuchos::ArrayView< typename DualViewType::t_dev::value_type > getArrayViewFromDualView(const DualViewType &x)
Get a Teuchos::ArrayView which views the host Kokkos::View of the input 1-D Kokkos::DualView.
Definition Tpetra_Util.hpp:939
Tpetra::Details::insertCrsIndices
size_t insertCrsIndices(typename Pointers::value_type const row, Pointers const &rowPtrs, InOutIndices &curIndices, size_t &numAssigned, InIndices const &newIndices, std::function< void(const size_t, const size_t, const size_t)> cb=std::function< void(const size_t, const size_t, const size_t)>())
Insert new indices in to current list of indices.
Definition Tpetra_Details_crsUtils.hpp:635
Tpetra::Details::packCrsGraphNew
void packCrsGraphNew(const CrsGraph< LO, GO, NT > &sourceGraph, const Kokkos::DualView< const LO *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exportLIDs, const Kokkos::DualView< const int *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exportPIDs, Kokkos::DualView< typename CrsGraph< LO, GO, NT >::packet_type *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exports, Kokkos::DualView< size_t *, typename CrsGraph< LO, GO, NT >::buffer_device_type > numPacketsPerLID, size_t &constantNumPackets, const bool pack_pids)
Pack specified entries of the given local sparse graph for communication, for "new" DistObject interf...
Definition Tpetra_Details_packCrsGraph_def.hpp:777
Tpetra::Details::convertColumnIndicesFromGlobalToLocal
OffsetType convertColumnIndicesFromGlobalToLocal(const Kokkos::View< LO *, DT > &lclColInds, const Kokkos::View< const GO *, DT > &gblColInds, const Kokkos::View< const OffsetType *, DT > &ptr, const LocalMap< LO, GO, DT > &lclColMap, const Kokkos::View< const NumEntType *, DT > &numRowEnt)
Convert a CrsGraph's global column indices into local column indices.
Definition Tpetra_CrsGraph_def.hpp:181
Tpetra::Details::createPrefix
std::unique_ptr< std::string > createPrefix(const int myRank, const char prefix[])
Create string prefix for each line of verbose output.
Definition Tpetra_Util.cpp:69
Tpetra::Details::computeOffsetsFromCounts
OffsetsViewType::non_const_value_type computeOffsetsFromCounts(const ExecutionSpace &execSpace, const OffsetsViewType &ptr, const CountsViewType &counts)
Compute offsets from counts.
Definition Tpetra_Details_computeOffsets.hpp:209
Tpetra::Details::computeOffsetsFromConstantCount
OffsetsViewType::non_const_value_type computeOffsetsFromConstantCount(const OffsetsViewType &ptr, const CountType count)
Compute offsets from a constant count.
Definition Tpetra_Details_computeOffsets.hpp:318
Tpetra::Details::findCrsIndices
size_t findCrsIndices(typename Pointers::value_type const row, Pointers const &rowPtrs, const size_t curNumEntries, Indices1 const &curIndices, Indices2 const &newIndices, Callback &&cb)
Finds offsets in to current list of indices.
Definition Tpetra_Details_crsUtils.hpp:702
Tpetra::Details::makeColMap
int makeColMap(Teuchos::RCP< const Tpetra::Map< LO, GO, NT > > &colMap, Teuchos::Array< int > &remotePIDs, const Teuchos::RCP< const Tpetra::Map< LO, GO, NT > > &domMap, const RowGraph< LO, GO, NT > &graph, const bool sortEachProcsGids=true, std::ostream *errStrm=NULL)
Make the graph's column Map.
Definition Tpetra_Details_makeColMap_def.hpp:291
Tpetra::Details::enableWDVTracking
void enableWDVTracking()
Enable WrappedDualView reference-count tracking and syncing. Call this after exiting a host-parallel ...
Definition Tpetra_Details_WrappedDualView.cpp:17
Tpetra::Details::packCrsGraphWithOwningPIDs
void packCrsGraphWithOwningPIDs(const CrsGraph< LO, GO, NT > &sourceGraph, Kokkos::DualView< typename CrsGraph< LO, GO, NT >::packet_type *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exports_dv, const Teuchos::ArrayView< size_t > &numPacketsPerLID, const Teuchos::ArrayView< const LO > &exportLIDs, const Teuchos::ArrayView< const int > &sourcePIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse graph for communication.
Definition Tpetra_Details_packCrsGraph_def.hpp:863
Tpetra::Details::gathervPrint
void gathervPrint(std::ostream &out, const std::string &s, const Teuchos::Comm< int > &comm)
On Process 0 in the given communicator, print strings from each process in that communicator,...
Definition Tpetra_Details_gathervPrint.cpp:17
Tpetra
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Tpetra::Array_size_type
Teuchos_Ordinal Array_size_type
Size type for Teuchos Array objects.
Definition Tpetra_ConfigDefs.hpp:21
Tpetra::global_size_t
size_t global_size_t
Global size_t object.
Definition Tpetra_ConfigDefs.hpp:70
Tpetra::createOneToOne
Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > createOneToOne(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &M)
Nonmember constructor for a contiguous Map with user-defined weights and a user-specified,...
Tpetra::CombineMode
CombineMode
Rule for combining data in an Import or Export.
Definition Tpetra_CombineMode.hpp:65
Tpetra::INSERT
@ INSERT
Insert new values that don't currently exist.
Definition Tpetra_CombineMode.hpp:67
Tpetra::Details::PackTraits
Traits class for packing / unpacking data of type T.
Definition Tpetra_Details_PackTraits.hpp:29
Tpetra::RowInfo
Allocation information for a locally owned row in a CrsGraph or CrsMatrix.
Definition Tpetra_CrsGraph_decl.hpp:78
Generated by doxygen 1.15.0