[OFFLOAD][OPENMP] 6.0 compatible interop interface #143491
Open
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
The following patch introduces a new interop interface implementation with the following characteristics: * It supports the new 6.0 prefer_type specification * It supports both explicit objects (from interop constructs) and implicit objects (from variant calls). * Implements a per-thread reuse mechanism for implicit objects to reduce overheads. * It provides a plugin interface that allows selecting the supported interop types, and managing all the backend related interop operations (init, sync, ...). * It enables cooperation with the OpenMP runtime to allow progress on OpenMP synchronizations. * It cleanups some vendor/fr_id mismatchs from the current query routines. * It supports extension to define interop callbacks for library cleanup.
|
@llvm/pr-subscribers-offload Author: Alex (adurang) ChangesThe following patch introduces a new interop interface implementation with the following characteristics:
Patch is 39.71 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/143491.diff 15 Files Affected:
diff --git a/offload/include/OpenMP/InteropAPI.h b/offload/include/OpenMP/InteropAPI.h
index 71c78760a3226..61cbedf06a9a6 100644
--- a/offload/include/OpenMP/InteropAPI.h
+++ b/offload/include/OpenMP/InteropAPI.h
@@ -13,17 +13,70 @@
#include "omp.h"
+#include "PerThreadTable.h"
#include "omptarget.h"
extern "C" {
typedef enum kmp_interop_type_t {
kmp_interop_type_unknown = -1,
- kmp_interop_type_platform,
- kmp_interop_type_device,
- kmp_interop_type_tasksync,
+ kmp_interop_type_target,
+ kmp_interop_type_targetsync,
} kmp_interop_type_t;
+struct interop_attrs_t {
+ bool inorder : 1;
+ int reserved : 31;
+
+ /* Check if the supported attributes are compatible with the current
+ attributes. Only if an attribute is supported can the value be true,
+ otherwise it needs to be false
+ */
+ bool checkSupportedOnly(interop_attrs_t supported) const {
+ return supported.inorder || (!supported.inorder && !inorder);
+ }
+};
+
+struct interop_spec_t {
+ int32_t fr_id;
+ interop_attrs_t attrs; // Common attributes
+ int64_t impl_attrs; // Implementation specific attributes (recognized by each
+ // plugin)
+};
+
+struct interop_flags_t {
+ bool implicit : 1; // dispatch (true) or interop (false)
+ bool nowait : 1; // has nowait flag
+ int reserved : 30;
+};
+
+struct interop_ctx_t {
+ uint16_t version; // version of the interface (current is 0)
+ interop_flags_t flags;
+ int gtid;
+};
+
+struct dep_pack_t {
+ int32_t ndeps;
+ kmp_depend_info_t *deplist;
+ int32_t ndeps_noalias;
+ kmp_depend_info_t *noalias_deplist;
+};
+
+struct omp_interop_val_t;
+
+typedef void ompx_interop_cb_t(omp_interop_val_t *interop, void *data);
+
+struct omp_interop_cb_instance_t {
+ ompx_interop_cb_t *cb;
+ void *data;
+
+ omp_interop_cb_instance_t(ompx_interop_cb_t *cb, void *data)
+ : cb(cb), data(data) {}
+
+ void operator()(omp_interop_val_t *interop) { cb(interop, data); }
+};
+
/// The interop value type, aka. the interop object.
typedef struct omp_interop_val_t {
/// Device and interop-type are determined at construction time and fix.
@@ -34,10 +87,96 @@ typedef struct omp_interop_val_t {
__tgt_device_info device_info;
const kmp_interop_type_t interop_type;
const intptr_t device_id;
- const omp_foreign_runtime_ids_t vendor_id = cuda;
- const intptr_t backend_type_id = omp_interop_backend_type_cuda_1;
+ omp_vendor_id_t vendor_id = omp_vendor_llvm;
+ omp_foreign_runtime_id_t fr_id = omp_fr_none;
+ interop_attrs_t attrs{false, 0}; // Common prefer specification attributes
+ int64_t impl_attrs = 0; // Implementation prefer specification attributes
+
+ void *RTLProperty = nullptr; // Plugin dependent information
+ // For implicitly created Interop objects (e.g., from a dispatch construct)
+ // who owns the object
+ int OwnerGtid = -1;
+ // Marks whether the object was requested since the last time it was synced
+ bool Clean = true;
+
+ typedef llvm::SmallVector<omp_interop_cb_instance_t> callback_list_t;
+
+ callback_list_t CompletionCbs;
+
+ void reset() {
+ OwnerGtid = -1;
+ markClean();
+ clearCompletionCbs();
+ }
+
+ bool hasOwner() const { return OwnerGtid != -1; }
+
+ void setOwner(int gtid) { OwnerGtid = gtid; }
+ bool isOwnedBy(int gtid) { return OwnerGtid == gtid; }
+ bool isCompatibleWith(int32_t InteropType, const interop_spec_t &Spec);
+ bool isCompatibleWith(int32_t InteropType, const interop_spec_t &Spec,
+ int64_t DeviceNum, int gtid);
+ void markClean() { Clean = true; }
+ void markDirty() { Clean = false; }
+ bool isClean() const { return Clean; }
+
+ int32_t flush(DeviceTy &Device);
+ int32_t sync_barrier(DeviceTy &Device);
+ int32_t async_barrier(DeviceTy &Device);
+ int32_t release(DeviceTy &Device);
+
+ int32_t flush();
+ int32_t syncBarrier();
+ int32_t asyncBarrier();
+ int32_t release();
+
+ void addCompletionCb(ompx_interop_cb_t *cb, void *data) {
+ CompletionCbs.push_back(omp_interop_cb_instance_t(cb, data));
+ }
+
+ int numCompletionCbs() const { return CompletionCbs.size(); }
+ void clearCompletionCbs() { CompletionCbs.clear(); }
+
+ void runCompletionCbs() {
+ for (auto &cbInstance : CompletionCbs)
+ cbInstance(this);
+ clearCompletionCbs();
+ }
} omp_interop_val_t;
} // extern "C"
+struct InteropTableEntry {
+ using ContainerTy = typename std::vector<omp_interop_val_t *>;
+ using iterator = typename ContainerTy::iterator;
+
+ ContainerTy Interops;
+
+ const int reservedEntriesPerThread =
+ 20; // reserve some entries to avoid reallocation
+
+ void add(omp_interop_val_t *obj) {
+ if (Interops.capacity() == 0)
+ Interops.reserve(reservedEntriesPerThread);
+ Interops.push_back(obj);
+ }
+
+ template <class ClearFuncTy> void clear(ClearFuncTy f) {
+ for (auto &Obj : Interops) {
+ f(Obj);
+ }
+ }
+
+ /* vector interface */
+ int size() const { return Interops.size(); }
+ iterator begin() { return Interops.begin(); }
+ iterator end() { return Interops.end(); }
+ iterator erase(iterator it) { return Interops.erase(it); }
+};
+
+struct InteropTblTy
+ : public PerThreadTable<InteropTableEntry, omp_interop_val_t *> {
+ void clear();
+};
+
#endif // OMPTARGET_OPENMP_INTEROP_API_H
diff --git a/offload/include/OpenMP/omp.h b/offload/include/OpenMP/omp.h
index b44c6aff1b289..67b3bab9e8599 100644
--- a/offload/include/OpenMP/omp.h
+++ b/offload/include/OpenMP/omp.h
@@ -80,15 +80,18 @@ typedef enum omp_interop_rc {
omp_irc_other = -6
} omp_interop_rc_t;
-typedef enum omp_interop_fr {
- omp_ifr_cuda = 1,
- omp_ifr_cuda_driver = 2,
- omp_ifr_opencl = 3,
- omp_ifr_sycl = 4,
- omp_ifr_hip = 5,
- omp_ifr_level_zero = 6,
- omp_ifr_last = 7
-} omp_interop_fr_t;
+/* Foreign runtime values from OpenMP Additional Definitions document v2.1 */
+typedef enum omp_foreign_runtime_id_t {
+ omp_fr_none = 0,
+ omp_fr_cuda = 1,
+ omp_fr_cuda_driver = 2,
+ omp_fr_opencl = 3,
+ omp_fr_sycl = 4,
+ omp_fr_hip = 5,
+ omp_fr_level_zero = 6,
+ omp_fr_hsa = 7,
+ omp_fr_last = 8
+} omp_foreign_runtime_id_t;
typedef void *omp_interop_t;
@@ -134,19 +137,23 @@ omp_get_interop_type_desc(const omp_interop_t, omp_interop_property_t);
extern const char *__KAI_KMPC_CONVENTION
omp_get_interop_rc_desc(const omp_interop_t, omp_interop_rc_t);
-typedef enum omp_interop_backend_type_t {
- // reserve 0
- omp_interop_backend_type_cuda_1 = 1,
-} omp_interop_backend_type_t;
-
-typedef enum omp_foreign_runtime_ids {
- cuda = 1,
- cuda_driver = 2,
- opencl = 3,
- sycl = 4,
- hip = 5,
- level_zero = 6,
-} omp_foreign_runtime_ids_t;
+/* Vendor defined values from OpenMP Additional Definitions document v2.1*/
+typedef enum omp_vendor_id {
+ omp_vendor_unknown = 0,
+ omp_vendor_amd = 1,
+ omp_vendor_arm = 2,
+ omp_vendor_bsc = 3,
+ omp_vendor_fujitsu = 4,
+ omp_vendor_gnu = 5,
+ omp_vendor_hpe = 6,
+ omp_vendor_ibm = 7,
+ omp_vendor_intel = 8,
+ omp_vendor_llvm = 9,
+ omp_vendor_nec = 10,
+ omp_vendor_nvidia = 11,
+ omp_vendor_ti = 12,
+ omp_vendor_last = 13
+} omp_vendor_id_t;
///} InteropAPI
diff --git a/offload/include/PerThreadTable.h b/offload/include/PerThreadTable.h
new file mode 100644
index 0000000000000..1e20b56c734d2
--- /dev/null
+++ b/offload/include/PerThreadTable.h
@@ -0,0 +1,109 @@
+//===-- PerThreadTable.h -- PerThread Storage Structure ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Table indexed with one entry per thread.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PERTHREADTABLE_H
+#define OFFLOAD_PERTHREADTABLE_H
+
+#include <list>
+#include <memory>
+#include <mutex>
+
+// Using an STL container (such as std::vector) indexed by thread ID has
+// too many race conditions issues so we store each thread entry into a
+// thread_local variable.
+// T is the container type used to store the objects, e.g., std::vector,
+// std::set, etc. by each thread. O is the type of the stored objects e.g.,
+// omp_interop_val_t *, ...
+
+template <typename ContainerType, typename ObjectType> struct PerThreadTable {
+ using iterator = typename ContainerType::iterator;
+
+ struct PerThreadData {
+ size_t NElements = 0;
+ std::unique_ptr<ContainerType> ThEntry;
+ };
+
+ std::mutex Mtx;
+ std::list<PerThreadData *> ThreadDataList;
+
+ // define default constructors, disable copy and move constructors
+ PerThreadTable() = default;
+ PerThreadTable(const PerThreadTable &) = delete;
+ PerThreadTable(PerThreadTable &&) = delete;
+ PerThreadTable &operator=(const PerThreadTable &) = delete;
+ PerThreadTable &operator=(PerThreadTable &&) = delete;
+ ~PerThreadTable() {
+ std::lock_guard<std::mutex> Lock(Mtx);
+ ThreadDataList.clear();
+ }
+
+private:
+ PerThreadData &getThreadData() {
+ static thread_local PerThreadData ThData;
+ return ThData;
+ }
+
+protected:
+ ContainerType &getThreadEntry() {
+ auto &ThData = getThreadData();
+ if (ThData.ThEntry)
+ return *ThData.ThEntry;
+ ThData.ThEntry = std::make_unique<ContainerType>();
+ std::lock_guard<std::mutex> Lock(Mtx);
+ ThreadDataList.push_back(&ThData);
+ return *ThData.ThEntry;
+ }
+
+ size_t &getThreadNElements() {
+ auto &ThData = getThreadData();
+ return ThData.NElements;
+ }
+
+public:
+ void add(ObjectType obj) {
+ auto &Entry = getThreadEntry();
+ auto &NElements = getThreadNElements();
+ NElements++;
+ Entry.add(obj);
+ }
+
+ iterator erase(iterator it) {
+ auto &Entry = getThreadEntry();
+ auto &NElements = getThreadNElements();
+ NElements--;
+ return Entry.erase(it);
+ }
+
+ size_t size() { return getThreadNElements(); }
+
+ // Iterators to traverse objects owned by
+ // the current thread
+ iterator begin() {
+ auto &Entry = getThreadEntry();
+ return Entry.begin();
+ }
+ iterator end() {
+ auto &Entry = getThreadEntry();
+ return Entry.end();
+ }
+
+ template <class F> void clear(F f) {
+ std::lock_guard<std::mutex> Lock(Mtx);
+ for (auto ThData : ThreadDataList) {
+ ThData->ThEntry->clear(f);
+ ThData->NElements = 0;
+ }
+ ThreadDataList.clear();
+ }
+};
+
+#endif
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index ec3adadf0819b..ea1f3b6406ce7 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -35,6 +35,8 @@
#include <mutex>
#include <string>
+#include "OpenMP/InteropAPI.h"
+
using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
/// Struct for the data required to handle plugins
@@ -88,6 +90,9 @@ struct PluginManager {
HostPtrToTableMapTy HostPtrToTableMap;
std::mutex TblMapMtx; ///< For HostPtrToTableMap
+ /// Table of cached implicit interop objects
+ InteropTblTy InteropTbl;
+
// Work around for plugins that call dlopen on shared libraries that call
// tgt_register_lib during their initialisation. Stash the pointers in a
// vector until the plugins are all initialised and then register them.
@@ -185,5 +190,5 @@ void initRuntime();
void deinitRuntime();
extern PluginManager *PM;
-
+extern std::atomic<bool> RTLAlive; // Indicates if the RTL has been initialized
#endif // OMPTARGET_PLUGIN_MANAGER_H
diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 978b53d5d69b9..f376c7dc861f9 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -36,6 +36,7 @@ struct __tgt_device_image {
struct __tgt_device_info {
void *Context = nullptr;
void *Device = nullptr;
+ void *Platform = nullptr;
};
/// This struct is a record of all the host code that may be offloaded to a
diff --git a/offload/libomptarget/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
index 29b573a27d087..134ab7c95ac0b 100644
--- a/offload/libomptarget/OffloadRTL.cpp
+++ b/offload/libomptarget/OffloadRTL.cpp
@@ -22,6 +22,7 @@ extern void llvm::omp::target::ompt::connectLibrary();
static std::mutex PluginMtx;
static uint32_t RefCount = 0;
+std::atomic<bool> RTLAlive{false};
void initRuntime() {
std::scoped_lock<decltype(PluginMtx)> Lock(PluginMtx);
@@ -41,6 +42,9 @@ void initRuntime() {
PM->init();
PM->registerDelayedLibraries();
+
+ // RTL initialization is complete
+ RTLAlive = true;
}
}
@@ -50,6 +54,8 @@ void deinitRuntime() {
if (RefCount == 1) {
DP("Deinit offload library!\n");
+ // RTL deinitialization has started
+ RTLAlive = false;
PM->deinit();
delete PM;
PM = nullptr;
diff --git a/offload/libomptarget/OpenMP/API.cpp b/offload/libomptarget/OpenMP/API.cpp
index 4576f9bd06121..f61f56772504b 100644
--- a/offload/libomptarget/OpenMP/API.cpp
+++ b/offload/libomptarget/OpenMP/API.cpp
@@ -683,3 +683,15 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
return TPR.TargetPointer;
}
+
+void syncImplicitInterops(int gtid, void *event);
+// This routine gets called from the Host RTL at sync points (taskwait, barrier,
+// ...) so we can synchronize the necessary objects from the offload side.
+EXTERN void __tgt_target_sync(ident_t *loc_ref, int gtid, void *current_task,
+ void *event) {
+
+ if (!RTLAlive)
+ return;
+
+ syncImplicitInterops(gtid, event);
+}
diff --git a/offload/libomptarget/OpenMP/InteropAPI.cpp b/offload/libomptarget/OpenMP/InteropAPI.cpp
index bdbc440c64a2c..55e47d87a865d 100644
--- a/offload/libomptarget/OpenMP/InteropAPI.cpp
+++ b/offload/libomptarget/OpenMP/InteropAPI.cpp
@@ -10,6 +10,7 @@
#include "OpenMP/InternalTypes.h"
#include "OpenMP/omp.h"
+#include "OffloadPolicy.h"
#include "PluginManager.h"
#include "device.h"
#include "omptarget.h"
@@ -56,22 +57,22 @@ void getTypeMismatch(omp_interop_property_t Property, int *Err) {
*Err = getPropertyErrorType(Property);
}
-const char *getVendorIdToStr(const omp_foreign_runtime_ids_t VendorId) {
- switch (VendorId) {
- case cuda:
- return ("cuda");
- case cuda_driver:
- return ("cuda_driver");
- case opencl:
- return ("opencl");
- case sycl:
- return ("sycl");
- case hip:
- return ("hip");
- case level_zero:
- return ("level_zero");
- }
- return ("unknown");
+static const char *VendorStrTbl[] = {
+ "unknown", "amd", "arm", "bsc", "fujitsu", "gnu", "hpe",
+ "ibm", "intel", "llvm", "nec", "nvidia", "ti"};
+const char *getVendorIdToStr(const omp_vendor_id_t VendorId) {
+ if (VendorId < omp_vendor_unknown || VendorId >= omp_vendor_last)
+ return ("unknown");
+ return VendorStrTbl[VendorId];
+}
+
+static const char *ForeignRuntimeStrTbl[] = {
+ "none", "cuda", "cuda_driver", "opencl",
+ "sycl", "hip", "level_zero", "hsa"};
+const char *getForeignRuntimeIdToStr(const omp_foreign_runtime_id_t FrId) {
+ if (FrId < omp_fr_none || FrId >= omp_fr_last)
+ return ("unknown");
+ return ForeignRuntimeStrTbl[FrId];
}
template <typename PropertyTy>
@@ -83,7 +84,7 @@ intptr_t getProperty<intptr_t>(omp_interop_val_t &InteropVal,
omp_interop_property_t Property, int *Err) {
switch (Property) {
case omp_ipr_fr_id:
- return InteropVal.backend_type_id;
+ return InteropVal.fr_id;
case omp_ipr_vendor:
return InteropVal.vendor_id;
case omp_ipr_device_num:
@@ -99,10 +100,8 @@ const char *getProperty<const char *>(omp_interop_val_t &InteropVal,
omp_interop_property_t Property,
int *Err) {
switch (Property) {
- case omp_ipr_fr_id:
- return InteropVal.interop_type == kmp_interop_type_tasksync
- ? "tasksync"
- : "device+context";
+ case omp_ipr_fr_name:
+ return getForeignRuntimeIdToStr(InteropVal.fr_id);
case omp_ipr_vendor_name:
return getVendorIdToStr(InteropVal.vendor_id);
default:
@@ -120,6 +119,8 @@ void *getProperty<void *>(omp_interop_val_t &InteropVal,
return InteropVal.device_info.Device;
*Err = omp_irc_no_value;
return const_cast<char *>(InteropVal.err_str);
+ case omp_ipr_platform:
+ return InteropVal.device_info.Platform;
case omp_ipr_device_context:
return InteropVal.device_info.Context;
case omp_ipr_targetsync:
@@ -145,13 +146,13 @@ bool getPropertyCheck(omp_interop_val_t **InteropPtr,
return false;
}
if (Property == omp_ipr_targetsync &&
- (*InteropPtr)->interop_type != kmp_interop_type_tasksync) {
+ (*InteropPtr)->interop_type != kmp_interop_type_targetsync) {
if (Err)
*Err = omp_irc_other;
return false;
}
if ((Property == omp_ipr_device || Property == omp_ipr_device_context) &&
- (*InteropPtr)->interop_type == kmp_interop_type_tasksync) {
+ (*InteropPtr)->interop_type == kmp_interop_type_targetsync) {
if (Err)
*Err = omp_irc_other;
return false;
@@ -166,7 +167,7 @@ bool getPropertyCheck(omp_interop_val_t **InteropPtr,
omp_interop_property_t property_id, \
int *err) { \
omp_interop_val_t *interop_val = (omp_interop_val_t *)interop; \
- assert((interop_val)->interop_type == kmp_interop_type_tasksync); \
+ assert((interop_val)->interop_type == kmp_interop_type_targetsync); \
if (!getPropertyCheck(&interop_val, property_id, err)) { \
return (RETURN_TYPE)(0); \
} \
@@ -193,119 +194,263 @@ __OMP_GET_INTEROP_TY3(const char *, type_desc)
__OMP_GET_INTEROP_TY3(const char *, rc_desc)
#undef __OMP_GET_INTEROP_TY3
-static const char *copyErrorString(llvm::Error &&Err) {
- // TODO: Use the error string while avoiding leaks.
- std::string ErrMsg = llvm::toString(std::move(Err));
- char *UsrMsg = reinterpret_cast<char *>(malloc(ErrMsg.size() + 1));
- strcpy(UsrMsg, ErrMsg.c_str());
- return UsrMsg;
-}
-
extern "C" {
-void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,
- omp_interop_val_t *&InteropPtr,
- kmp_interop_type_t InteropType, int32_t DeviceId,
- int32_t Ndeps, kmp_depend_info_t *DepList,
- int32_t HaveNowait) {
- int32_t NdepsNoalias = 0;
- kmp_depend_info_t *NoaliasDepList = NULL;
- assert(InteropType != kmp_interop_type_unknown &&
- "Cannot initialize with unknown interop_type!");
- if (DeviceId == -1) {
- DeviceId = omp_get_default_device();
+omp_interop_val_t *__tgt_interop_get(ident_t *LocRef, int32_t InteropType,
+ int64_t DeviceNum, int32_t NumPrefers,
+ interop_spec_t *Prefers,
+ interop_ctx_t *Ctx, dep_pack_t *Deps) {
+
+ DP("Call to %s with device_num %" PRId64 ", interop type %" PRId32
+ ", number of preferred specs %" PRId32 "%s%s\n",
+ __func__, DeviceNum, InteropType, NumPrefers,
+ Ctx->flags.implicit ? " (implicit)" : "",
+ Ctx->flags.nowait ? " (nowait)" : "");
+
+ if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED)
+ return omp_interop_none;
+
+ // Now, try to create an interop with device_num.
+ if (DeviceNum == OFFLOAD_DEVICE_DEFAULT)
+ DeviceNum = omp_get_default_device();
+
+ auto gtid = Ctx->gtid;
+
+ if (InteropType == kmp_interop_type_targetsync) {
+ if (Ctx->flags.nowait)
+ DP("Warning: nowait flag on interop creation not supported yet. "
+ "Ignored\n");
+ if (Deps)
+ __kmpc_omp_wait_deps(LocRef, gtid, Deps->ndeps, Deps->deplist,
+ Deps->ndeps_noalias, Deps->noalias_deplist);
}
- if (InteropType == kmp_interop_type_tasksync) {
- __kmpc_omp_wait_deps(LocRef, Gtid, Ndeps, DepList, NdepsNoalias,
- NoaliasDepList);
+ auto DeviceOrErr = PM->getDevice(DeviceNum);
+ if (!DeviceOrErr) {
+ [[maybe_unused]] std::string ErrStr = toString(DeviceOrErr.takeError());
+ DP("Couldn't find device %" PRId64
+ " wh...
[truncated]
|
|
@jhuber6, @jdoerfert can you review this PR. Thanks. |
# for free
to join this conversation on GitHub.
Already have an account?
# to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
The following patch introduces a new interop interface implementation with the following characteristics: