diff --git a/libnd4j/blas/NDArray.h b/libnd4j/blas/NDArray.h index 671f72a57..fe5f90bc3 100644 --- a/libnd4j/blas/NDArray.h +++ b/libnd4j/blas/NDArray.h @@ -195,6 +195,56 @@ namespace nd4j { NDArray(std::shared_ptr buffer, const char order, const std::vector &shape, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + /** + * This contructors create scalar array containing string utf8 + * + */ + NDArray(const char* str, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()) + : NDArray(std::string(str), dtype, context) { + } + NDArray(const std::string& string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + + /** + * This contructors create scalar array containing string utf16 + * + */ + NDArray(const char16_t* u16string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()) + : NDArray(std::u16string(u16string), dtype, context) { + } + + NDArray(const std::u16string& u16string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + + /** + * This contructors create scalar array containing string utf32 + * + */ + NDArray(const char32_t* u32string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()) + : NDArray(std::u32string(u32string), dtype, context) { + } + + NDArray(const std::u32string& u32string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + + /** + * This contructors create array from vector of utf8 strings + * + */ + NDArray(const std::vector& shape, const std::vector& strings, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + NDArray(const std::vector& shape, const std::vector& string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + + /** + * This contructors create array from vector of utf16 strings + * + */ + NDArray(const std::vector& shape, const std::vector& strings, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + NDArray(const std::vector& shape, const std::vector& string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + + /** + * This contructors create array from vector of utf32 strings + * + */ + NDArray(const std::vector& shape, const std::vector& strings, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + NDArray(const std::vector& shape, const std::vector& string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + #endif /** @@ -250,7 +300,6 @@ namespace nd4j { */ NDArray(void *buffer, const char order, const std::vector &shape, nd4j::DataType dtype, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext(), const bool isBuffAlloc = false); - /** * This method returns new array with the same shape & data type * @return @@ -1148,6 +1197,9 @@ namespace nd4j { template NDArray asT() const; + template + NDArray asS() const; + NDArray asT(DataType dtype) const; @@ -1441,7 +1493,7 @@ namespace nd4j { * @return */ bool isS() const; - + template std::vector asVectorT(); diff --git a/libnd4j/blas/NDArray.hpp b/libnd4j/blas/NDArray.hpp index 42b29cf78..79137ac3a 100644 --- a/libnd4j/blas/NDArray.hpp +++ b/libnd4j/blas/NDArray.hpp @@ -34,11 +34,15 @@ template <> ND4J_EXPORT utf8string NDArray::e(const Nd4jLong i) const; template <> ND4J_EXPORT std::string NDArray::e(const Nd4jLong i) const; +template <> +ND4J_EXPORT std::u16string NDArray::e(const Nd4jLong i) const; +template <> +ND4J_EXPORT std::u32string NDArray::e(const Nd4jLong i) const; //////////////////////////////////////////////////////////////////////// // copy constructor NDArray::NDArray(const NDArray& other) { - + _context = other._context; _offset = 0; @@ -293,6 +297,560 @@ NDArray::NDArray(std::shared_ptr buffer, const char order, const std _isView = _length * DataTypeUtils::sizeOf(_dataType) < buffer->getLenInBytes(); } +///////////////////////////////////////////////////////////////////////// +// u16 string constructors +NDArray::NDArray(const std::u16string& u16string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dtype)) { + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + } + + if (!unicode::isStringValidU16(u16string.data(), u16string.data() + u16string.size())) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + + // one word that is why used 1 + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(1); + + Nd4jLong dataLength = [&] { + if (dtype == DataType::UTF16) { + return static_cast(u16string.size() * sizeof(uint16_t)); + } + if (dtype == DataType::UTF32) { + return unicode::offsetUtf16StringInUtf32(u16string.data(), u16string.size()); + } + return unicode::offsetUtf16StringInUtf8(u16string.data(), u16string.size()); + }(); + + Nd4jLong offsets[2] = { 0 , dataLength }; + + _buffer = std::make_shared(headerLength + dataLength, dtype, context->getWorkspace(), true); + + _context = context; + _isAttached = getContext()->getWorkspace() != nullptr; + _offset = 0; + + setShapeInfo(ShapeDescriptor::scalarDescriptor(dtype)); + + memcpy(bufferAsT(), &offsets[0], 2 * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + if (dtype == DataType::UTF8) { + unicode::utf16to8(u16string.data(), data, u16string.size()); + } + else if (dtype == DataType::UTF16) { + memcpy(data, u16string.data(), dataLength); + } + else { + unicode::utf16to32(u16string.data(), data, u16string.size()); + } + + tickWriteHost(); + syncToDevice(); +} + +///////////////////////////////////////////////////////////////////////// +// u32 string constructors +NDArray::NDArray(const std::u32string& u32string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dtype)) { + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + } + + if (!unicode::isStringValidU32(u32string.data(), u32string.data() + u32string.size())) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + // one word that is why used 1 + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(1); + + Nd4jLong dataLength = [&] { + if (dtype == DataType::UTF16) { + return unicode::offsetUtf32StringInUtf16(u32string.data(), u32string.size()); + } + if (dtype == DataType::UTF32) { + return static_cast(sizeof(uint32_t) * u32string.size()); + } + return unicode::offsetUtf32StringInUtf8(u32string.data(), u32string.size()); + }(); + + Nd4jLong offsets[2] = { 0 , dataLength }; + + _buffer = std::make_shared(headerLength + dataLength, dtype, context->getWorkspace(), true); + + _context = context; + _isAttached = getContext()->getWorkspace() != nullptr; + _offset = 0; + + setShapeInfo(ShapeDescriptor::scalarDescriptor(dtype)); + + memcpy(bufferAsT(), &offsets[0], 2 * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + if (dtype == DataType::UTF8) { + unicode::utf32to8(u32string.data(), data, u32string.size()); + } + else if (dtype == DataType::UTF16) { + unicode::utf32to16(u32string.data(), data, u32string.size()); + } + else { + memcpy(data, u32string.data(), u32string.size() * sizeof(uint32_t)); + } + + tickWriteHost(); + syncToDevice(); +} + +///////////////////////////////////////////////////////////////////////// +// u8 string constructors +///////////////////////////////////////////////////////////////////////// +NDArray::NDArray(const std::string& str, nd4j::DataType dtype, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dtype)) { + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + } + + if (!unicode::isStringValidU8(str.data(), str.data() + str.size())) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + + // one word that is why used 1 + auto headerLength = ShapeUtils::stringBufferHeaderRequirements(1); + + Nd4jLong dataLength = [&] { + if (dtype == DataType::UTF16) { + return unicode::offsetUtf8StringInUtf16(str.data(), str.size()); + } + if (dtype == DataType::UTF32) { + return unicode::offsetUtf8StringInUtf32(str.data(), str.size()); + } + return static_cast(str.size()); + }(); + + Nd4jLong offsets[2] = { 0 , dataLength }; + + _buffer = std::make_shared(headerLength + dataLength, dtype, context->getWorkspace(), true); + + _context = context; + _isAttached = getContext()->getWorkspace() != nullptr; + _offset = 0; + + setShapeInfo(ShapeDescriptor::scalarDescriptor(dtype)); + + memcpy(bufferAsT(), &offsets[0], 2 * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + + if (dtype == DataType::UTF8) { + memcpy(data, str.data(), str.size()); + } + else if (dtype == DataType::UTF16) { + unicode::utf8to16(str.data(), data, str.size()); + } + else { + unicode::utf8to32(str.data(), data, str.size()); + } + + tickWriteHost(); + syncToDevice(); +} +///////////////////////////////////////////////////////////////////////// +// constructors for vector of strings +NDArray::NDArray(const std::vector& shape, const std::vector& string, const nd4j::DataType dataType, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dataType)) + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + + if (shape::prodLong(shape.data(), shape.size()) != string.size()) + throw std::invalid_argument("NDArray::NDArray: Number of strings should match length of array"); + + for (const auto& str : string) { + if (!unicode::isStringValidU8(str, str + std::char_traits::length(str)) ) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + } + + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(string.size()); + + std::vector offsets(string.size() + 1); + Nd4jLong dataLength = 0; + for (int e = 0; e < string.size(); e++) { + offsets[e] = dataLength; + dataLength += [&] { + if (dataType == DataType::UTF16) + return unicode::offsetUtf8StringInUtf16(string[e], std::char_traits::length(string[e])); + if (dataType == DataType::UTF32) + return unicode::offsetUtf8StringInUtf32(string[e], std::char_traits::length(string[e])); + return static_cast(std::char_traits::length(string[e])); + }(); + } + offsets[string.size()] = dataLength; + + _buffer = std::make_shared(headerLength + dataLength, dataType, context->getWorkspace(), true); + + _context = context; + _offset = 0; + + setShapeInfo(ShapeDescriptor(dataType, 'c', shape)); + + _isView = false; + + setAttached(context->getWorkspace() != nullptr); + + memcpy(bufferAsT(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + + auto func = PRAGMA_THREADS_FOR{ + for (auto e = start; e < stop; e += increment) { + auto cdata = data + offsets[e]; + if (dataType == DataType::UTF16) { + unicode::utf8to16(string[e], cdata, std::char_traits::length(string[e])); + } + else if (dataType == DataType::UTF32) { + unicode::utf8to32(string[e], cdata, std::char_traits::length(string[e])); + } + else { + memcpy(cdata, string[e], std::char_traits::length(string[e])); + } + } + }; + + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + tickWriteHost(); + syncToDevice(); +} +///////////////////////////////////////////////////////////////////////// +NDArray::NDArray(const std::vector& shape, const std::vector& string, const nd4j::DataType dataType, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dataType)) + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + + if (shape::prodLong(shape.data(), shape.size()) != string.size()) + throw std::invalid_argument("NDArray::NDArray: Number of strings should match length of array"); + + for (const auto& str : string) { + if (!unicode::isStringValidU8(str.data(), str.data() + str.size())) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + } + + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(string.size()); + + std::vector offsets(string.size() + 1); + Nd4jLong dataLength = 0; + for (int e = 0; e < string.size(); e++) { + offsets[e] = dataLength; + dataLength += [&] { + if (dataType == DataType::UTF16) + return unicode::offsetUtf8StringInUtf16(string[e].data(), string[e].size()); + if (dataType == DataType::UTF32) + return unicode::offsetUtf8StringInUtf32(string[e].data(), string[e].size()); + return static_cast(string[e].size()); + }(); + } + + offsets[string.size()] = dataLength; + + _buffer = std::make_shared(headerLength + dataLength, dataType, context->getWorkspace(), true); + + _context = context; + _offset = 0; + + setShapeInfo(ShapeDescriptor(dataType, 'c', shape)); + + _isView = false; + + setAttached(context->getWorkspace() != nullptr); + + memcpy(bufferAsT(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + + auto func = PRAGMA_THREADS_FOR{ + for (auto e = start; e < stop; e += increment) { + auto cdata = data + offsets[e]; + if (dataType == DataType::UTF16) { + unicode::utf8to16(string[e].data(), cdata, string[e].size()); + } + else if (dataType == DataType::UTF32) { + unicode::utf8to32(string[e].data(), cdata, string[e].size()); + } + else { + memcpy(cdata, string[e].data(), string[e].size()); + } + } + }; + + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + + tickWriteHost(); + syncToDevice(); +} +///////////////////////////////////////////////////////////////////////// +NDArray::NDArray(const std::vector& shape, const std::vector& string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dtype)) + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + + if (shape::prodLong(shape.data(), shape.size()) != string.size()) + throw std::invalid_argument("NDArray::NDArray: Number of strings should match length of array"); + + for (const auto& str : string) { + if (!unicode::isStringValidU16(str.data(), str.data() + str.size())) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + } + + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(string.size()); + + std::vector offsets(string.size() + 1); + Nd4jLong dataLength = 0; + for (int e = 0; e < string.size(); e++) { + offsets[e] = dataLength; + dataLength += [&] { + if (dtype == DataType::UTF16) + return static_cast(sizeof(uint16_t) * string[e].size()); + if (dtype == DataType::UTF32) + return unicode::offsetUtf16StringInUtf32(string[e].data(), string[e].size()); + return unicode::offsetUtf16StringInUtf8(string[e].data(), string[e].size()); + }(); + } + offsets[string.size()] = dataLength; + + _buffer = std::make_shared(headerLength + dataLength, dtype, context->getWorkspace(), true); + + _context = context; + _offset = 0; + + setShapeInfo(ShapeDescriptor(dtype, 'c', shape)); + + _isView = false; + + setAttached(context->getWorkspace() != nullptr); + + memcpy(bufferAsT(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + + auto func = PRAGMA_THREADS_FOR{ + for (auto e = start; e < stop; e += increment) { + auto cdata = data + offsets[e]; + if (dtype == DataType::UTF16) { + memcpy(cdata, string[e].data(), string[e].size() * sizeof(uint16_t)); + } + else if (dtype == DataType::UTF32) { + unicode::utf16to32(string[e].data(), cdata, string[e].size()); + } + else { + unicode::utf16to8(string[e].data(), cdata, string[e].size()); + } + } + }; + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + tickWriteHost(); + syncToDevice(); +} +///////////////////////////////////////////////////////////////////////// +NDArray::NDArray(const std::vector& shape, const std::vector& string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dtype)) + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + + if (shape::prodLong(shape.data(), shape.size()) != string.size()) + throw std::invalid_argument("NDArray::NDArray: Number of strings should match length of array"); + + for (const auto& str : string) { + if (!unicode::isStringValidU16(str, str + std::char_traits::length(str))) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + } + + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(string.size()); + + std::vector offsets(string.size() + 1); + Nd4jLong dataLength = 0; + for (int e = 0; e < string.size(); e++) { + offsets[e] = dataLength; + dataLength += [&] { + if (dtype == DataType::UTF16) + return static_cast(sizeof(uint16_t) * std::char_traits::length(string[e])); + if (dtype == DataType::UTF32) + return unicode::offsetUtf16StringInUtf32(string[e], std::char_traits::length(string[e])); + return unicode::offsetUtf16StringInUtf8(string[e], std::char_traits::length(string[e])); + }(); + } + offsets[string.size()] = dataLength; + + _buffer = std::make_shared(headerLength + dataLength, dtype, context->getWorkspace(), true); + + _context = context; + _offset = 0; + + setShapeInfo(ShapeDescriptor(dtype, 'c', shape)); + + _isView = false; + + setAttached(context->getWorkspace() != nullptr); + + memcpy(bufferAsT(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + + + auto func = PRAGMA_THREADS_FOR{ + for (auto e = start; e < stop; e += increment) { + auto cdata = data + offsets[e]; + if (dtype == DataType::UTF16) { + memcpy(cdata, string[e], std::char_traits::length(string[e]) * sizeof(uint16_t)); + } + else if (dtype == DataType::UTF32) { + unicode::utf16to32(string[e], cdata, std::char_traits::length(string[e])); + } + else { + unicode::utf16to8(string[e], cdata, std::char_traits::length(string[e])); + } + } + }; + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + tickWriteHost(); + syncToDevice(); +} +///////////////////////////////////////////////////////////////////////// +NDArray::NDArray(const std::vector& shape, const std::vector& string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dtype)) + throw std::invalid_argument("NDArray::NDArray: invalid DataType, only string dataTypes have to be used"); + + if (shape::prodLong(shape.data(), shape.size()) != string.size()) + throw std::invalid_argument("NDArray::NDArray: Number of strings should match length of array"); + + for (auto str : string) { + if (!unicode::isStringValidU32(str.data(), str.data() + str.size())) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + } + + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(string.size()); + + std::vector offsets(string.size() + 1); + + Nd4jLong dataLength = 0; + for (int e = 0; e < string.size(); e++) { + offsets[e] = dataLength; + dataLength += [&] { + if (dtype == DataType::UTF16) + return unicode::offsetUtf32StringInUtf16(string[e].data(), string[e].size()); + if (dtype == DataType::UTF32) + return static_cast(sizeof(uint32_t) * string[e].size()); + return unicode::offsetUtf32StringInUtf16(string[e].data(), string[e].size()); + }(); + } + offsets[string.size()] = dataLength; + + _buffer = std::make_shared(headerLength + dataLength, dtype, context->getWorkspace(), true); + + _context = context; + _offset = 0; + + setShapeInfo(ShapeDescriptor(dtype, 'c', shape)); + + _isView = false; + + setAttached(context->getWorkspace() != nullptr); + + memcpy(bufferAsT(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + + auto func = PRAGMA_THREADS_FOR{ + for (auto e = start; e < stop; e += increment) { + auto cdata = data + offsets[e]; + if (dtype == DataType::UTF16) { + unicode::utf32to16(string[e].data(), cdata, string[e].size()); + } + else if (dtype == DataType::UTF32) { + memcpy(cdata, string[e].data(), string[e].size() * sizeof(uint32_t)); + } + else { + unicode::utf32to8(string[e].data(), cdata, string[e].size()); + } + } + }; + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + tickWriteHost(); + syncToDevice(); +} +///////////////////////////////////////////////////////////////////////// +NDArray::NDArray(const std::vector& shape, const std::vector& string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + + if (!DataTypeUtils::isS(dtype)) + throw std::invalid_argument("NDArray::NDArray: invalid DataType used"); + + if (shape::prodLong(shape.data(), shape.size()) != string.size()) + throw std::invalid_argument("NDArray::NDArray: Number of strings should match length of array"); + + for (const auto& str : string) { + if (!unicode::isStringValidU32(str, str + std::char_traits::length(str))) { + throw std::invalid_argument("NDArray::NDArray: invalid character in input string"); + } + } + + Nd4jLong headerLength = ShapeUtils::stringBufferHeaderRequirements(string.size()); + + std::vector offsets(string.size() + 1); + + Nd4jLong dataLength = 0; + for (int e = 0; e < string.size(); e++) { + offsets[e] = dataLength; + dataLength += [&] { + if (dtype == DataType::UTF16) + return unicode::offsetUtf32StringInUtf16(string[e], std::char_traits::length(string[e])); + if (dtype == DataType::UTF32) + return static_cast(sizeof(uint32_t) * std::char_traits::length(string[e])); + return unicode::offsetUtf32StringInUtf16(string[e], std::char_traits::length(string[e])); + }(); + } + offsets[string.size()] = dataLength; + + _buffer = std::make_shared(headerLength + dataLength, dtype, context->getWorkspace(), true); + + _context = context; + _offset = 0; + + setShapeInfo(ShapeDescriptor(dtype, 'c', shape)); + + _isView = _length * DataTypeUtils::sizeOf(_dataType) < _buffer->getLenInBytes(); + + setAttached(context->getWorkspace() != nullptr); + + memcpy(bufferAsT(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); + + auto data = reinterpret_cast(bufferAsT() + headerLength); + + auto func = PRAGMA_THREADS_FOR{ + for (auto e = start; e < stop; e += increment) { + auto cdata = data + offsets[e]; + if (dtype == DataType::UTF16) { + unicode::utf32to16(string[e], cdata, std::char_traits::length(string[e])); + } + else if (dtype == DataType::UTF32) { + memcpy(cdata, string[e], std::char_traits::length(string[e]) * sizeof(uint32_t)); + } + else { + unicode::utf32to8(string[e], cdata, std::char_traits::length(string[e])); + } + } + }; + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + tickWriteHost(); + syncToDevice(); +} //////////////////////////////////////////////////////////////////////// // assignment operator @@ -329,7 +887,9 @@ bool NDArray::isC() const { ////////////////////////////////////////////////////////////////////////// bool NDArray::isS() const { - return dataType() == DataType::UTF8; + return (dataType() == DataType::UTF8 || + dataType() == DataType::UTF16 || + dataType() == DataType::UTF32); } ////////////////////////////////////////////////////////////////////////// @@ -407,7 +967,7 @@ std::string NDArray::asString(Nd4jLong limit) { os << toStringValue(this->e(e)); else if (this->isB()) os << toStringValue(this->e(e)); - else if (this->isS()) + else if (this->isS()) // todo add utf16 and utf32 os << this->e(e); if (e < limit - 1) os << ", "; @@ -477,8 +1037,6 @@ std::vector NDArray::getShapeInfoAsVector() { //////////////////////////////////////////////////////////////////////// std::vector NDArray::asByteVector() { - - if (isS()) { // string data type requires special treatment syncToHost(); @@ -1066,8 +1624,17 @@ void NDArray::printBuffer(const char* msg, Nd4jLong limit, const bool sync) cons if (e < limit - 1) printf(", "); } - } + } else if (this->isS()) { + // todo do we need this print offsets + /* + for (Nd4jLong e = 0; e < limit; e++) { + printf("\"%lld\"", this->getOffset(e)); + if (e < limit - 1) + printf(", "); + } + printf("]\n["); + */ for (Nd4jLong e = 0; e < limit; e++) { printf("\"%s\"", this->e(e).c_str()); if (e < limit - 1) @@ -1123,8 +1690,9 @@ static void printFormatted(NDArray const* arr, int depth, int limit) { printf("%lld, ", arr->e(i)); else if (arr->isB()) printf("%s, ", arr->e(i)?"true":"false"); - else if (arr->isS()) + else if (arr->isS()) { printf("\"%s\", ", arr->e(i).c_str()); + } } printf("]\n"); } @@ -1149,8 +1717,9 @@ static void printFormatted(NDArray const* arr, int depth, int limit) { printf("%lld", arr->e(row, col)); else if (arr->isB()) printf("%s", arr->e(row, col)?"true":"false"); - else if (arr->isS()) + else if (arr->isS()) { printf("\"%s\"", arr->e(row * cols + col).c_str()); + } } if (row < rows - 1) printf("]\n"); @@ -1204,6 +1773,8 @@ void NDArray::printIndexedBuffer(const char* msg, Nd4jLong limit) const { printf("%s\n", this->e(0)?"true":"false"); } else if (this->isS()) { + // todo do we need this + // printf("\"%lld\"\n", this->getOffset(e)); printf("\"%s\"\n", this->e(0).c_str()); } } @@ -1708,9 +2279,8 @@ NDArray NDArray::subarray(const Intervals& idx) const { template NDArray NDArray::asT() const{ - auto result = isScalar() ? NDArray('c', {}, {0.}, DataTypeUtils::fromT(), this->getContext()) : NDArray(ordering(), getShapeAsVector(), DataTypeUtils::fromT(), this->getContext()); - auto l = this->lengthOf(); - + auto result = isScalar() ? NDArray('c', {}, std::vector{0.}, DataTypeUtils::fromT(), this->getContext()) : NDArray(ordering(), getShapeAsVector(), DataTypeUtils::fromT(), this->getContext()); + NDArray::prepareSpecialUse({&result}, {this}); NativeOpExecutioner::execTransformAny(getContext(), transform::AnyOps::Assign, getBuffer(), getShapeInfo(), getSpecialBuffer(), getSpecialShapeInfo(), result.getBuffer(), result.getShapeInfo(), result.getSpecialBuffer(), result.getSpecialShapeInfo(), nullptr, nullptr, nullptr); NDArray::registerSpecialUse({&result}, {this}); @@ -1719,20 +2289,145 @@ NDArray NDArray::asT() const{ } BUILD_SINGLE_TEMPLATE(template ND4J_EXPORT NDArray NDArray::asT, () const, LIBND4J_TYPES); +////////////////////////////////////////////////////////////////////////// +template +NDArray NDArray::asS() const { + + if (!isS()) + throw std::runtime_error("NDArray::asS: you can use this method only for String array!"); + + auto dtype = DataTypeUtils::fromT(); + + if (!(DataTypeUtils::isS(dtype))) + throw std::invalid_argument("NDArray::asS: invalid DataType used"); + + if (dtype == dataType()) { + + Nd4jLong offsetsLength = ShapeUtils::stringBufferHeaderRequirements(lengthOf()); + const auto nInputoffsets = bufferAsT(); + std::shared_ptr pBuffer = std::make_shared(offsetsLength + nInputoffsets[lengthOf()], dtype, getContext()->getWorkspace(), true); + + NDArray res(pBuffer, ShapeDescriptor(dtype, ordering(), getShapeAsVector()), getContext()); + res.setAttached(getContext()->getWorkspace() != nullptr); + + preparePrimaryUse({ &res }, { this }); + memcpy(res.bufferAsT(), nInputoffsets, offsetsLength); + auto data = res.bufferAsT() + offsetsLength; + const auto inData = bufferAsT() + offsetsLength; + memcpy(data, inData, nInputoffsets[lengthOf()]); + + registerPrimaryUse({ &res }, { this }); + return res; + } + + Nd4jLong offsetsLength = ShapeUtils::stringBufferHeaderRequirements(lengthOf()); + + std::vector offsets(lengthOf() + 1); + + const auto nInputoffsets = bufferAsT(); + + Nd4jLong start = 0, stop = 0; + Nd4jLong dataLength = 0; + + auto data = bufferAsT() + offsetsLength; + for (int e = 0; e < lengthOf(); e++) { + offsets[e] = dataLength; + start = nInputoffsets[e]; + stop = nInputoffsets[e + 1]; + if (dataType() == DataType::UTF8) { + dataLength += (dtype == DataType::UTF16) ? unicode::offsetUtf8StringInUtf16(data + start, stop) + : unicode::offsetUtf8StringInUtf32(data + start, stop); + } + else if (dataType() == DataType::UTF16) { + dataLength += (dtype == DataType::UTF32) ? unicode::offsetUtf16StringInUtf32(data + start, (stop / sizeof(char16_t)) ) + : unicode::offsetUtf16StringInUtf8(data + start, (stop / sizeof(char16_t))); + } + else { + dataLength += (dtype == DataType::UTF16) ? unicode::offsetUtf32StringInUtf16(data + start, (stop / sizeof(char32_t))) + : unicode::offsetUtf32StringInUtf8(data + start, (stop / sizeof(char32_t))); + } + } + offsets[lengthOf()] = dataLength; + + std::shared_ptr pBuffer = std::make_shared(offsetsLength + dataLength, dtype, getContext()->getWorkspace(), true); + + NDArray res(pBuffer, ShapeDescriptor(dtype, ordering(), getShapeAsVector()), getContext()); + res.setAttached(getContext()->getWorkspace() != nullptr); + + preparePrimaryUse({ &res }, { this }); + + memcpy(res.bufferAsT(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); + + auto outData = res.bufferAsT() + offsetsLength; + const auto inData = bufferAsT() + offsetsLength; + + auto func = PRAGMA_THREADS_FOR{ + for (int e = start; e < stop; e += increment) { + auto cdata = outData + offsets[e]; + auto end = nInputoffsets[e + 1]; + auto idata = inData + nInputoffsets[e]; + if (dtype == DataType::UTF16) { + if (dataType() == DataType::UTF8) { + unicode::utf8to16(idata, outData, end); + } + else { + unicode::utf32to16(idata, outData, (end / sizeof(char32_t))); + } + } + else if (dtype == DataType::UTF32) { + if (dataType() == DataType::UTF8) { + unicode::utf8to32(idata, cdata, end); + } + else { + unicode::utf16to32(idata, outData, (end / sizeof(char16_t))); + } + } + else { + if (dataType() == DataType::UTF16) { + unicode::utf16to8(idata, outData, (end / sizeof(char16_t))); + } + else { + unicode::utf32to8(idata, outData, (end / sizeof(char32_t))); + } + } + } + }; + + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + registerPrimaryUse({ &res }, { this }); + + return res; +} +BUILD_SINGLE_TEMPLATE(template ND4J_EXPORT NDArray NDArray::asS, () const, LIBND4J_STRINGTYPES); + //////////////////////////////////////////////////////////////////////// NDArray NDArray::asT(DataType dtype) const { - if (isS()) - throw std::runtime_error("NDArray::asT: you can't use this method on String array!"); + + if (isS() && !DataTypeUtils::isS(dtype)) + throw std::runtime_error("NDArray::asT: you can't use this method on String array with not string DataType!"); - BUILD_SINGLE_SELECTOR(dtype, return asT, (), LIBND4J_TYPES); + if (!isS() && DataTypeUtils::isS(dtype)) + throw std::runtime_error("NDArray::asT: you can't use this method on not String array with string DataType!"); + + if (isS()){ + BUILD_SINGLE_SELECTOR(dtype, return asS, (), LIBND4J_STRINGTYPES); + } else { + BUILD_SINGLE_SELECTOR(dtype, return asT, (), LIBND4J_TYPES); + } return NDArray(); } //////////////////////////////////////////////////////////////////////// NDArray NDArray::cast(DataType dtype) const { - if (isS()) - throw std::runtime_error("NDArray::cast: you can't use this method on String array!"); + + if (isS() && !DataTypeUtils::isS(dtype)) + throw std::runtime_error("NDArray::cast: you can't use this method on String array with not string DataType!"); + + if (!isS() && DataTypeUtils::isS(dtype)) + throw std::runtime_error("NDArray::cast: you can't use this method on not String array with string DataType!"); + return this->asT(dtype); } @@ -2765,14 +3460,44 @@ NDArray NDArray::dup(const char newOrder) const { char order = newOrder == 'a' ? ordering() : newOrder; // for now string arrays require special treatment - if (dataType() == DataType::UTF8) { + if (isS()) { + if (dataType() == DataType::UTF8) { + std::vector strings(lengthOf()); + + auto func = PRAGMA_THREADS_FOR{ + for (auto i = start; i < stop; i += increment) { + strings[i] = std::move(this->e(i)); + } + }; - std::vector strings(lengthOf()); - for (int e = 0; e < lengthOf(); e++) - strings[e] = this->e(e); + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); - auto result = NDArrayFactory::string(order, getShapeAsVector(), strings, getContext()); - return result; + return NDArray(getShapeAsVector(), strings, dataType(), getContext()); + } + if (dataType() == DataType::UTF16) { + std::vector strings(lengthOf()); + + auto func = PRAGMA_THREADS_FOR{ + for (auto i = start; i < stop; i += increment) { + strings[i] = std::move(this->e(i)); + } + }; + + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + return NDArray(getShapeAsVector(), strings, dataType(), getContext()); + } + + std::vector strings(lengthOf()); + auto func = PRAGMA_THREADS_FOR{ + for (auto i = start; i < stop; i += increment) { + strings[i] = std::move(this->e(i)); + } + }; + + samediff::Threads::parallel_for(func, 0, lengthOf(), 1); + + return NDArray(getShapeAsVector(), strings, dataType(), getContext()); } NDArray result(order, isScalar() ? std::vector({0}) : getShapeAsVector(), dataType(), getContext()); @@ -2796,12 +3521,33 @@ bool NDArray::equalsTo(const NDArray *other, double eps) const { if (isS()) { // string is special case, we'll compare them one by one, considering both arrays are guaranteed to have the same length - for (int e = 0; e < this->lengthOf(); e++) { - auto s1 = this->e(e); - auto s2 = other->e(e); + + if (dataType() == DataType::UTF8) { + for (int e = 0; e < this->lengthOf(); e++) { + auto s1 = this->e(e); + auto s2 = other->e(e); - if (s1 != s2) - return false; + if (s1 != s2) + return false; + } + } + else if (dataType() == DataType::UTF16) { + for (int e = 0; e < this->lengthOf(); e++) { + auto s1 = this->e(e); + auto s2 = other->e(e); + + if (s1 != s2) + return false; + } + } + else { + for (int e = 0; e < this->lengthOf(); e++) { + auto s1 = this->e(e); + auto s2 = other->e(e); + + if (s1 != s2) + return false; + } } return true; @@ -2836,15 +3582,31 @@ std::string NDArray::e(const Nd4jLong i) const { if (!isS()) throw std::runtime_error("Can't get std::string out of non-string array"); + if (i == lengthOf()) + throw std::runtime_error("Can't get std::string for index out of range"); + + + if (this->dataType() == DataType::UTF16) { + auto u16 = this->e(i); + std::string s; + StringUtils::u16StringToU8String(u16, s); + return s; + } + + if (this->dataType() == DataType::UTF32) { + auto u32 = this->e(i); + std::string s; + StringUtils::u32StringToU8String(u32, s); + return s; + } + NDArray::preparePrimaryUse({}, {this}); - // getting "virtual" offset. it's not real though,since it doesn't take lengths into account - auto offset = getOffset(i); - auto offsets = reinterpret_cast(getBuffer()); + auto offsets = bufferAsT(); auto offsetsLength = ShapeUtils::stringBufferHeaderRequirements(lengthOf()); - auto start = offsets[offset]; - auto end = offsets[offset + 1]; - auto data = static_cast(getBuffer()) + offsetsLength + start; + auto start = offsets[i]; + auto end = offsets[i + 1]; + auto data = bufferAsT() + offsetsLength + start; std::string r(reinterpret_cast(data), (end - start)); @@ -2853,6 +3615,83 @@ std::string NDArray::e(const Nd4jLong i) const { return r; } +template <> +std::u16string NDArray::e(const Nd4jLong i) const { + + if (!isS()) + throw std::runtime_error("Can't get std::u16string out of non-string array"); + + if(i == lengthOf()) + throw std::runtime_error("Can't get std::u16string for index out of range"); + + if (this->dataType() == DataType::UTF8) { + auto u = this->e(i); + std::u16string s; + StringUtils::u8StringToU16String(u, s); + return s; + } + + if (this->dataType() == DataType::UTF32) { + auto u32 = this->e(i); + std::u16string s; + StringUtils::u32StringToU16String(u32, s); + return s; + } + + NDArray::preparePrimaryUse({}, { this }); + + auto offsets = bufferAsT(); + Nd4jLong offsetsLength = ShapeUtils::stringBufferHeaderRequirements(lengthOf()); + Nd4jLong start = offsets[i]; + Nd4jLong end = offsets[i + 1]; + auto data = bufferAsT() + offsetsLength + start; + + std::u16string r(reinterpret_cast(data), (end - start) / sizeof(char16_t)); + + registerPrimaryUse({}, { this }); + + return r; +} + +template <> +std::u32string NDArray::e(const Nd4jLong i) const { + + if (!isS()) + throw std::runtime_error("Can't get std::u32string out of non-string array"); + + if (i == lengthOf()) + throw std::runtime_error("Can't get std::u32string for index out of range"); + + if (this->dataType() == DataType::UTF8) { + auto u = this->e(i); + std::u32string s; + StringUtils::u8StringToU32String(u, s); + return s; + } + + if (this->dataType() == DataType::UTF16) { + auto u16 = this->e(i); + std::u32string s; + StringUtils::u16StringToU32String(u16, s); + return s; + } + + NDArray::preparePrimaryUse({}, { this }); + + auto offsets = bufferAsT(); + Nd4jLong offsetsLength = ShapeUtils::stringBufferHeaderRequirements(lengthOf()); + Nd4jLong start = offsets[i]; + Nd4jLong end = offsets[i + 1]; + + auto data = bufferAsT() + offsetsLength + start; + + std::u32string r(reinterpret_cast(data), (end - start) / sizeof(char32_t)); + + registerPrimaryUse({}, { this }); + + return r; +} + ////////////////////////////////////////////////////////////////////////// template <> utf8string NDArray::e(const Nd4jLong i) const { diff --git a/libnd4j/blas/NDArrayFactory.h b/libnd4j/blas/NDArrayFactory.h index cdd8d9f9f..5e979f1d8 100644 --- a/libnd4j/blas/NDArrayFactory.h +++ b/libnd4j/blas/NDArrayFactory.h @@ -1,5 +1,6 @@ /******************************************************************************* * Copyright (c) 2015-2018 Skymind, Inc. + * Copyright (c) 2019-2020 Konduit K.K. * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -16,6 +17,7 @@ // // Created by raver119 on 2018-09-16. +// @author Oleg Semeniv // #ifndef DEV_TESTS_NDARRAYFACTORY_H @@ -106,25 +108,72 @@ namespace nd4j { template static NDArray create(char order, const std::vector &shape, const std::initializer_list& data, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - static NDArray string(const char *string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + /** + * This factory create array from utf8 string + * @return NDArray default dataType UTF8 + */ + static NDArray string(const char *string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray* string_(const char *string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray* string_(const std::string &string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray string(const std::string& string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); - static NDArray* string_(const char *string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + /** + * This factory create array from utf16 string + * @return NDArray default dataType UTF16 + */ + static NDArray string(const char16_t* u16string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_(const char16_t* u16string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_(const std::u16string& u16string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string(const std::u16string& u16string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + + /** + * This factory create array from utf32 string + * @return NDArray default dataType UTF32 + */ + static NDArray string(const char32_t* u32string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_(const char32_t* u32string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_(const std::u32string& u32string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string(const std::u32string& u32string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); - static NDArray string(const std::string &string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + /** + * This factory create array from vector of utf8 strings + * @return NDArray default dataType UTF8 + */ + static NDArray string( const std::vector &shape, const std::initializer_list &strings, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray string( const std::vector &shape, const std::initializer_list &string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray string( const std::vector &shape, const std::vector &strings, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray string( const std::vector &shape, const std::vector &string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray* string_( const std::vector &shape, const std::initializer_list &strings, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray* string_( const std::vector &shape, const std::initializer_list &string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray* string_( const std::vector &shape, const std::vector &strings, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + static NDArray* string_( const std::vector &shape, const std::vector &string, nd4j::DataType dtype = nd4j::DataType::UTF8, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - static NDArray* string_(const std::string &string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + /** + * This factory create array from vector of utf16 strings + * @return NDArray default dataType UTF16 + */ + static NDArray string( const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string( const std::vector& shape, const std::vector& strings, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string( const std::vector& shape, const std::vector& string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::vector& strings, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::vector& string, nd4j::DataType dtype = nd4j::DataType::UTF16, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); - static NDArray string(char order, const std::vector &shape, const std::initializer_list &strings, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - static NDArray string(char order, const std::vector &shape, const std::initializer_list &string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); + /** + * This factory create array from vector of utf32 strings + * @return NDArray default dataType UTF32 + */ + static NDArray string( const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string( const std::vector& shape, const std::vector& strings, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray string( const std::vector& shape, const std::vector& string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::vector& strings, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); + static NDArray* string_( const std::vector& shape, const std::vector& string, nd4j::DataType dtype = nd4j::DataType::UTF32, nd4j::LaunchContext* context = nd4j::LaunchContext::defaultContext()); - static NDArray string(char order, const std::vector &shape, const std::vector &strings, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - static NDArray string(char order, const std::vector &shape, const std::vector &string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - - static NDArray* string_(char order, const std::vector &shape, const std::initializer_list &strings, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - static NDArray* string_(char order, const std::vector &shape, const std::initializer_list &string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - - static NDArray* string_(char order, const std::vector &shape, const std::vector &strings, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); - static NDArray* string_(char order, const std::vector &shape, const std::vector &string, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); static ResultSet createSetOfArrs(const Nd4jLong numOfArrs, const void* buffer, const Nd4jLong* shapeInfo, const Nd4jLong* offsets, nd4j::LaunchContext * context = nd4j::LaunchContext ::defaultContext()); diff --git a/libnd4j/blas/cpu/NDArrayFactory.cpp b/libnd4j/blas/cpu/NDArrayFactory.cpp index 54cc6bba8..738dccdbe 100644 --- a/libnd4j/blas/cpu/NDArrayFactory.cpp +++ b/libnd4j/blas/cpu/NDArrayFactory.cpp @@ -1,5 +1,6 @@ /******************************************************************************* * Copyright (c) 2015-2018 Skymind, Inc. + * Copyright (c) 2019-2020 Konduit K.K. * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -16,6 +17,7 @@ // // Created by GS on 2018-12-20. +// @author Oleg Semeniv // #include @@ -25,6 +27,9 @@ #include #include + +#include + namespace nd4j { //////////////////////////////////////////////////////////////////////// @@ -85,45 +90,6 @@ namespace nd4j { template ND4J_EXPORT NDArray NDArrayFactory::create(const char order, const std::vector &shape, const std::vector& data, nd4j::LaunchContext * context); template ND4J_EXPORT NDArray NDArrayFactory::create(const char order, const std::vector &shape, const std::vector& data, nd4j::LaunchContext * context); - NDArray NDArrayFactory::string(const char *str, nd4j::LaunchContext * context) { - std::string s(str); - return string(s, context); - } - - NDArray* NDArrayFactory::string_(const char *str, nd4j::LaunchContext * context) { - return string_(std::string(str), context); - } - - NDArray NDArrayFactory::string(const std::string &str, nd4j::LaunchContext * context) { - - auto headerLength = ShapeUtils::stringBufferHeaderRequirements(1); - - std::shared_ptr pBuffer = std::make_shared(headerLength + str.length(), DataType::UTF8, context->getWorkspace(), true); - - NDArray res(pBuffer, ShapeDescriptor::scalarDescriptor(DataType::UTF8), context); - - int8_t* buffer = reinterpret_cast(res.getBuffer()); - - auto offsets = reinterpret_cast(buffer); - offsets[0] = 0; - offsets[1] = str.length(); - - auto data = buffer + headerLength; - - memcpy(data, str.c_str(), str.length()); - - res.tickWriteHost(); - res.syncToDevice(); - - return res; - } - - NDArray* NDArrayFactory::string_(const std::string &str, nd4j::LaunchContext * context) { - auto res = new NDArray(); - *res = NDArrayFactory::string(str, context); - return res; - } - //////////////////////////////////////////////////////////////////////// template NDArray* NDArrayFactory::create_(const char order, const std::vector &shape, nd4j::LaunchContext * context) { @@ -551,91 +517,175 @@ template ND4J_EXPORT NDArray NDArrayFactory::create(uint8_t * buffer, const char template ND4J_EXPORT NDArray NDArrayFactory::create(int8_t* buffer, const char order, const std::initializer_list& shape, nd4j::LaunchContext * context); template ND4J_EXPORT NDArray NDArrayFactory::create(int16_t* buffer, const char order, const std::initializer_list& shape, nd4j::LaunchContext * context); - - NDArray NDArrayFactory::string(char order, const std::vector &shape, const std::initializer_list &strings, nd4j::LaunchContext * context) { - std::vector vec(strings); - return NDArrayFactory::string(order, shape, vec, context); - } - - NDArray NDArrayFactory::string(char order, const std::vector &shape, const std::vector &strings, nd4j::LaunchContext * context) { - std::vector vec(strings.size()); - int cnt = 0; - for (auto s:strings) - vec[cnt++] = std::string(s); - - return NDArrayFactory::string(order, shape, vec, context); - } - - - NDArray NDArrayFactory::string(char order, const std::vector &shape, const std::initializer_list &string, nd4j::LaunchContext * context) { - std::vector vec(string); - return NDArrayFactory::string(order, shape, vec, context); - } - - NDArray* NDArrayFactory::string_(char order, const std::vector &shape, const std::initializer_list &strings, nd4j::LaunchContext * context) { - std::vector vec(strings); - return NDArrayFactory::string_(order, shape, vec, context); - } - - NDArray* NDArrayFactory::string_(char order, const std::vector &shape, const std::vector &strings, nd4j::LaunchContext * context) { - std::vector vec(strings.size()); - int cnt = 0; - for (auto s:strings) - vec[cnt++] = std::string(s); - - return NDArrayFactory::string_(order, shape, vec, context); - } - - - NDArray* NDArrayFactory::string_(char order, const std::vector &shape, const std::initializer_list &string, nd4j::LaunchContext * context) { - std::vector vec(string); - return NDArrayFactory::string_(order, shape, vec, context); - } - - NDArray NDArrayFactory::string(char order, const std::vector &shape, const std::vector &string, nd4j::LaunchContext * context) { - - if (context == nullptr) - context = nd4j::LaunchContext ::defaultContext(); - - auto headerLength = ShapeUtils::stringBufferHeaderRequirements(string.size()); - - std::vector offsets(string.size() + 1); - Nd4jLong dataLength = 0; - for (int e = 0; e < string.size(); e++) { - offsets[e] = dataLength; - dataLength += string[e].length(); - } - offsets[string.size()] = dataLength; - - std::shared_ptr pBuffer = std::make_shared(headerLength + dataLength, DataType::UTF8, context->getWorkspace(), true); - - NDArray res(pBuffer, ShapeDescriptor(DataType::UTF8, order, shape), context); - res.setAttached(context->getWorkspace() != nullptr); - - if (res.lengthOf() != string.size()) - throw std::invalid_argument("Number of strings should match length of array"); - - memcpy(res.buffer(), offsets.data(), offsets.size() * sizeof(Nd4jLong)); - - auto data = static_cast(res.buffer()) + headerLength; - int resLen = res.lengthOf(); - for (int e = 0; e < resLen; e++) { - auto length = offsets[e+1] - offsets[e]; - auto cdata = data + offsets[e]; - memcpy(cdata, string[e].c_str(), string[e].length()); - } - - res.tickWriteHost(); - res.syncToDevice(); - - return res; - } - - NDArray* NDArrayFactory::string_(char order, const std::vector &shape, const std::vector &string, nd4j::LaunchContext * context) { - auto res = new NDArray(); - *res = NDArrayFactory::string(order, shape, string, context); - return res; - } - + ///////////////////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const char16_t* u16string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray(u16string, dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_(const char16_t* u16string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return string_(std::u16string(u16string), dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_(const std::u16string& u16string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + auto res = new NDArray(); + *res = NDArray(u16string, dtype, context); + return res; + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const std::u16string& u16string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray(u16string, dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const char32_t* u32string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray(u32string, dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_(const char32_t* u32string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return string_(std::u32string(u32string), dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_(const std::u32string& u32string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + auto res = new NDArray(); + *res = NDArray(u32string, dtype, context); + return res; + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const std::u32string& u32string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray(u32string, dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const char* str, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray(str, dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_(const char* str, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return string_(std::string(str), dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_(const std::string& str, nd4j::DataType dtype, nd4j::LaunchContext* context) { + auto res = new NDArray(); + *res = NDArray(str, dtype, context); + return res; + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const std::string& str, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray(str, dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const std::vector &shape, const std::initializer_list &strings, nd4j::DataType dataType, nd4j::LaunchContext * context) { + return NDArray(shape, std::vector(strings), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector &shape, const std::vector &strings, nd4j::DataType dataType, nd4j::LaunchContext * context) { + return NDArray( shape, strings, dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector &shape, const std::initializer_list &string, nd4j::DataType dataType, nd4j::LaunchContext * context) { + return NDArray( shape, std::vector(string), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector &shape, const std::initializer_list &strings, nd4j::DataType dataType, nd4j::LaunchContext * context) { + return NDArrayFactory::string_( shape, std::vector(strings), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector &shape, const std::vector &strings, nd4j::DataType dataType, nd4j::LaunchContext * context) { + std::vector vec(strings.size()); + int cnt = 0; + for (auto s:strings) + vec[cnt++] = std::string(s); + + return NDArrayFactory::string_( shape, vec, dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector &shape, const std::initializer_list &string, nd4j::DataType dataType, nd4j::LaunchContext * context) { + return NDArrayFactory::string_( shape, std::vector(string), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector &shape, const std::vector &string, nd4j::DataType dataType, nd4j::LaunchContext * context) { + return NDArray(shape, string, dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_(const std::vector &shape, const std::vector &string, nd4j::DataType dataType, nd4j::LaunchContext * context) { + auto res = new NDArray(); + *res = NDArray( shape, string, dataType, context); + return res; + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArray( shape, std::vector(strings), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector& shape, const std::vector& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArray( shape, strings, dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArray( shape, std::vector(string), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArrayFactory::string_( shape, std::vector(strings), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::vector& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + std::vector vec(strings.size()); + int cnt = 0; + for (auto s : strings) + vec[cnt++] = std::u16string(s); + + return NDArrayFactory::string_( shape, vec, dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArrayFactory::string_( shape, std::vector(string), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::vector& string, nd4j::DataType dataType, nd4j::LaunchContext* context) { + auto res = new NDArray(); + *res = NDArray( shape, string, dataType, context); + return res; + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector& shape, const std::vector& string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray( shape, string, dtype, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArray( shape, std::vector(strings), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector& shape, const std::vector& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArray( shape, strings, dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArray(shape, std::vector(string), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::initializer_list& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArrayFactory::string_( shape, std::vector(strings), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::vector& strings, nd4j::DataType dataType, nd4j::LaunchContext* context) { + std::vector vec(strings.size()); + int cnt = 0; + for (auto s : strings) + vec[cnt++] = std::u32string(s); + return NDArrayFactory::string_( shape, vec, dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::initializer_list& string, nd4j::DataType dataType, nd4j::LaunchContext* context) { + return NDArrayFactory::string_( shape, std::vector(string), dataType, context); + } + ///////////////////////////////////////////////////////////////////////// + NDArray* NDArrayFactory::string_( const std::vector& shape, const std::vector& string, nd4j::DataType dataType, nd4j::LaunchContext* context) { + auto res = new NDArray(); + *res = NDArray( shape, string, dataType, context); + return res; + } + ///////////////////////////////////////////////////////////////////////// + NDArray NDArrayFactory::string(const std::vector& shape, const std::vector& string, nd4j::DataType dtype, nd4j::LaunchContext* context) { + return NDArray( shape, string, dtype, context); + } } diff --git a/libnd4j/include/array/DataTypeUtils.h b/libnd4j/include/array/DataTypeUtils.h index 5d17c28b0..c307ecd4e 100644 --- a/libnd4j/include/array/DataTypeUtils.h +++ b/libnd4j/include/array/DataTypeUtils.h @@ -122,7 +122,7 @@ namespace nd4j { } FORCEINLINE bool DataTypeUtils::isS(nd4j::DataType dataType) { - return dataType == nd4j::DataType::UTF8; + return dataType == nd4j::DataType::UTF8 || dataType == nd4j::DataType::UTF16 || dataType == nd4j::DataType::UTF32; } FORCEINLINE bool DataTypeUtils::isZ(nd4j::DataType dataType) { @@ -370,6 +370,10 @@ FORCEINLINE std::string DataTypeUtils::asString(DataType dataType) { return std::string("UINT64"); case UTF8: return std::string("UTF8"); + case UTF16: + return std::string("UTF16"); + case UTF32: + return std::string("UTF32"); default: throw std::runtime_error("Unknown data type used"); } @@ -431,6 +435,8 @@ FORCEINLINE _CUDA_HD T DataTypeUtils::eps() { case nd4j::DataType::UINT16: return (size_t) 2; case nd4j::DataType::UTF8: + case nd4j::DataType::UTF16: + case nd4j::DataType::UTF32: case nd4j::DataType::INT32: case nd4j::DataType::UINT32: case nd4j::DataType::HALF2: @@ -455,6 +461,10 @@ FORCEINLINE _CUDA_HD T DataTypeUtils::eps() { return nd4j::DataType::BOOL; } else if (std::is_same::value) { return nd4j::DataType::UTF8; + } else if (std::is_same::value) { + return nd4j::DataType::UTF16; + } else if (std::is_same::value) { + return nd4j::DataType::UTF32; } else if (std::is_same::value) { return nd4j::DataType::FLOAT32; } else if (std::is_same::value) { diff --git a/libnd4j/include/graph/impl/FlatUtils.cpp b/libnd4j/include/graph/impl/FlatUtils.cpp index ec76cb4d2..f582220da 100644 --- a/libnd4j/include/graph/impl/FlatUtils.cpp +++ b/libnd4j/include/graph/impl/FlatUtils.cpp @@ -49,12 +49,11 @@ namespace nd4j { delete[] newShape; return NDArrayFactory::empty_(dtype, nullptr); } - + // TODO fix UTF16 and UTF32 if (dtype == UTF8) { bool isBe = BitwiseUtils::isBE(); bool canKeep = (isBe && flatArray->byteOrder() == nd4j::graph::ByteOrder_BE) || (!isBe && flatArray->byteOrder() == nd4j::graph::ByteOrder_LE); - auto order = shape::order(newShape); - + std::vector substrings(length); std::vector shapeVector(rank); for (int e = 0; e < rank; e++) @@ -88,8 +87,8 @@ namespace nd4j { delete[] offsets; delete[] newShape; - - return NDArrayFactory::string_(order, shapeVector, substrings); + // string order always 'c' + return NDArrayFactory::string_(shapeVector, substrings); } diff --git a/libnd4j/include/helpers/ShapeUtils.h b/libnd4j/include/helpers/ShapeUtils.h index 5f76c11b5..c99a0b0de 100644 --- a/libnd4j/include/helpers/ShapeUtils.h +++ b/libnd4j/include/helpers/ShapeUtils.h @@ -171,7 +171,10 @@ namespace nd4j { * @param numStrings * @return */ - static Nd4jLong stringBufferHeaderRequirements(Nd4jLong numStrings); + static FORCEINLINE Nd4jLong stringBufferHeaderRequirements(Nd4jLong numStrings) { + // we store +1 offset + return (numStrings + 1) * sizeof(Nd4jLong); + } /* * check whether arr1/arr2 is sub-array of arr2/arr1, diff --git a/libnd4j/include/helpers/StringUtils.h b/libnd4j/include/helpers/StringUtils.h index 2a562de4b..7a0e2a960 100644 --- a/libnd4j/include/helpers/StringUtils.h +++ b/libnd4j/include/helpers/StringUtils.h @@ -1,5 +1,6 @@ /******************************************************************************* * Copyright (c) 2015-2018 Skymind, Inc. + * Copyright (c) 2019-2020 Konduit K.K. * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -16,6 +17,7 @@ // // Created by raver119 on 20/04/18. +// @author Oleg Semeniv // #ifndef LIBND4J_STRINGUTILS_H @@ -27,6 +29,7 @@ #include #include #include +#include namespace nd4j { class ND4J_EXPORT StringUtils { @@ -85,6 +88,55 @@ namespace nd4j { * @return */ static std::vector split(const std::string &haystack, const std::string &delimiter); + + + /** + * This method convert u8 string to u16 + * @param const reference to input string + * @param reference to output u16string + * @return boolean status + */ + static bool u8StringToU16String(const std::string& u8, std::u16string& u16); + + /** + * This method convert u8 string to u32 + * @param const reference to input string + * @param reference to output u32string + * @return boolean status + */ + static bool u8StringToU32String(const std::string& u8, std::u32string& u32); + + /** + * This method convert u16 string to u32 + * @param const reference to input u16string + * @param reference to output u32string + * @return boolean status + */ + static bool u16StringToU32String(const std::u16string& u16, std::u32string& u32); + + /** + * This method convert u16 string to u8 string + * @param const reference to input u16string + * @param reference to output string + * @return boolean status + */ + static bool u16StringToU8String(const std::u16string& u16, std::string& u8); + + /** + * This method convert u32 string to u16 string + * @param const reference to input u32string + * @param reference to output u16string + * @return boolean status + */ + static bool u32StringToU16String(const std::u32string& u32, std::u16string& u16); + + /** + * This method convert u32 string to u8 string + * @param const reference to input u32string + * @param reference to output string + * @return boolean status + */ + static bool u32StringToU8String(const std::u32string& u32, std::string& u8); }; } diff --git a/libnd4j/include/helpers/impl/ShapeUtils.cpp b/libnd4j/include/helpers/impl/ShapeUtils.cpp index b040eb73c..165ed5ffd 100644 --- a/libnd4j/include/helpers/impl/ShapeUtils.cpp +++ b/libnd4j/include/helpers/impl/ShapeUtils.cpp @@ -1019,15 +1019,6 @@ std::vector ShapeUtils::tadAxesForSimpleBroadcast(const NDArray& max, const return numOfMinTads == 1 ? maxTadDims : std::vector(); } - -Nd4jLong ShapeUtils::stringBufferHeaderRequirements(Nd4jLong numStrings) { - // we store +1 offset - auto base = numStrings + 1; - - // since we return number of bytes... - return base * sizeof(Nd4jLong); -} - //////////////////////////////////////////////////////////////////////////////// /* bool ShapeUtils::isSubArrayCase(const NDArray& arr1, const NDArray& arr2, std::vector& sameDims) { diff --git a/libnd4j/include/helpers/impl/StringUtils.cpp b/libnd4j/include/helpers/impl/StringUtils.cpp index faace2c63..045dcea73 100644 --- a/libnd4j/include/helpers/impl/StringUtils.cpp +++ b/libnd4j/include/helpers/impl/StringUtils.cpp @@ -1,5 +1,6 @@ /******************************************************************************* * Copyright (c) 2015-2018 Skymind, Inc. + * Copyright (c) 2019-2020 Konduit K.K. * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -16,6 +17,7 @@ // // Created by raver119 on 20/04/18. +// @author Oleg Semeniv // #include @@ -49,13 +51,8 @@ namespace nd4j { if (!array.isS()) throw nd4j::datatype_exception::build("StringUtils::byteLength expects one of String types;", array.dataType()); - uint64_t result = 0; - - // our buffer stores offsets, and the last value is basically number of bytes used auto buffer = array.bufferAsT(); - result = buffer[array.lengthOf()]; - - return result; + return buffer[array.lengthOf()]; } std::vector StringUtils::split(const std::string &haystack, const std::string &delimiter) { @@ -73,4 +70,89 @@ namespace nd4j { return output; } + + bool StringUtils::u8StringToU16String(const std::string& u8, std::u16string& u16) { + + if (u8.empty()) + return false; + + u16.resize(unicode::offsetUtf8StringInUtf16(u8.data(), u8.size()) / sizeof(char16_t)); + if (u8.size() == u16.size()) + u16.assign(u8.begin(), u8.end()); + else + return unicode::utf8to16(u8.data(), &u16[0], u8.size()); + + return true; + } + + bool StringUtils::u8StringToU32String(const std::string& u8, std::u32string& u32) { + + if (u8.empty()) + return false; + + u32.resize( unicode::offsetUtf8StringInUtf32(u8.data(), u8.size()) / sizeof(char32_t) ); + if (u8.size() == u32.size()) + u32.assign(u8.begin(), u8.end()); + else + return unicode::utf8to32(u8.data(), &u32[0], u8.size()); + + return true; + } + + bool StringUtils::u16StringToU32String(const std::u16string& u16, std::u32string& u32) { + + if (u16.empty()) + return false; + + u32.resize(unicode::offsetUtf16StringInUtf32(u16.data(), u16.size()) / sizeof(char32_t)); + if (u16.size() == u32.size()) + u32.assign(u16.begin(), u16.end()); + else + return unicode::utf16to32(u16.data(), &u32[0], u16.size()); + + return true; + } + + bool StringUtils::u16StringToU8String(const std::u16string& u16, std::string& u8) { + + if (u16.empty()) + return false; + + u8.resize(unicode::offsetUtf16StringInUtf8(u16.data(), u16.size())); + if (u16.size() == u8.size()) + u8.assign(u16.begin(), u16.end()); + else + return unicode::utf16to8(u16.data(), &u8[0], u16.size()); + + return true; + } + + bool StringUtils::u32StringToU16String(const std::u32string& u32, std::u16string& u16) { + + if (u32.empty()) + return false; + + u16.resize(unicode::offsetUtf32StringInUtf16(u32.data(), u32.size()) / sizeof(char16_t)); + if (u32.size() == u16.size()) + u16.assign(u32.begin(), u32.end()); + else + return unicode::utf32to16(u32.data(), &u16[0], u32.size()); + + return true; + } + + bool StringUtils::u32StringToU8String(const std::u32string& u32, std::string& u8) { + + if (u32.empty()) + return false; + + u8.resize(unicode::offsetUtf32StringInUtf8(u32.data(), u32.size())); + if (u32.size() == u8.size()) + u8.assign(u32.begin(), u32.end()); + else + return unicode::utf32to8(u32.data(), &u8[0], u32.size()); + + return true; + } + } diff --git a/libnd4j/include/helpers/impl/unicode.cpp b/libnd4j/include/helpers/impl/unicode.cpp new file mode 100644 index 000000000..2e49faf3e --- /dev/null +++ b/libnd4j/include/helpers/impl/unicode.cpp @@ -0,0 +1,456 @@ +/******************************************************************************* + * Copyright (c) 2015-2020 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +// +// @author Oleg Semeniv +// + +#include + +namespace nd4j { +namespace unicode { + + constexpr uint32_t ONEBYTEBOUND = 0x00000080; + constexpr uint32_t TWOBYTEBOUND = 0x00000800; + constexpr uint32_t THREEBYTEBOUND = 0x00010000; + constexpr uint16_t HIGHBYTEMIN = 0xd800u; + constexpr uint16_t HIGHBYTEMAX = 0xdbffu; + constexpr uint16_t TRAILBYTEMIN = 0xdc00u; + constexpr uint16_t TRAILBYTEMAX = 0xdfffu; + constexpr uint16_t HIGHBYTEOFFSET = HIGHBYTEMIN - (0x10000 >> 10); + constexpr uint32_t BYTEOFFSET = 0x10000u - (HIGHBYTEMIN << 10) - TRAILBYTEMIN; + // Maximum valid value for a Unicode code point + constexpr uint32_t CODEPOINTMAX = 0x0010ffffu; + + template + FORCEINLINE uint8_t castToU8(const T cp) { + return static_cast(0xff & cp); + } + + template + FORCEINLINE uint16_t castToU16(const T cp) { + return static_cast(0xffff & cp); + } + + template + FORCEINLINE uint32_t castToU32(const T cp) { + return static_cast(0xffffff & cp); + } + + template + FORCEINLINE bool isTrail(const T cp) { + return ((castToU8(cp) >> 6) == 0x2); + } + + template + FORCEINLINE bool isHighSurrogate(const T cp) { + return (cp & 0xfffffc00) == 0xd800; + } + + template + bool isLowSurrogate(const T cp) { + return (cp & 0xfffffc00) == 0xdc00; + } + + template + FORCEINLINE bool isLeadSurrogate(const T cp) { + return (cp >= HIGHBYTEMIN && cp <= HIGHBYTEMAX); + } + + template + FORCEINLINE bool isTrailSurrogate(const T cp) { + return (cp >= TRAILBYTEMIN && cp <= TRAILBYTEMAX); + } + + template + FORCEINLINE bool isSurrogateU8(const T cp) { + return (cp >= HIGHBYTEMIN && cp <= TRAILBYTEMAX); + } + + template + FORCEINLINE bool isSurrogateU16(const T cp) { + return ((cp - 0xd800u) < 2048u); + } + + template + FORCEINLINE bool isSymbolU8Valid(const T cp) { + return (cp <= CODEPOINTMAX && !isSurrogateU8(cp)); + } + + template + FORCEINLINE bool isSymbolValid(const T cp) { + return (cp <= CODEPOINTMAX); + } + + template + FORCEINLINE uint32_t surrogateU32(const T& high, const T& low) { + return (high << 10) + low - 0x35fdc00; + } + + template + Nd4jLong symbolLength(const T* it) { + uint8_t lead = castToU8(*it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + Nd4jLong symbolLength32(const T* it) { + auto lead = castToU32(*it); + if (lead < ONEBYTEBOUND) + return 1; + else if (lead < TWOBYTEBOUND) + return 2; + else if (lead < THREEBYTEBOUND) + return 3; + else if (lead <= CODEPOINTMAX) + return 4; + else + return 0; + } + + template + Nd4jLong symbolLength16(const T* it) { + + uint32_t lead = castToU16(*it); + if (!isLeadSurrogate(lead)) { + if (lead < ONEBYTEBOUND) + return 1; + else if (lead < TWOBYTEBOUND) + return 2; + else if (lead < THREEBYTEBOUND) + return 3; + else + return 0; + } + else { + return 4; + } + } + + Nd4jLong offsetUtf8StringInUtf32(const void* start, const void* end) { + + Nd4jLong count = 0; + for (auto it = static_cast(start); it != end; it++) { + auto length = symbolLength(it); + it += (length > 0) ? (length - 1) : 0; + count += 1; + } + return static_cast(count * sizeof(char32_t)); + } + + Nd4jLong offsetUtf16StringInUtf32(const void* start, const void* end) { + + Nd4jLong count = 0; + for (auto it = static_cast(start); it != end;) { + auto length = symbolLength16(it); + it += (4 == length) ? 2 : 1; + count += 1; + } + return static_cast(count*sizeof(char32_t)); + } + + Nd4jLong offsetUtf8StringInUtf16(const void* start, const void* end) { + + Nd4jLong count = 0; + for (auto it = static_cast(start); it != end; it++) { + auto length = symbolLength(it); + auto step = ((length > 0) ? (length - 1) : 0); + it += step; + count += (4 == length) ? 2 : 1; + } + return static_cast(count*sizeof(char16_t)); + } + + Nd4jLong offsetUtf16StringInUtf8(const void* start, const void* end) { + + Nd4jLong count = 0; + for (auto it = static_cast(start); it != end;) { + auto length = symbolLength16(it); + it += (4 == length) ? 2 : 1; + count += length; + } + return static_cast(count); + } + + Nd4jLong offsetUtf32StringInUtf16(const void* start, const void* end) { + + Nd4jLong count = 0; + for (auto it = static_cast(start); it != end; it++) { + auto length = symbolLength32(it); + count += (4 == length) ? 2 : 1;; + } + return static_cast(count*sizeof(char16_t)); + } + + Nd4jLong offsetUtf32StringInUtf8(const void* start, const void* end) { + + Nd4jLong count = 0; + for (auto it = static_cast(start); it != end; it++) { + count += symbolLength32(it); + } + return count; + } + + bool isStringValidU8(const void* start, const void* stop) { + for (auto it = static_cast(start); it != stop; it++) { + if (!isSymbolU8Valid( castToU8(*it) )) { + return false; + } + } + return true; + } + + bool isStringValidU16(const void* start, const void* stop) { + for (auto it = static_cast(start); it != stop; it++) { + if (!isSymbolValid( castToU32(*it) )) { + return false; + } + } + return true; + } + + bool isStringValidU32(const void* start, const void* stop) { + for (auto it = static_cast(start); it != stop; it++) { + if (!isSymbolValid( castToU32(*it) )) { + return false; + } + } + return true; + } + + void* utf16to8Ptr(const void* start, const void* end, void* res) { + + auto result = static_cast(res); + // result have to be pre-allocated + for (auto it = static_cast(start); it != end;) { + uint32_t cp = castToU16(*it++); + if (!isLeadSurrogate(cp)) { + if (cp < 0x80) { // for one byte + *(result++) = static_cast(cp); + } + else if (cp < 0x800) { // for two bytes + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else{ // for three bytes + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + } + else { + if (it != end) { + uint32_t trail_surrogate = castToU16(*it++); + if (isTrailSurrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + BYTEOFFSET; + } + // for four bytes + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + } + return result; + } + + void* utf8to16Ptr(const void* start, const void* end, void* res) { + + auto result = static_cast(res); + // result have to be pre-allocated + for (auto it = static_cast(start); it != end;) { + + auto nLength = symbolLength(it); + uint32_t cp = castToU8(*it++); + if (4 != nLength) { + if (2 == nLength) { + cp = ((cp << 6) & 0x7ff) + ((*it++) & 0x3f); + } + else if (3 == nLength) { + cp = ((cp << 12) & 0xffff) + ((castToU8(*it++) << 6) & 0xfff); + cp += (*it++) & 0x3f; + } + *(result++) = static_cast(cp); + } + else { + cp = ((cp << 18) & 0x1fffff) + ((castToU8(*it++) << 12) & 0x3ffff); + cp += (castToU8(*it++) << 6) & 0xfff; + cp += (*it++) & 0x3f; + //make a surrogate pair + *(result++) = static_cast((cp >> 10) + HIGHBYTEOFFSET); + *(result++) = static_cast((cp & 0x3ff) + TRAILBYTEMIN); + } + } + return result; + } + + void* utf32to8Ptr( const void* start, const void* end, void* result) { + + auto res = static_cast(result); + // result have to be pre-allocated + for (auto it = static_cast(start); it != end; it++) { + + if (*it < 0x80) // for one byte + *(res++) = static_cast(*it); + else if (*it < 0x800) { // for two bytes + *(res++) = static_cast((*it >> 6) | 0xc0); + *(res++) = static_cast((*it & 0x3f) | 0x80); + } + else if (*it < 0x10000) { // for three bytes + *(res++) = static_cast((*it >> 12) | 0xe0); + *(res++) = static_cast(((*it >> 6) & 0x3f) | 0x80); + *(res++) = static_cast((*it & 0x3f) | 0x80); + } + else { // for four bytes + *(res++) = static_cast((*it >> 18) | 0xf0); + *(res++) = static_cast(((*it >> 12) & 0x3f) | 0x80); + *(res++) = static_cast(((*it >> 6) & 0x3f) | 0x80); + *(res++) = static_cast((*it & 0x3f) | 0x80); + } + } + return result; + } + + void* utf8to32Ptr(const void* start, const void* end, void* res) { + + auto result = static_cast(res); + // result have to be pre-allocated + for (auto it = static_cast(start); it != end;) { + + auto nLength = symbolLength(it); + uint32_t cp = castToU8(*it++); + if (2 == nLength) { + cp = ((cp << 6) & 0x7ff) + ((*it++) & 0x3f); + } + else if (3 == nLength) { + cp = ((cp << 12) & 0xffff) + ((castToU8(*it++) << 6) & 0xfff); + cp += (*it++) & 0x3f; + } + else if (4 == nLength) { + cp = ((cp << 18) & 0x1fffff) + ((castToU8(*it++) << 12) & 0x3ffff); + cp += (castToU8(*it++) << 6) & 0xfff; + cp += (*it++) & 0x3f; + } + (*result++) = cp; + } + return result; + } + + void* utf16to32Ptr(const void* start, const void* end, void* res) { + + auto result = static_cast(res); + // result have to be pre-allocated + for (auto it = static_cast(start); it != end; it++) { + + uint32_t cpHigh = castToU32(*it); + if (!isSurrogateU16(cpHigh)) { + *result++ = cpHigh; + } + else { + it++; + uint32_t cpLow = castToU32(*it); + if (isHighSurrogate(cpHigh) && it != end && isLowSurrogate(cpLow)) { + *result++ = surrogateU32(cpHigh, cpLow); + } + } + } + return result; + } + + void* utf32to16Ptr(const void* start, const void* end, void* res) { + + auto result = static_cast(res); + // result have to be pre-allocate + for (auto it = static_cast(start); it != end; it++) { + + uint32_t cpHigh = castToU32(*it); + // todo check do we need this as we have pre-validation, if yes find out how to check u16 + if (cpHigh < 0 || cpHigh > 0x10FFFF || (cpHigh >= 0xD800 && cpHigh <= 0xDFFF)) { + // Invalid code point. Replace with sentinel, per Unicode standard: + *result++ = u'\uFFFD'; + } + else if (cpHigh < 0x10000UL) { // In the BMP. + *result++ = static_cast(cpHigh); + } + else { + *result++ = static_cast(((cpHigh - 0x10000UL) / 0x400U) + 0xD800U); + *result++ = static_cast(((cpHigh - 0x10000UL) % 0x400U) + 0xDC00U); + } + } + return result; + } + + Nd4jLong offsetUtf8StringInUtf32(const void* input, uint32_t nInputSize) { + return offsetUtf8StringInUtf32(input, static_cast(input) + nInputSize); + } + + Nd4jLong offsetUtf16StringInUtf32(const void* input, uint32_t nInputSize) { + return offsetUtf16StringInUtf32(input, static_cast(input) + nInputSize); + } + + Nd4jLong offsetUtf8StringInUtf16(const void* input, uint32_t nInputSize) { + return offsetUtf8StringInUtf16(input, static_cast(input) + nInputSize); + } + + Nd4jLong offsetUtf16StringInUtf8(const void* input, uint32_t nInputSize) { + return offsetUtf16StringInUtf8(input, static_cast(input) + nInputSize); + } + + Nd4jLong offsetUtf32StringInUtf8(const void* input, uint32_t nInputSize) { + return offsetUtf32StringInUtf8(input, static_cast(input) + nInputSize); + } + + Nd4jLong offsetUtf32StringInUtf16(const void* input, const uint32_t nInputSize) { + return offsetUtf32StringInUtf16(input, static_cast(input) + nInputSize); + } + + bool utf8to16(const void* input, void* output, uint32_t nInputSize) { + return utf8to16Ptr(input, static_cast(input) + nInputSize, output); + } + + bool utf8to32(const void* input, void* output, uint32_t nInputSize) { + return utf8to32Ptr(input, static_cast(input) + nInputSize, output); + } + + bool utf16to32(const void* input, void* output, uint32_t nInputSize) { + return utf16to32Ptr(input, static_cast(input) + nInputSize, output); + } + + bool utf16to8(const void* input, void* output, uint32_t nInputSize) { + return utf16to8Ptr(input, static_cast(input) + nInputSize, output); + } + + bool utf32to16(const void* input, void* output, uint32_t nInputSize) { + return utf32to16Ptr(input, static_cast(input) + nInputSize, output); + } + + bool utf32to8(const void* input, void* output, const Nd4jLong nInputSize) { + return utf32to8Ptr(input, static_cast(input) + nInputSize, output); + } + + } + +} + diff --git a/libnd4j/include/helpers/unicode.h b/libnd4j/include/helpers/unicode.h new file mode 100644 index 000000000..239b71201 --- /dev/null +++ b/libnd4j/include/helpers/unicode.h @@ -0,0 +1,189 @@ +/******************************************************************************* + * Copyright (c) 2019-2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +// +// @author Oleg Semeniv +// + +#ifndef LIBND4J_UNICODE_H +#define LIBND4J_UNICODE_H + +#include + +namespace nd4j { +namespace unicode { + + /** + * This method calculate u16 offset based on utf8 + * @param const pointer to the utf8 string start point + * @param size of the string + * @return offset of utf16 + */ + Nd4jLong offsetUtf8StringInUtf16(const void* start, const void* end); + + /** + * This method calculate u8 offset based on utf16 + * @param const pointer to the utf16 string start point + * @param size of the string + * @return offset of utf8 + */ + Nd4jLong offsetUtf16StringInUtf8(const void* start, const void* end); + + /** + * This method calculate u32 offset based on utf16 + * @param const pointer to the utf16 string start point + * @param size of the string + * @return offset of utf32 + */ + Nd4jLong offsetUtf32StringInUtf16(const void* start, const void* end); + + /** + * This method calculate u32 offset based on utf8 + * @param const pointer to the utf16 string start point + * @param size of the string + * @return offset of utf8 + */ + Nd4jLong offsetUtf32StringInUtf8(const void* start, const void* end); + + /* + * This function check is valid charecter in u8 string + */ + bool isStringValidU8(const void* start, const void* stop); + + /* + * This function check is valid charecter in u16 string + */ + bool isStringValidU16(const void* start, const void* stop); + + /* + * This function check is valid u32 charecter in string + */ + bool isStringValidU32(const void* start, const void* stop); + + /** + * This method count offset for utf8 string in utf32 + * @param const pointer to the utf8 string start point + * @param size of the string + * @return offset + */ + Nd4jLong offsetUtf8StringInUtf32(const void* input, uint32_t nInputSize); + + /** + * This method count offset for utf8 string in utf32 + * @param const pointer to the utf8 string start point + * @param const end pointer to the utf8 string + * @return offset + */ + Nd4jLong offsetUtf8StringInUtf32(const void* input, const void* stop); + + /** + * This method count offset for utf32 based on utf16 string + * @param const pointer to the utf16 string start point + * @param size of the string + * @return offset + */ + Nd4jLong offsetUtf16StringInUtf32(const void* input, uint32_t nInputSize); + + /** + * This method calculate offset of u16 based on utf8 + * @param const pointer to the utf8 string start point + * @param size of the string + * @return offset of utf16 + */ + Nd4jLong offsetUtf8StringInUtf16(const void* input, uint32_t nInputSize); + + /** + * This method calculate offset of u8 based on utf16 + * @param const pointer to the utf16 string start point + * @param size of the string + * @return offset of utf8 + */ + Nd4jLong offsetUtf16StringInUtf8(const void* input, uint32_t nInputSize); + + /** + * This method calculate offset of u32 based on utf8 + * @param const pointer to the utf16 string start point + * @param size of the string + * @return offset of utf32 + */ + Nd4jLong offsetUtf32StringInUtf8(const void* input, uint32_t nInputSize); + + /** + * This method calculate offset of u32 based on utf16 + * @param const pointer to the utf16 string start point + * @param size of the string + * @return offset of utf32 + */ + Nd4jLong offsetUtf32StringInUtf16(const void* input, const uint32_t nInputSize); + + /** + * This method convert utf8 string to utf16 string + * @param const pointer to the utf8 string start point + * @param reference to start point to utf16 + * @param size of input utf8 string + * @return status of convertion + */ + bool utf8to16(const void* input, void* output, uint32_t nInputSize); + + /** + * This method convert utf8 string to utf32 string + * @param const pointer to the utf8 string start point + * @param reference to start point to utf32 + * @param size of input utf8 string + * @return status of convertion + */ + bool utf8to32(const void* input, void* output, uint32_t nInputSize); + + /** + * This method convert utf16 string to utf32 string + * @param const pointer to the utf16 string start point + * @param reference to start point to utf32 + * @param size of input utf16 string + * @return status of convertion + */ + bool utf16to32(const void* input, void* output, uint32_t nInputSize); + + /** + * This method convert utf16 string to utf8 string + * @param const pointer to the utf16 string start point + * @param reference to start point to utf8 + * @param size of input utf16 string + * @return status of convertion + */ + bool utf16to8(const void* input, void* output, uint32_t nInputSize); + + /** + * This method convert utf32 string to utf16 string + * @param const pointer to the utf32 string start point + * @param reference to start point to utf16 + * @param size of input utf32 string + * @return status of convertion + */ + bool utf32to16(const void* input, void* output, uint32_t nInputSize); + + /** + * This method convert utf32 string to utf8 string + * @param const pointer to the utf32 string start point + * @param reference to start point to utf8 + * @param size of input utf32 string + * @return status of convertion + */ + bool utf32to8(const void* input, void* output, const Nd4jLong nInputSize); +} +} + + +#endif //LIBND4J_UNICODE_H diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp b/libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp index 56f77737d..1fe3e359b 100644 --- a/libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp +++ b/libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp @@ -118,7 +118,7 @@ namespace ops { DECLARE_TYPES(Pow_bp) { getOpDescriptor() ->setAllowedInputTypes({ ALL_FLOATS, ALL_INTS }) - ->setAllowedOutputTypes({ ALL_FLOATS }); // TODO maybe wourth to add ALL_INTS + ->setAllowedOutputTypes({ ALL_FLOATS }); } } diff --git a/libnd4j/include/ops/declarable/generic/compat/compat_string_split.cpp b/libnd4j/include/ops/declarable/generic/compat/compat_string_split.cpp index 9d7b57ee4..ac88a4a60 100644 --- a/libnd4j/include/ops/declarable/generic/compat/compat_string_split.cpp +++ b/libnd4j/include/ops/declarable/generic/compat/compat_string_split.cpp @@ -81,7 +81,7 @@ namespace nd4j { } // now once we have all strings in single vector time to fill - auto tmp = NDArrayFactory::string('c', {(Nd4jLong) strings.size()}, strings); + auto tmp = NDArrayFactory::string({(Nd4jLong) strings.size()}, strings); auto blen = StringUtils::byteLength(tmp) + ShapeUtils::stringBufferHeaderRequirements(strings.size()); // for CUDA mostly diff --git a/libnd4j/include/types/types.h b/libnd4j/include/types/types.h index 9c8dcb273..92fada8d3 100644 --- a/libnd4j/include/types/types.h +++ b/libnd4j/include/types/types.h @@ -33,6 +33,11 @@ #include +#define LIBND4J_STRINGTYPES \ + (nd4j::DataType::UTF8, std::string),\ + (nd4j::DataType::UTF16, std::u16string), \ + (nd4j::DataType::UTF32, std::u32string) + #define LIBND4J_TYPES \ (nd4j::DataType::BFLOAT16, bfloat16),\ (nd4j::DataType::HALF, float16), \ diff --git a/libnd4j/tests_cpu/layers_tests/BroadcastableOpsTests.cpp b/libnd4j/tests_cpu/layers_tests/BroadcastableOpsTests.cpp index 0b1daa3af..1f6000f06 100644 --- a/libnd4j/tests_cpu/layers_tests/BroadcastableOpsTests.cpp +++ b/libnd4j/tests_cpu/layers_tests/BroadcastableOpsTests.cpp @@ -599,7 +599,7 @@ TEST_F(BroadcastableOpsTests, broadcast_empty_2) { TEST_F(BroadcastableOpsTests, broadcast_empty_3) { NDArray x = NDArrayFactory::create('c', {1, 0, 2}); - NDArray y('c', {}, {0.1}, nd4j::DataType::FLOAT32); + NDArray y('c', {}, std::vector{0.1}, nd4j::DataType::FLOAT32); NDArray e = NDArrayFactory::create('c', {1, 0, 2});; nd4j::ops::maximum op; diff --git a/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp b/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp index 376234019..0c63f527e 100644 --- a/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp +++ b/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp @@ -626,7 +626,7 @@ TEST_F(ConvolutionTests2, deconv3d_bp_test1) { NDArray expGradI('c', {bS, oD, oH, oW, oC}, {62., 67.6, 68.4, 74.8, 81.2, 89.2, 87.6, 96.4, 119.6, 132.4, 126., 139.6, 138.8, 154., 145.2, 161.2}, nd4j::DataType::FLOAT32); NDArray expGradW('c', {kD, kH, kW, iC, oC}, {28., 28., 32., 32., 40., 40., 44., 44., 64, 64., 68., 68., 76., 76., 80., 80.}, nd4j::DataType::FLOAT32); - NDArray expGradB('c', {iC}, {364.5}, nd4j::DataType::FLOAT32); + NDArray expGradB('c', {iC}, std::vector{364.5}, nd4j::DataType::FLOAT32); input = 0.5; weights.linspace(0.1, 0.1); diff --git a/libnd4j/tests_cpu/layers_tests/CudaBasicsTests1.cu b/libnd4j/tests_cpu/layers_tests/CudaBasicsTests1.cu index 593d47bb5..0a22272d8 100644 --- a/libnd4j/tests_cpu/layers_tests/CudaBasicsTests1.cu +++ b/libnd4j/tests_cpu/layers_tests/CudaBasicsTests1.cu @@ -132,11 +132,11 @@ TEST_F(CudaBasicsTests1, execIndexReduceScalar_1) { NDArray x2('c', {2,2}, {0.5, 1.5, -4.5, 3.5}, nd4j::DataType::BFLOAT16); NDArray x3('c', {2,2}, {0, -1, 0, 1}, nd4j::DataType::BOOL); - NDArray scalar('c', {}, {0}, nd4j::DataType::INT64); + NDArray scalar('c', {}, std::vector{0}, nd4j::DataType::INT64); - NDArray exp1('c', {}, {3}, nd4j::DataType::INT64); - NDArray exp2('c', {}, {2}, nd4j::DataType::INT64); - NDArray exp3('c', {}, {1}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{3}, nd4j::DataType::INT64); + NDArray exp2('c', {}, std::vector{2}, nd4j::DataType::INT64); + NDArray exp3('c', {}, std::vector{1}, nd4j::DataType::INT64); void *dX1, *dX2, *dX3, *dZ; Nd4jLong *dX1ShapeInfo, *dX2ShapeInfo, *dX3ShapeInfo, *dZShapeInfo; @@ -262,11 +262,11 @@ TEST_F(CudaBasicsTests1, execReduce3Scalar_1) { NDArray x3('c', {2,2}, {1.5,1.5,1.5,1.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {2,2}, {1,2,3,4}, nd4j::DataType::DOUBLE); - NDArray exp1('c', {}, {-30.f}, nd4j::DataType::FLOAT32); - NDArray exp2('c', {}, {15.}, nd4j::DataType::DOUBLE); + NDArray exp1('c', {}, std::vector{-30.f}, nd4j::DataType::FLOAT32); + NDArray exp2('c', {}, std::vector{15.}, nd4j::DataType::DOUBLE); - NDArray scalar1('c', {}, {100.f}, nd4j::DataType::FLOAT32); - NDArray scalar2('c', {}, {100.}, nd4j::DataType::DOUBLE); + NDArray scalar1('c', {}, std::vector{100.f}, nd4j::DataType::FLOAT32); + NDArray scalar2('c', {}, std::vector{100.}, nd4j::DataType::DOUBLE); void *dX1, *dX2, *dX3, *dX4, *dZ1, *dZ2; Nd4jLong *dX1ShapeInfo, *dX3ShapeInfo, *dZ1ShapeInfo, *dZ2ShapeInfo; @@ -363,8 +363,8 @@ TEST_F(CudaBasicsTests1, execReduce3_1) { NDArray x('c', {2,2}, {1,2,3,4}, nd4j::DataType::INT32); NDArray y('c', {2,2}, {-1,-2,-3,-4}, nd4j::DataType::INT32); - NDArray exp('c', {}, {-30.f}, nd4j::DataType::FLOAT32); - NDArray z('c', {}, {100.f}, nd4j::DataType::FLOAT32); + NDArray exp('c', {}, std::vector{-30.f}, nd4j::DataType::FLOAT32); + NDArray z('c', {}, std::vector{100.f}, nd4j::DataType::FLOAT32); std::vector dimensions = {0, 1}; @@ -415,8 +415,8 @@ TEST_F(CudaBasicsTests1, execReduce3_2) { NDArray x('c', {2,2}, {1.5,1.5,1.5,1.5}, nd4j::DataType::DOUBLE); NDArray y('c', {2,2}, {1,2,3,4}, nd4j::DataType::DOUBLE); - NDArray exp('c', {}, {15.}, nd4j::DataType::DOUBLE); - NDArray z('c', {}, {100.}, nd4j::DataType::DOUBLE); + NDArray exp('c', {}, std::vector{15.}, nd4j::DataType::DOUBLE); + NDArray z('c', {}, std::vector{100.}, nd4j::DataType::DOUBLE); std::vector dimensions = {0, 1}; @@ -975,7 +975,7 @@ TEST_F(CudaBasicsTests1, execScalar_1) { NDArray x('c', {2,3}, {0,1,2,3,4,5}, nd4j::DataType::INT64); NDArray exp('c',{2,3}, {0,0,1,1,2,2}, nd4j::DataType::INT64); - NDArray scalar('c',{}, {2.f}, nd4j::DataType::FLOAT32); + NDArray scalar('c',{}, std::vector{2.f}, nd4j::DataType::FLOAT32); NDArray z('c', {2,3}, {100,100,100,100,100,100}, nd4j::DataType::INT64); // create cuda stream and LaunchContext @@ -1010,7 +1010,7 @@ TEST_F(CudaBasicsTests1, execScalar_2) { NDArray x('c', {2,3}, {-1,-2,-3,-4,-5,-6}, nd4j::DataType::INT64); NDArray exp('c',{2,3}, {10,10,10,10,10,10}, nd4j::DataType::FLOAT32); - NDArray scalar('c',{}, {10.f}, nd4j::DataType::FLOAT32); + NDArray scalar('c',{}, std::vector{10.f}, nd4j::DataType::FLOAT32); NDArray z('c', {2,3}, {100,100,100,100,100,100}, nd4j::DataType::FLOAT32); // create cuda stream and LaunchContext @@ -1103,7 +1103,7 @@ TEST_F(CudaBasicsTests1, execScalar_3) { TEST_F(CudaBasicsTests1, execScalarBool_1) { NDArray x('c', {2,3}, {-1,-2,0,1,2,3}, nd4j::DataType::BFLOAT16); - NDArray scalar('c',{}, {0}, nd4j::DataType::BFLOAT16); + NDArray scalar('c',{}, std::vector{0}, nd4j::DataType::BFLOAT16); NDArray exp('c',{2,3}, {0,0,0,1,1,1}, nd4j::DataType::BOOL); NDArray z('c', {2,3}, {100,100,100,100,100,100,}, nd4j::DataType::BOOL); @@ -2245,8 +2245,8 @@ TEST_F(CudaBasicsTests1, execReduceLong_2) { TEST_F(CudaBasicsTests1, execReduceFloatScalar_1) { NDArray x('c', {2,3,4}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18}, nd4j::DataType::INT32); - NDArray z('c', {}, {100}, nd4j::DataType::FLOAT32); - NDArray exp('c', {}, {6.5}, nd4j::DataType::FLOAT32); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::FLOAT32); + NDArray exp('c', {}, std::vector{6.5}, nd4j::DataType::FLOAT32); x.permutei({2,1,0}); // create cuda stream and LaunchContext @@ -2282,8 +2282,8 @@ TEST_F(CudaBasicsTests1, execReduceFloatScalar_1) { TEST_F(CudaBasicsTests1, execReduceFloatScalar_2) { NDArray x('c', {2,3,4}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18}, nd4j::DataType::INT32); - NDArray z('c', {}, {100}, nd4j::DataType::DOUBLE); - NDArray exp('c', {}, {6.5}, nd4j::DataType::DOUBLE); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::DOUBLE); + NDArray exp('c', {}, std::vector{6.5}, nd4j::DataType::DOUBLE); // create cuda stream and LaunchContext cudaError_t cudaResult; @@ -2318,8 +2318,8 @@ TEST_F(CudaBasicsTests1, execReduceFloatScalar_2) { TEST_F(CudaBasicsTests1, execReduceSameScalar_1) { NDArray x('c', {2,3,4}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18}, nd4j::DataType::INT32); - NDArray z('c', {}, {100}, nd4j::DataType::INT32); - NDArray exp('c', {}, {156}, nd4j::DataType::INT32); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::INT32); + NDArray exp('c', {}, std::vector{156}, nd4j::DataType::INT32); x.permutei({2,1,0}); // create cuda stream and LaunchContext @@ -2355,8 +2355,8 @@ TEST_F(CudaBasicsTests1, execReduceSameScalar_1) { TEST_F(CudaBasicsTests1, execReduceSameScalar_2) { NDArray x('c', {2,3,4}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18}, nd4j::DataType::DOUBLE); - NDArray z('c', {}, {100}, nd4j::DataType::DOUBLE); - NDArray exp('c', {}, {156}, nd4j::DataType::DOUBLE); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::DOUBLE); + NDArray exp('c', {}, std::vector{156}, nd4j::DataType::DOUBLE); // create cuda stream and LaunchContext cudaError_t cudaResult; @@ -2391,8 +2391,8 @@ TEST_F(CudaBasicsTests1, execReduceSameScalar_2) { TEST_F(CudaBasicsTests1, execReduceBoolScalar_1) { NDArray x('c', {2,3,4}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18}, nd4j::DataType::INT32); - NDArray z('c', {}, {100}, nd4j::DataType::BOOL); - NDArray exp('c', {}, {1}, nd4j::DataType::BOOL); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::BOOL); + NDArray exp('c', {}, std::vector{1}, nd4j::DataType::BOOL); x.permutei({2,1,0}); x.syncShape(); @@ -2429,8 +2429,8 @@ TEST_F(CudaBasicsTests1, execReduceBoolScalar_1) { TEST_F(CudaBasicsTests1, execReduceBoolScalar_2) { NDArray x('c', {2,3,4}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18}, nd4j::DataType::DOUBLE); - NDArray z('c', {}, {100}, nd4j::DataType::BOOL); - NDArray exp('c', {}, {1}, nd4j::DataType::BOOL); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::BOOL); + NDArray exp('c', {}, std::vector{1}, nd4j::DataType::BOOL); // create cuda stream and LaunchContext cudaError_t cudaResult; @@ -2465,8 +2465,8 @@ TEST_F(CudaBasicsTests1, execReduceBoolScalar_2) { TEST_F(CudaBasicsTests1, execReduceLongScalar_1) { NDArray x('c', {2,3,4}, {-5,0,-3,0,-1,0,1,2,3,4,5,6,7,0,9,10,11,0,13,14,0,16,0,18}, nd4j::DataType::INT32); - NDArray z('c', {}, {100}, nd4j::DataType::INT64); - NDArray exp('c', {}, {17}, nd4j::DataType::INT64); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::INT64); + NDArray exp('c', {}, std::vector{17}, nd4j::DataType::INT64); x.permutei({2,1,0}); x.syncShape(); @@ -2503,8 +2503,8 @@ TEST_F(CudaBasicsTests1, execReduceLongScalar_1) { TEST_F(CudaBasicsTests1, execReduceLongScalar_2) { NDArray x('c', {2,3,4}, {-5,0,-3,0,-1,0,1,2,3,4,5,6,7,0,9,10,11,0,13,14,0,16,0,18}, nd4j::DataType::DOUBLE); - NDArray z('c', {}, {100}, nd4j::DataType::INT64); - NDArray exp('c', {}, {17}, nd4j::DataType::INT64); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::INT64); + NDArray exp('c', {}, std::vector{17}, nd4j::DataType::INT64); // create cuda stream and LaunchContext cudaError_t cudaResult; @@ -2685,8 +2685,8 @@ TEST_F(CudaBasicsTests1, execReduce3TAD_4) { NDArray x('c', {2,2,3}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6}, nd4j::DataType::DOUBLE); NDArray y('c', {2,2,3}, {10,20,30,40,50,60,70,80,90,100,110,120}, nd4j::DataType::DOUBLE); - NDArray exp('c', {}, {1820}, nd4j::DataType::FLOAT32); - NDArray z('c', {}, {100}, nd4j::DataType::FLOAT32); + NDArray exp('c', {}, std::vector{1820}, nd4j::DataType::FLOAT32); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::FLOAT32); std::vector dimensions = {0,1,2}; @@ -2739,8 +2739,8 @@ TEST_F(CudaBasicsTests1, execReduce3TAD_4) { TEST_F(CudaBasicsTests1, execSummaryStats_1) { NDArray x('c', {2,2,3}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6}, nd4j::DataType::INT64); - NDArray exp('c', {}, {3.605551}, nd4j::DataType::FLOAT32); - NDArray z('c', {}, {100}, nd4j::DataType::FLOAT32); + NDArray exp('c', {}, std::vector{3.605551}, nd4j::DataType::FLOAT32); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::FLOAT32); // create cuda stream and LaunchContext cudaError_t cudaResult; @@ -2881,8 +2881,8 @@ TEST_F(CudaBasicsTests1, execSummaryStats_3) { TEST_F(CudaBasicsTests1, execSummaryStatsScalar_1) { NDArray x('c', {2,2,3}, {-5,-4,-3,-2,-1,0,1,2,3,4,5,6}, nd4j::DataType::INT64); - NDArray exp('c', {}, {3.605551}, nd4j::DataType::FLOAT32); - NDArray z('c', {}, {100}, nd4j::DataType::FLOAT32); + NDArray exp('c', {}, std::vector{3.605551}, nd4j::DataType::FLOAT32); + NDArray z('c', {}, std::vector{100}, nd4j::DataType::FLOAT32); // create cuda stream and LaunchContext cudaError_t cudaResult; diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp index 4875ce8c5..a0722f9d0 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp @@ -775,7 +775,7 @@ TEST_F(DeclarableOpsTests10, sparse_softmax_cross_entropy_loss_with_logits_test2 /////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, sparse_softmax_cross_entropy_loss_with_logits_test3) { - NDArray labels('c', {1}, {0}, nd4j::DataType::INT32); + NDArray labels('c', {1}, std::vector{0}, nd4j::DataType::INT32); auto logits = NDArrayFactory::create('c', {1,3}); auto expected = NDArrayFactory::create('c', {1}, {1.20194}); @@ -2735,7 +2735,7 @@ TEST_F(DeclarableOpsTests10, Image_CropAndResize_3) { NDArray images ('c', {1,2,2,1}, {1,2,3,4}, nd4j::DataType::FLOAT32); NDArray boxes('c', {1,4}, {0,0,1,1}, nd4j::DataType::FLOAT32); - NDArray boxI('c', {1}, {0}, nd4j::DataType::INT64); + NDArray boxI('c', {1}, std::vector{0}, nd4j::DataType::INT64); NDArray cropSize = NDArrayFactory::create({3, 3}); //NDArray ('c', {6}, {0.9f, .75f, .6f, .95f, .5f, .3f}); @@ -2759,7 +2759,7 @@ TEST_F(DeclarableOpsTests10, Image_CropAndResize_4) { NDArray images('c', {1,2,2,1}, {1, 2, 3, 4}, nd4j::DataType::FLOAT32); NDArray boxes('c', {1,4}, {0,0,1,1}, nd4j::DataType::FLOAT32); - NDArray boxI('c', {1}, {0}, nd4j::DataType::INT32); + NDArray boxI('c', {1}, std::vector({0.}), nd4j::DataType::INT32); NDArray cropSize = NDArrayFactory::create({3, 3}); //NDArray ('c', {6}, {0.9f, .75f, .6f, .95f, .5f, .3f}); @@ -2933,8 +2933,8 @@ TEST_F(DeclarableOpsTests10, FakeQuantWithMinMaxVars_Test_1) { NDArray x('c', {2,3}, {-63.80f, -63.75f, -63.70f, -63.5f, 0.0f, 0.1f}, nd4j::DataType::FLOAT32); NDArray exp('c', {2,3}, {-63.75f, -63.75f, -63.75f, -63.5f, 0.f, 0.f}, nd4j::DataType::FLOAT32); - NDArray min('c', {}, {-63.65f}, nd4j::DataType::FLOAT32); - NDArray max('c', {}, {0.1f}, nd4j::DataType::FLOAT32); + NDArray min('c', {}, std::vector{-63.65f}, nd4j::DataType::FLOAT32); + NDArray max('c', {}, std::vector{0.1f}, nd4j::DataType::FLOAT32); nd4j::ops::fake_quant_with_min_max_vars op; auto results = op.evaluate({&x, &min, &max}, {}, {}); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests11.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests11.cpp index 27f742316..71ebdc7e6 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests11.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests11.cpp @@ -121,7 +121,7 @@ TEST_F(DeclarableOpsTests11, log_loss_grad_test3) { NDArray dLdpExp('c', {2,3,4}, {-12.49997,-13.04346, -13.63635, -14.28571,-14.99999,-15.78947, -16.66666, -17.64705,-18.75 ,-20. , -21.42857, -23.07692, -24.99999,-27.27272, -29.99999, -33.33332,-37.49999,-42.85713, -49.99998, -59.99998,-74.99995,-99.99992,-149.99986,-299.99911}); - NDArray dLdwExp('c', {}, {-227.77286}); + NDArray dLdwExp('c', {}, std::vector{-227.77286}); NDArray dLdlExp('c', {2,3,4}, {1.58903, 1.22117, 0.99621, 0.82911, 0.69315, 0.57634, 0.47223, 0.37689, 0.28768, 0.20273, 0.12058, 0.04002, -0.04002,-0.12058,-0.20273,-0.28768,-0.37689,-0.47223,-0.57634,-0.69315,-0.82911,-0.99621,-1.22117,-1.58903}); @@ -246,7 +246,7 @@ TEST_F(DeclarableOpsTests11, log_loss_grad_test7) { NDArray predictions('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights(nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {}, {0.}); + NDArray dLdwExp('c', {}, std::vector{0.}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -350,7 +350,7 @@ TEST_F(DeclarableOpsTests11, log_loss_grad_test10) { NDArray predictions('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights('c', {1,1}, nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {1,1}, {-9.49054}); + NDArray dLdwExp('c', {1,1}, std::vector{-9.49054}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -1611,7 +1611,7 @@ TEST_F(DeclarableOpsTests11, mean_sqerr_loss_grad_test3) { NDArray dLdpExp('c', {2,3,4}, {-0.96, -1.92, -2.88, -3.84, -4.8 , -5.76, -6.72, -7.68, -8.64, -9.6 ,-10.56,-11.52, -12.48,-13.44,-14.4 ,-15.36,-16.32,-17.28,-18.24,-19.2 ,-20.16,-21.12,-22.08,-23.04}); - NDArray dLdwExp('c', {}, {4515.84}); + NDArray dLdwExp('c', {}, std::vector{4515.84}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -1730,7 +1730,7 @@ TEST_F(DeclarableOpsTests11, mean_sqerr_loss_grad_test7) { NDArray predictions('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights(nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {}, {0.}); + NDArray dLdwExp('c', {}, std::vector{0.}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -1830,7 +1830,7 @@ TEST_F(DeclarableOpsTests11, mean_sqerr_loss_grad_test10) { NDArray predictions('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights('c', {1,1}, nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {1,1}, {188.16}); + NDArray dLdwExp('c', {1,1}, std::vector{188.16}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -2056,7 +2056,7 @@ TEST_F(DeclarableOpsTests11, absolute_difference_loss_grad_test3) { NDArray dLdpExp('c', {2,3,4}, {-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5, -0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5,-0.5}); - NDArray dLdwExp('c', {}, {288.}); + NDArray dLdwExp('c', {}, std::vector{288.}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -2175,7 +2175,7 @@ TEST_F(DeclarableOpsTests11, absolute_difference_loss_grad_test7) { NDArray predictions('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights(nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {}, {0.}); + NDArray dLdwExp('c', {}, std::vector{0.}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -2275,7 +2275,7 @@ TEST_F(DeclarableOpsTests11, absolute_difference_loss_grad_test10) { NDArray predictions('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights('c', {1,1}, nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {1,1}, {12.}); + NDArray dLdwExp('c', {1,1}, std::vector{12.}); predictions.linspace(0.04, 0.04); labels.linspace(1); @@ -2541,7 +2541,7 @@ TEST_F(DeclarableOpsTests11, sigm_cross_entropy_loss_grad_test3) { NDArray dLdpExp('c', {2,3,4}, {-0.18499,-0.53 ,-0.875 ,-1.22 ,-1.56501,-1.91002,-2.25504,-2.60008,-2.94514,-3.29023,-3.63534,-3.98048, -4.32566,-4.67087,-5.01613,-5.36143,-5.70677,-6.05217,-6.39762,-6.74313,-7.0887 ,-7.43432,-7.78001,-8.12577}); - NDArray dLdwExp('c', {}, {-91.52109}); + NDArray dLdwExp('c', {}, std::vector{-91.52109}); NDArray dLdlExp('c', {2,3,4}, {0.028, 0.014, -0., -0.014,-0.028, -0.042, -0.056, -0.07 ,-0.084, -0.098, -0.112, -0.126, -0.14 , -0.154, -0.168, -0.182,-0.196, -0.21 , -0.224, -0.238,-0.252, -0.266, -0.28 , -0.294}); @@ -2664,7 +2664,7 @@ TEST_F(DeclarableOpsTests11, sigm_cross_entropy_loss_grad_test7) { NDArray logits('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights(nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {}, {0.}); + NDArray dLdwExp('c', {}, std::vector{0.}); logits.linspace(-0.08, 0.04); labels.linspace(1); @@ -2766,7 +2766,7 @@ TEST_F(DeclarableOpsTests11, sigm_cross_entropy_loss_grad_test10) { NDArray logits('c', {2,3,4}, nd4j::DataType::DOUBLE); NDArray weights('c', {1,1}, nd4j::DataType::DOUBLE); - NDArray dLdwExp('c', {1,1}, {-3.81338}); + NDArray dLdwExp('c', {1,1}, std::vector{-3.81338}); logits.linspace(-0.08, 0.04); labels.linspace(1); @@ -2992,7 +2992,7 @@ TEST_F(DeclarableOpsTests11, softmax_cross_entropy_loss_grad_test2) { NDArray weights('c', {1}, nd4j::DataType::DOUBLE); NDArray dLdpExp('c', {4}, {0.125, 0.125, -0.375, 0.125}); - NDArray dLdwExp('c', {1}, {1.38629}); + NDArray dLdwExp('c', {1}, std::vector{1.38629}); logits = 2.; weights.assign(0.5); @@ -3020,10 +3020,10 @@ TEST_F(DeclarableOpsTests11, softmax_cross_entropy_loss_grad_test3) { NDArray labels('c', {4}, {0,0,1,0}, nd4j::DataType::INT32); NDArray logits('c', {4}, nd4j::DataType::DOUBLE); - NDArray weights('c', {}, {0}, nd4j::DataType::DOUBLE); + NDArray weights('c', {}, std::vector{0}, nd4j::DataType::DOUBLE); NDArray dLdpExp('c', {4}, {0.125, 0.125, -0.375, 0.125}); - NDArray dLdwExp('c', {}, {1.38629}); + NDArray dLdwExp('c', {}, std::vector{1.38629}); logits = 2.; weights.assign(0.5); @@ -3051,10 +3051,10 @@ TEST_F(DeclarableOpsTests11, softmax_cross_entropy_loss_grad_test4) { NDArray labels('c', {4}, {0,0,1,0}, nd4j::DataType::INT32); NDArray logits('c', {4}, nd4j::DataType::DOUBLE); - NDArray weights('c', {}, {0}, nd4j::DataType::DOUBLE); + NDArray weights('c', {}, std::vector{0}, nd4j::DataType::DOUBLE); NDArray dLdpExp('c', {4}, {0.23521, 0.2448 , -0.7452 , 0.26519}); - NDArray dLdwExp('c', {}, {0.}); + NDArray dLdwExp('c', {}, std::vector{0.}); logits.linspace(-0.08, 0.04); weights = 0.5; @@ -3085,7 +3085,7 @@ TEST_F(DeclarableOpsTests11, softmax_cross_entropy_loss_grad_test5) { NDArray weights('c', {1}, nd4j::DataType::DOUBLE); NDArray dLdpExp('c', {4}, {0.1176, 0.1224, -0.3726, 0.1326}); - NDArray dLdwExp('c', {1}, {1.36729}); + NDArray dLdwExp('c', {1}, std::vector{1.36729}); logits.linspace(-0.08, 0.04); weights = 0.5; @@ -3321,7 +3321,7 @@ TEST_F(DeclarableOpsTests11, softmaxCrossEntropyWithLogits_grad_test4) { ///////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests11, softmaxCrossEntropyWithLogits_grad_test5) { - NDArray labels('c', {2,1}, {1,0}); + NDArray labels('c', {2,1}, std::vector{1,0}); NDArray logits('c', {2,1}, {-0.04, 0.04}); NDArray dLdpExp('c', {2,1}, {-0.51999, 0.51999}); @@ -3343,10 +3343,10 @@ TEST_F(DeclarableOpsTests11, softmaxCrossEntropyWithLogits_grad_test5) { ///////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests11, softmaxCrossEntropyWithLogits_grad_test6) { - NDArray labels('c', {1,2}, {1,1}); + NDArray labels('c', {1,2}, {1,1.}); NDArray logits('c', {1,2}, {-0.04, 0.04}); - NDArray dLdpExp('c', {1,2}, {0, 0}); + NDArray dLdpExp('c', {1,2}, {0, 0.}); nd4j::ops::softmax_cross_entropy_loss_with_logits_grad op; @@ -3387,10 +3387,10 @@ TEST_F(DeclarableOpsTests11, softmaxCrossEntropyWithLogits_grad_test7) { ///////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests11, softmaxCrossEntropyWithLogits_grad_test8) { - NDArray labels('c', {1}, {1}); - NDArray logits('c', {1}, {0.04}); + NDArray labels('c', {1}, std::vector{1}); + NDArray logits('c', {1}, std::vector{0.04}); - NDArray dLdpExp('c', {1}, {0}); + NDArray dLdpExp('c', {1}, std::vector{0}); nd4j::ops::softmax_cross_entropy_loss_with_logits_grad op; @@ -3483,7 +3483,7 @@ TEST_F(DeclarableOpsTests11, sparseSoftmaxCrossEntropyWithLogits_grad_test2) { ///////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests11, sparseSoftmaxCrossEntropyWithLogits_grad_test3) { - NDArray labels('c', {}, {1}, nd4j::DataType::INT64); + NDArray labels('c', {}, std::vector{1}, nd4j::DataType::INT64); NDArray logits('c', {2}, {-0.2, 0.3}); NDArray dLdpExp('c', {2}, {0.37754, -0.37754}); @@ -3529,7 +3529,7 @@ TEST_F(DeclarableOpsTests11, sparseSoftmaxCrossEntropyWithLogits_grad_test4) { ///////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests11, sparseSoftmaxCrossEntropyWithLogits_grad_test5) { - NDArray labels('c', {1,1}, {0}, nd4j::DataType::INT64); + NDArray labels('c', {1,1}, std::vector({0}), nd4j::DataType::INT64); NDArray logits('c', {1,1,2}, {-0.3,0.2}); NDArray dLdpExp('c', {1,1,2}, {-0.62246, 0.62246}); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests12.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests12.cpp index 80a9d67a4..142a3dbd4 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests12.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests12.cpp @@ -127,7 +127,7 @@ TEST_F(DeclarableOpsTests12, cosine_distance_loss_grad_test3) { NDArray weights('c', {1}, nd4j::DataType::DOUBLE); NDArray dLdpExp('c', {4}, {0.05, -0.15, -1., 0.7}); - NDArray dLdwExp('c', {1}, {1.3}); + NDArray dLdwExp('c', {1}, std::vector{1.3}); NDArray dLdlExp('c', {4}, {0.2, 0.1, -0. , -0.1}); predictions.linspace(-0.4, 0.2); @@ -158,10 +158,10 @@ TEST_F(DeclarableOpsTests12, cosine_distance_loss_grad_test4) { NDArray labels('c', {1,4}, {-0.1, 0.3, 2, -1.4}); NDArray predictions('c', {1,4}, nd4j::DataType::DOUBLE); - NDArray weights('c', {}, {0.}, nd4j::DataType::DOUBLE); + NDArray weights('c', {}, std::vector{0.}, nd4j::DataType::DOUBLE); NDArray dLdpExp('c', {1,4}, {0.05, -0.15, -1., 0.7}); - NDArray dLdwExp('c', {}, {1.3}); + NDArray dLdwExp('c', {}, std::vector{1.3}); NDArray dLdlExp('c', {1,4}, {0.2, 0.1, -0. , -0.1}); predictions.linspace(-0.4, 0.2); @@ -196,7 +196,7 @@ TEST_F(DeclarableOpsTests12, cosine_distance_loss_grad_test5) { NDArray weights('c', {1,1}, nd4j::DataType::DOUBLE); NDArray dLdpExp('c', {4}, {0.1, -0.3, -2. , 1.4}); - NDArray dLdwExp('c', {1,1}, {0.}); + NDArray dLdwExp('c', {1,1}, std::vector{0.}); NDArray dLdlExp('c', {4}, {0.4, 0.2, -0. , -0.2}); predictions.linspace(-0.4, 0.2); @@ -369,10 +369,10 @@ TEST_F(DeclarableOpsTests12, cosine_distance_loss_grad_test9) { TEST_F(DeclarableOpsTests12, hinge_loss_14) { NDArray logits('c', {3,4}, nd4j::DataType::DOUBLE); - NDArray weights('c', {}, {1.}); + NDArray weights('c', {}, std::vector{1.}); NDArray labels('c', {3,4}, {0,1,1,0,1,0,1,0,1,0,1,0}); - NDArray output('c', {}, {0.}, nd4j::DataType::DOUBLE); + NDArray output('c', {}, std::vector{0.}, nd4j::DataType::DOUBLE); logits.linspace(1.); weights.assign(1.); @@ -576,7 +576,7 @@ TEST_F(DeclarableOpsTests12, TestMinimumBP_1) { TEST_F(DeclarableOpsTests12, reverse_test15) { NDArray x('c', {5}, {1,2,3,4,5}, nd4j::DataType::DOUBLE); - NDArray axis('c', {}, {0}, nd4j::DataType::INT32); + NDArray axis('c', {}, std::vector{0}, nd4j::DataType::INT32); NDArray z('c', {5}, nd4j::DataType::DOUBLE); NDArray exp('c', {5}, {5,4,3,2,1}, nd4j::DataType::DOUBLE); @@ -711,7 +711,7 @@ TEST_F(DeclarableOpsTests12, multiUnique_2) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests12, tensormmul_6) { - NDArray x('c', {1}, {2}, nd4j::DataType::FLOAT32); + NDArray x('c', {1}, std::vector{2}, nd4j::DataType::FLOAT32); NDArray y('c', {2,1,2}, {1,2,3,4}, nd4j::DataType::FLOAT32); NDArray exp('c', {2,2}, {2,4,6,8}, nd4j::DataType::FLOAT32); @@ -1140,9 +1140,9 @@ TEST_F(DeclarableOpsTests12, lrn_bp_9) { ////////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests12, lrn_bp_10) { - NDArray input('c', {1,1,1,1}, {1}); - NDArray gradO('c', {1,1,1,1}, {1}); - NDArray exp('c', {1,1,1,1}, {0.19245008}); + NDArray input('c', {1,1,1,1}, std::vector{1}); + NDArray gradO('c', {1,1,1,1}, std::vector{1}); + NDArray exp('c', {1,1,1,1}, std::vector{0.19245008}); nd4j::ops::lrn_bp op; @@ -1193,8 +1193,8 @@ TEST_F(DeclarableOpsTests12, lrn_2) { ////////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests12, lrn_3) { - NDArray input('c', {1,1,1,1}, {1.}); - NDArray exp('c', {1,1,1,1}, {0.69006556}); + NDArray input('c', {1,1,1,1}, std::vector{1.}); + NDArray exp('c', {1,1,1,1}, std::vector{0.69006556}); nd4j::ops::lrn op; @@ -1208,8 +1208,8 @@ TEST_F(DeclarableOpsTests12, lrn_3) { ////////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests12, lrn_4) { - NDArray input('c', {1,1,1,1}, {1.}); - NDArray exp('c', {1,1,1,1}, {0.69006556}); + NDArray input('c', {1,1,1,1}, std::vector{1.}); + NDArray exp('c', {1,1,1,1}, std::vector{0.69006556}); nd4j::ops::lrn op; @@ -1239,10 +1239,10 @@ TEST_F(DeclarableOpsTests12, lrn_5) { TEST_F(DeclarableOpsTests12, inTopK_1) { NDArray x('c', {4, 5}, {11.0, 14.0, 6.0, 9.0, 3.5, 7.0, 21.0, 3.0, 15.0, 6.0, 9.0, 3.5, 7.0, 11.0, 13.0, 5.0, 16.0, 9.0, 13.5, 7.0}); - NDArray y('c', {4}, {0, 0, 0, 0}, nd4j::DataType::INT64); - NDArray z('c', {4}, {1, 1, 1, 1}, nd4j::DataType::BOOL); + NDArray y('c', {4}, {0., 0, 0, 0}, nd4j::DataType::INT64); + NDArray z('c', {4}, {1., 1, 1, 1}, nd4j::DataType::BOOL); - NDArray expV('c', {4}, {1, 0, 0, 0}, nd4j::DataType::BOOL); + NDArray expV('c', {4}, {1., 0, 0, 0}, nd4j::DataType::BOOL); nd4j::ops::in_top_k op; Nd4jStatus status = op.execute({&x, &y, }, {&z}, {}, {2}, {}); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests13.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests13.cpp index a445666df..e964d397d 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests13.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests13.cpp @@ -809,7 +809,7 @@ TEST_F(DeclarableOpsTests13, space_to_batch_nd_1) { NDArray x('c', {1, 2, 2, 2, 3}, nd4j::DataType::FLOAT32); NDArray blockShape('c', {3}, {2, 2, 2} , nd4j::DataType::INT32); // three spatial dimensions - NDArray paddings('c', {3, 2}, {0, 0, 0, 0, 0, 0} , nd4j::DataType::INT32); + NDArray paddings('c', {3, 2}, std::vector{0, 0, 0, 0, 0, 0} , nd4j::DataType::INT32); NDArray exp('c', {8, 1, 1, 1, 3}, nd4j::DataType::FLOAT32); @@ -892,8 +892,8 @@ TEST_F(DeclarableOpsTests13, batch_to_space_nd_1) { NDArray x('c', {8, 1, 1, 1, 3}, nd4j::DataType::FLOAT32); - NDArray blockShape('c', {3}, {2, 2, 2} , nd4j::DataType::INT32); // three spatial dimensions - NDArray crop('c', {3, 2}, {0, 0, 0, 0, 0, 0} , nd4j::DataType::INT32); + NDArray blockShape('c', {3}, {2., 2, 2} , nd4j::DataType::INT32); // three spatial dimensions + NDArray crop('c', {3, 2}, {0., 0, 0, 0, 0, 0} , nd4j::DataType::INT32); NDArray exp('c', {1, 2, 2, 2, 3}, nd4j::DataType::FLOAT32); @@ -990,7 +990,7 @@ TEST_F(DeclarableOpsTests13, mergemax_1) { TEST_F(DeclarableOpsTests13, mergemax_2) { NDArray x1('c', {1, 3}, {0., 1, 2}, nd4j::DataType::FLOAT32); - NDArray x2('c', {1, 1}, {1.}, nd4j::DataType::FLOAT32); + NDArray x2('c', {1, 1}, std::vector{1.}, nd4j::DataType::FLOAT32); NDArray out('c', {1, 3}, {-1., -1, -1}, nd4j::DataType::FLOAT32); nd4j::ops::mergemax op; @@ -2143,10 +2143,10 @@ TEST_F(DeclarableOpsTests13, batchnorm_test7) { NDArray input2('c', {3,15,15,3}, nd4j::DataType::FLOAT32); input2.permutei({0,3,1,2}); - NDArray mean ('c', {3}, {0, 0, 0}, nd4j::DataType::FLOAT32); - NDArray variance('c', {3}, {1, 1, 1}, nd4j::DataType::FLOAT32); - NDArray gamma ('c', {3}, {1, 1, 1}, nd4j::DataType::FLOAT32); - NDArray beta ('c', {3}, {0, 0, 0}, nd4j::DataType::FLOAT32); + NDArray mean ('c', {3}, {0., 0, 0}, nd4j::DataType::FLOAT32); + NDArray variance('c', {3}, {1., 1, 1}, nd4j::DataType::FLOAT32); + NDArray gamma ('c', {3}, {1., 1, 1}, nd4j::DataType::FLOAT32); + NDArray beta ('c', {3}, {0., 0, 0}, nd4j::DataType::FLOAT32); NDArray out1('c', {3,3,15,15}, nd4j::DataType::FLOAT32); NDArray out2('c', {3,3,15,15}, nd4j::DataType::FLOAT32); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests15.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests15.cpp index cd7f84610..d154039f3 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests15.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests15.cpp @@ -858,7 +858,7 @@ TEST_F(DeclarableOpsTests15, test_empty_decreasing_1) { TEST_F(DeclarableOpsTests15, test_rgb_to_grs_1) { // rank 1 NDArray rgbs('c', { 3 }, { 10, 50, 200 }, nd4j::DataType::INT32); - NDArray expected('c', { 1 }, { 55 }, nd4j::DataType::INT32); + NDArray expected('c', { 1 }, std::vector{ 55 }, nd4j::DataType::INT32); nd4j::ops::rgb_to_grs op; auto result = op.evaluate({&rgbs}, {}, {}); auto output = result->at(0); @@ -1395,7 +1395,7 @@ TEST_F(DeclarableOpsTests15, Pow_BP_Test6) { y.assign(4.0); dLdzC.linspace(0.1, 0.1); - NDArray dLdxExpXC('c', { 1 }, { 115.2 }, nd4j::DataType::FLOAT32); + NDArray dLdxExpXC('c', { 1 }, std::vector{ 115.2 }, nd4j::DataType::FLOAT32); NDArray dLdyExpXC('c', { 2, 2, 2 }, { 1.10904, 2.21807, 3.32711, 4.43614, 5.54518, 6.65421, 7.76325, 8.87228 }, nd4j::DataType::FLOAT32); nd4j::ops::Pow_bp op; diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests17.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests17.cpp index 3f200854d..497475262 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests17.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests17.cpp @@ -55,11 +55,11 @@ TEST_F(DeclarableOpsTests17, test_sparse_to_dense_1) { } TEST_F(DeclarableOpsTests17, test_sparse_to_dense_2) { - auto values = NDArrayFactory::string('c', {3}, {"alpha", "beta", "gamma"}); + auto values = NDArrayFactory::string({3}, {"alpha", "beta", "gamma"}); auto shape = NDArrayFactory::create({3, 3}); auto ranges = NDArrayFactory::create({0,0, 1,1, 2,2}); auto def = NDArrayFactory::string("d"); - auto exp = NDArrayFactory::string('c', {3, 3}, {"alpha","d","d", "d","beta","d", "d","d","gamma"}); + auto exp = NDArrayFactory::string( {3, 3}, {"alpha","d","d", "d","beta","d", "d","d","gamma"}); nd4j::ops::compat_sparse_to_dense op; @@ -70,11 +70,11 @@ TEST_F(DeclarableOpsTests17, test_sparse_to_dense_2) { } TEST_F(DeclarableOpsTests17, test_compat_string_split_1) { - auto x = NDArrayFactory::string('c', {2}, {"first string", "second"}); + auto x = NDArrayFactory::string( {2}, {"first string", "second"}); auto delimiter = NDArrayFactory::string(" "); auto exp0 = NDArrayFactory::create({0,0, 0,1, 1,0}); - auto exp1 = NDArrayFactory::string('c', {3}, {"first", "string", "second"}); + auto exp1 = NDArrayFactory::string( {3}, {"first", "string", "second"}); nd4j::ops::compat_string_split op; auto result = op.evaluate({&x, &delimiter}); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests2.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests2.cpp index 118463e3e..fa129b1af 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests2.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests2.cpp @@ -79,7 +79,7 @@ TEST_F(DeclarableOpsTests2, gather_2) { TEST_F(DeclarableOpsTests2, gather_3) { NDArray input ('c', {2,3,4}, {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24}); - NDArray indices ('c', {1,1}, {2}, nd4j::DataType::INT32); + NDArray indices ('c', {1,1}, std::vector{2}, nd4j::DataType::INT32); NDArray expected('c', {2,1,1,4}, {9,10,11,12,21,22,23,24}); nd4j::ops::gather op; @@ -186,7 +186,7 @@ TEST_F(DeclarableOpsTests2, gather_7) { TEST_F(DeclarableOpsTests2, gather_8) { NDArray input('c', {3,5}, {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}, nd4j::DataType::FLOAT32); - NDArray indices('c', {1}, {2}, nd4j::DataType::INT32); + NDArray indices('c', {1}, std::vector{2}, nd4j::DataType::INT32); NDArray expected('c', {1,5}, {11, 12, 13, 14, 15.}, nd4j::DataType::FLOAT32); nd4j::ops::gather op; @@ -206,7 +206,7 @@ TEST_F(DeclarableOpsTests2, gather_8) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests2, gather_9) { NDArray x('c', {2, 4, 3, 2}, nd4j::DataType::FLOAT32); - NDArray indices('c', {2}, {1, 0}, nd4j::DataType::INT32); + NDArray indices('c', {2}, std::vector{1, 0}, nd4j::DataType::INT32); nd4j::ops::gather op; auto result = op.evaluate({&x, &indices}, {}, {-2}); @@ -238,7 +238,7 @@ TEST_F(DeclarableOpsTests2, gather_10) { TEST_F(DeclarableOpsTests2, gather_11) { NDArray x('c', {2, 2}, {1, 2, 3, 4}); - NDArray indices('c', {2}, {1, 0}, nd4j::DataType::INT64); + NDArray indices('c', {2}, std::vector{1, 0}, nd4j::DataType::INT64); NDArray e('c', {2, 2}, {3, 4, 1, 2}); nd4j::ops::gather op; diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests5.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests5.cpp index 2a5697ce8..62868f67f 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests5.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests5.cpp @@ -243,7 +243,7 @@ TEST_F(DeclarableOpsTests5, Test_SetSeed_1) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests5, scatterMul_test1) { auto matrix = NDArrayFactory::create('c', {2, 2}, {1.f, 2.f, 3.f, 4.f}); - NDArray idc('c', {1}, {0LL}, nd4j::DataType::INT64); + NDArray idc('c', {1}, std::vector({0LL}), nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2}, {10.f, 1.f}); auto exp = NDArrayFactory::create('c', {2, 2}, {10.f, 2.f, 3.f, 4.f}); @@ -261,7 +261,7 @@ TEST_F(DeclarableOpsTests5, scatterMul_test1) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests5, scatterDiv_test1) { auto matrix = NDArrayFactory::create('c', {2, 2}, {1.f, 2.f, 3.f, 4.f}); - NDArray idc('c', {1}, {0LL}, nd4j::DataType::INT64); + NDArray idc('c', {1}, std::vector({0LL}), nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2}, {10.f, 1.f}); auto exp = NDArrayFactory::create('c', {2, 2}, {0.10f, 2.f, 3.f, 4.f}); @@ -279,7 +279,7 @@ TEST_F(DeclarableOpsTests5, scatterDiv_test1) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests5, scatterSub_test1) { auto matrix = NDArrayFactory::create('c', {2, 2}, {1.f, 2.f, 3.f, 4.f}); - NDArray idc('c', {1}, {0LL}, nd4j::DataType::INT64); + NDArray idc('c', {1}, std::vector({0LL}), nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2}, {10.f, 1.f}); auto exp = NDArrayFactory::create('c', {2, 2}, {-9.f, 1.f, 3.f, 4.f}); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests6.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests6.cpp index 5be2eeebd..5a919d132 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests6.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests6.cpp @@ -1411,7 +1411,7 @@ TEST_F(DeclarableOpsTests6, MatrixDeterminant_2) { TEST_F(DeclarableOpsTests6, MatrixDeterminant_3) { auto x = NDArrayFactory::create('c', {1, 3, 3}, {3.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 3.0}); - NDArray exp('c', {1}, {-54.0}); + NDArray exp('c', {1}, std::vector{-54.0}); nd4j::ops::matrix_determinant op; auto result = op.evaluate({&x}, {}, {}); @@ -1453,7 +1453,7 @@ TEST_F(DeclarableOpsTests6, MatrixDeterminant_4) { TEST_F(DeclarableOpsTests6, MatrixDeterminant_5) { auto x = NDArrayFactory::create('c', {1, 4, 4}); - NDArray exp('c', {1}, {-16.0}); + NDArray exp('c', {1}, std::vector{-16.0}); x.linspace(1); x.p(5, 4.0); x.p(12, 12.0); diff --git a/libnd4j/tests_cpu/layers_tests/FlatUtilsTests.cpp b/libnd4j/tests_cpu/layers_tests/FlatUtilsTests.cpp index bf428b833..31aa997c6 100644 --- a/libnd4j/tests_cpu/layers_tests/FlatUtilsTests.cpp +++ b/libnd4j/tests_cpu/layers_tests/FlatUtilsTests.cpp @@ -83,7 +83,7 @@ TEST_F(FlatUtilsTests, flat_bool_serde_1) { } TEST_F(FlatUtilsTests, flat_string_serde_1) { - auto array = NDArrayFactory::string('c', {3}, {"alpha", "beta", "gamma"}); + auto array = NDArrayFactory::string( {3}, {"alpha", "beta", "gamma"}); flatbuffers::FlatBufferBuilder builder(1024); auto flatArray = FlatUtils::toFlatArray(builder, array); diff --git a/libnd4j/tests_cpu/layers_tests/JavaInteropTests.cpp b/libnd4j/tests_cpu/layers_tests/JavaInteropTests.cpp index ee828a6e2..2f3f93d56 100644 --- a/libnd4j/tests_cpu/layers_tests/JavaInteropTests.cpp +++ b/libnd4j/tests_cpu/layers_tests/JavaInteropTests.cpp @@ -1277,14 +1277,14 @@ TEST_F(JavaInteropTests, test_size_dtype_1) { } TEST_F(JavaInteropTests, test_expandable_array_op_1) { - auto x = NDArrayFactory::string('c', {2}, {"first string", "second"}); - auto d = NDArrayFactory::string(" "); + auto x = NDArrayFactory::string( {2}, {"first string", "second"}); + auto d = NDArrayFactory::string(" ", nd4j::DataType::UTF8); auto z0 = NDArrayFactory::create('c', {6}); - auto z1 = NDArrayFactory::string('c', {3}, {"", "", ""}); + auto z1 = NDArrayFactory::string( {3}, {"", "", ""}); auto exp0 = NDArrayFactory::create({0,0, 0,1, 1,0}); - auto exp1 = NDArrayFactory::string('c', {3}, {"first", "string", "second"}); + auto exp1 = NDArrayFactory::string( {3}, {"first", "string", "second"}); InteropDataBuffer iz0(z0.dataBuffer()); InteropDataBuffer iz1(z1.dataBuffer()); diff --git a/libnd4j/tests_cpu/layers_tests/MultiDataTypeTests.cpp b/libnd4j/tests_cpu/layers_tests/MultiDataTypeTests.cpp index aae4493ab..127b3c7d3 100644 --- a/libnd4j/tests_cpu/layers_tests/MultiDataTypeTests.cpp +++ b/libnd4j/tests_cpu/layers_tests/MultiDataTypeTests.cpp @@ -204,7 +204,7 @@ TEST_F(MultiDataTypeTests, ndarray_repeat_test1) { //////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_bufferAsT_test1) { NDArray x('f', {2}, {1.5, 3.5}, nd4j::DataType::FLOAT32); - NDArray y('c', {}, {1.5}, nd4j::DataType::FLOAT32); + NDArray y('c', {}, std::vector{1.5}, nd4j::DataType::FLOAT32); const int* buffX = x.bufferAsT(); const int* buffY = y.bufferAsT(); @@ -217,8 +217,8 @@ TEST_F(MultiDataTypeTests, ndarray_assign_test1) { NDArray x('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::UINT8); NDArray exp('c', {2,2}, {10, 10, 20, 20}, nd4j::DataType::UINT8); - NDArray scalar1('c', {}, {10.5}, nd4j::DataType::FLOAT32); - NDArray scalar2('c', {}, {20.8}, nd4j::DataType::DOUBLE); + NDArray scalar1('c', {}, std::vector{10.5}, nd4j::DataType::FLOAT32); + NDArray scalar2('c', {}, std::vector{20.8}, nd4j::DataType::DOUBLE); x(0,{0}).assign(scalar1); x(1,{0}).assign(scalar2); @@ -233,9 +233,9 @@ TEST_F(MultiDataTypeTests, ndarray_assign_test1) { //////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test1) { NDArray x('f', {2,2}, {0, 1.5, 2.5, 3.5}, nd4j::DataType::HALF); - NDArray exp1('c', {}, {3}, nd4j::DataType::INT64); - NDArray exp2('c', {1,1}, {1}, nd4j::DataType::INT64); - NDArray exp3('c', {2}, {1,2}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{3}, nd4j::DataType::INT64); + NDArray exp2('c', {1,1}, std::vector{1}, nd4j::DataType::INT64); + NDArray exp3('c', {2}, std::vector{1,2}, nd4j::DataType::INT64); auto scalar1 = x.reduceAlongDimension(nd4j::reduce::CountNonZero, {}/*whole range*/); ASSERT_EQ(scalar1, exp1); @@ -250,7 +250,7 @@ TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test1) { //////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test2) { NDArray x('c', {2, 2}, {0, 1, 2, 3}, nd4j::DataType::INT32); - NDArray exp1('c', {}, {1.5}, nd4j::DataType::FLOAT32); + NDArray exp1('c', {}, std::vector{1.5}, nd4j::DataType::FLOAT32); NDArray exp2('c', {2}, {0.5,2.5}, nd4j::DataType::FLOAT32); auto scalar1 = x.reduceAlongDimension(nd4j::reduce::Mean, {}/*whole range*/); @@ -265,7 +265,7 @@ TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test2) { //////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test3) { NDArray x('c', {2, 2}, {0.5, 1.5, 2.5, 3.5}, nd4j::DataType::HALF); - NDArray exp1('c', {}, {8.}, nd4j::DataType::HALF); + NDArray exp1('c', {}, std::vector{8.}, nd4j::DataType::HALF); NDArray exp2('c', {2}, {2.,6.}, nd4j::DataType::HALF); auto scalar1 = x.reduceAlongDimension(nd4j::reduce::Sum, {}/*whole range*/); @@ -278,8 +278,8 @@ TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test3) { //////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test4) { NDArray x('c', {2, 2}, {10.5, 1.5, -2.5, -3.5}, nd4j::DataType::HALF); - NDArray exp1('c', {}, {1}, nd4j::DataType::BOOL); - NDArray exp2('c', {2}, {1,0}, nd4j::DataType::BOOL); + NDArray exp1('c', {}, std::vector{1}, nd4j::DataType::BOOL); + NDArray exp2('c', {2}, std::vector{1, 0}, nd4j::DataType::BOOL); auto scalar1 = x.reduceAlongDimension(nd4j::reduce::IsPositive, {}/*whole range*/); ASSERT_EQ(scalar1, exp1); @@ -291,8 +291,8 @@ TEST_F(MultiDataTypeTests, ndarray_reduceAlongDimension_test4) { //////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_varianceNumber_test1) { NDArray x('f', {2, 2}, {0, 1, 2, 3}, nd4j::DataType::INT64); - NDArray exp1('c', {}, {1.666666667}, nd4j::DataType::FLOAT32); - NDArray exp2('c', {}, {1.118033989}, nd4j::DataType::FLOAT32); + NDArray exp1('c', {}, std::vector{1.666666667}, nd4j::DataType::FLOAT32); + NDArray exp2('c', {}, std::vector{1.118033989}, nd4j::DataType::FLOAT32); auto scalar1 = x.varianceNumber(variance::SummaryStatsVariance); ASSERT_EQ(scalar1, exp1); @@ -475,8 +475,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorPlusEqual_test1) { if (!Environment::getInstance()->isExperimentalBuild()) return; - NDArray scalar1('c', {0}, {4}, nd4j::DataType::INT32); - NDArray scalar2('c', {0}, {1.5}, nd4j::DataType::HALF); + NDArray scalar1('c', {0}, std::vector{4}, nd4j::DataType::INT32); + NDArray scalar2('c', {0}, std::vector{1.5}, nd4j::DataType::HALF); NDArray x1('c', {2,3}, {1.5, 2.5, 3.5, 4.5, 5.5, 6.5}, nd4j::DataType::FLOAT32); NDArray x2('c', {3,2}, {10, 20, 30, 40, 50, 60}, nd4j::DataType::INT64); @@ -485,8 +485,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorPlusEqual_test1) { NDArray x5('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::HALF); NDArray x6('c', {2}, {0.4, 0.5}, nd4j::DataType::FLOAT32); - NDArray exp1('c', {0}, {5}, nd4j::DataType::INT32); - NDArray exp2('c', {0}, {6.5}, nd4j::DataType::HALF); + NDArray exp1('c', {0}, std::vector{5}, nd4j::DataType::INT32); + NDArray exp2('c', {0}, std::vector{6.5}, nd4j::DataType::HALF); NDArray exp3('c', {3,2}, {11, 22, 33, 44, 55, 66}, nd4j::DataType::INT64); NDArray exp4('c', {2,3}, {12.5, 24.5, 36.5, 48.5, 60.5, 72.5}, nd4j::DataType::FLOAT32); NDArray exp5('c', {2,2}, {0.4, 1.5, 2.4, 3.5}, nd4j::DataType::HALF); @@ -553,8 +553,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorMinusEqual_test1) { if (!Environment::getInstance()->isExperimentalBuild()) return; - NDArray scalar1('c', {0}, {4}, nd4j::DataType::INT32); - NDArray scalar2('c', {0}, {1.5}, nd4j::DataType::HALF); + NDArray scalar1('c', {0}, std::vector{4}, nd4j::DataType::INT32); + NDArray scalar2('c', {0}, std::vector{1.5}, nd4j::DataType::HALF); NDArray x1('c', {2,3}, {1.5, 2.5, 3.5, 4.5, 5.5, 6.5}, nd4j::DataType::FLOAT32); NDArray x2('c', {3,2}, {10, 20, 30, 40, 50, 60}, nd4j::DataType::INT64); @@ -563,8 +563,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorMinusEqual_test1) { NDArray x5('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::HALF); NDArray x6('c', {2}, {0.4, 0.5}, nd4j::DataType::FLOAT32); - NDArray exp1('c', {0}, {2}, nd4j::DataType::INT32); - NDArray exp2('c', {0}, {-0.5}, nd4j::DataType::HALF); + NDArray exp1('c', {0}, std::vector{2}, nd4j::DataType::INT32); + NDArray exp2('c', {0}, std::vector{-0.5}, nd4j::DataType::HALF); NDArray exp3('c', {3,2}, {8, 17, 26, 35, 44, 53}, nd4j::DataType::INT64); NDArray exp4('c', {2,3}, {-6.5, -14.5, -22.5, -30.5, -38.5, -46.5}, nd4j::DataType::FLOAT32); NDArray exp5('c', {2,2}, {0.4, -0.5, -1.6, -2.5}, nd4j::DataType::HALF); @@ -631,8 +631,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorMultiplyEqual_test1) { if (!Environment::getInstance()->isExperimentalBuild()) return; - NDArray scalar1('c', {0}, {3}, nd4j::DataType::INT32); - NDArray scalar2('c', {0}, {2.5}, nd4j::DataType::HALF); + NDArray scalar1('c', {0}, std::vector{3}, nd4j::DataType::INT32); + NDArray scalar2('c', {0}, std::vector{2.5}, nd4j::DataType::HALF); NDArray x1('c', {2,3}, {1.5, 2.5, 3.5, 4.5, 5.5, 6.5}, nd4j::DataType::FLOAT32); NDArray x2('c', {3,2}, {1, 2, 3, 4, 5, 6}, nd4j::DataType::INT64); @@ -641,8 +641,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorMultiplyEqual_test1) { NDArray x5('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::HALF); NDArray x6('c', {2}, {0.4, 0.5}, nd4j::DataType::FLOAT32); - NDArray exp1('c', {0}, {7}, nd4j::DataType::INT32); - NDArray exp2('c', {0}, {17.5}, nd4j::DataType::HALF); + NDArray exp1('c', {0}, std::vector{7}, nd4j::DataType::INT32); + NDArray exp2('c', {0}, std::vector{17.5}, nd4j::DataType::HALF); NDArray exp3('c', {3,2}, {1, 5, 10, 18, 27, 39}, nd4j::DataType::INT64); NDArray exp4('c', {2,3}, {1.5, 12.5, 35, 81, 148.5, 253.5}, nd4j::DataType::FLOAT32); NDArray exp5('c', {2,2}, {0., 0.5, 0.8, 1.5}, nd4j::DataType::HALF); @@ -709,8 +709,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorDivideEqual_test1) { if (!Environment::getInstance()->isExperimentalBuild()) return; - NDArray scalar1('c', {0}, {3}, nd4j::DataType::INT32); - NDArray scalar2('c', {0}, {2.5}, nd4j::DataType::HALF); + NDArray scalar1('c', {0}, std::vector{3}, nd4j::DataType::INT32); + NDArray scalar2('c', {0}, std::vector{2.5}, nd4j::DataType::HALF); NDArray x1('c', {2,3}, {1.5, 2.5, 3.5, 4.5, 5.5, 6.5}, nd4j::DataType::FLOAT32); NDArray x2('c', {3,2}, {10, 20, 30, 40, 50, 60}, nd4j::DataType::INT64); @@ -719,8 +719,8 @@ TEST_F(MultiDataTypeTests, ndarray_operatorDivideEqual_test1) { NDArray x5('c', {2,2}, {1, 2, 3, 4}, nd4j::DataType::HALF); NDArray x6('c', {2}, {0.4, 0.5}, nd4j::DataType::FLOAT32); - NDArray exp1('c', {0}, {1}, nd4j::DataType::INT32); - NDArray exp2('c', {0}, {2.5}, nd4j::DataType::HALF); + NDArray exp1('c', {0}, std::vector{1}, nd4j::DataType::INT32); + NDArray exp2('c', {0}, std::vector{2.5}, nd4j::DataType::HALF); NDArray exp3('c', {3,2}, {6, 8, 8, 8, 9, 9}, nd4j::DataType::INT64); NDArray exp4('c', {2,3}, {0.25, 0.3125, 0.4375, 0.5625, 0.611111111, 0.722222222}, nd4j::DataType::FLOAT32); NDArray exp5('c', {2,2}, {0.4, 0.25, 0.1333333, 0.125}, nd4j::DataType::HALF); @@ -792,10 +792,10 @@ TEST_F(MultiDataTypeTests, ndarray_reduceNumberFloat_test1) { NDArray x3('c', {2,2}, {0.5, 1.5, 2.5, 3.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {2,2}, {0, 1, 0, 1}, nd4j::DataType::BOOL); - NDArray exp1('c', {0}, {1.5}, nd4j::DataType::FLOAT32); - NDArray exp2('c', {0}, {2}, nd4j::DataType::HALF); - NDArray exp3('c', {0}, {2}, nd4j::DataType::DOUBLE); - NDArray exp4('c', {0}, {0.25},nd4j::DataType::FLOAT32); + NDArray exp1('c', {0}, std::vector{1.5}, nd4j::DataType::FLOAT32); + NDArray exp2('c', {0}, std::vector{2}, nd4j::DataType::HALF); + NDArray exp3('c', {0}, std::vector{2}, nd4j::DataType::DOUBLE); + NDArray exp4('c', {0}, std::vector{0.25},nd4j::DataType::FLOAT32); NDArray scalar = x1.reduceNumber(reduce::Mean); @@ -829,10 +829,10 @@ TEST_F(MultiDataTypeTests, ndarray_reduceNumberSame_test1) { NDArray x3('c', {2,2}, {0.5, 1.5, 2.5, 3.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {2,2}, {0, 1, 0, 1}, nd4j::DataType::BOOL); - NDArray exp1('c', {0}, {6}, nd4j::DataType::INT64); - NDArray exp2('c', {0}, {8}, nd4j::DataType::HALF); - NDArray exp3('c', {0}, {8}, nd4j::DataType::DOUBLE); - NDArray exp4('c', {0}, {1}, nd4j::DataType::BOOL); + NDArray exp1('c', {0}, std::vector{6}, nd4j::DataType::INT64); + NDArray exp2('c', {0}, std::vector{8}, nd4j::DataType::HALF); + NDArray exp3('c', {0}, std::vector{8}, nd4j::DataType::DOUBLE); + NDArray exp4('c', {0}, std::vector{1}, nd4j::DataType::BOOL); NDArray scalar = x1.reduceNumber(reduce::Sum); @@ -866,7 +866,7 @@ TEST_F(MultiDataTypeTests, ndarray_reduceNumberBool_test1) { NDArray x3('c', {2,2}, {0.5, 1.5, 2.5, 3.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {2,2}, {-2, -1, 0, 1}, nd4j::DataType::BOOL); - NDArray exp1('c', {0}, {1}, nd4j::DataType::BOOL); + NDArray exp1('c', {0}, std::vector{1}, nd4j::DataType::BOOL); NDArray scalar = x1.reduceNumber(reduce::IsFinite); ASSERT_EQ(scalar, exp1); @@ -899,10 +899,10 @@ TEST_F(MultiDataTypeTests, ndarray_reduceNumberLong_test1) { NDArray x3('c', {2,2}, {0.5, -1.5, 0, 3.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {2,2}, {0, 1, 0, 1}, nd4j::DataType::BOOL); - NDArray exp1('c', {0}, {3}, nd4j::DataType::INT64); - NDArray exp2('c', {0}, {4}, nd4j::DataType::INT64); - NDArray exp3('c', {0}, {3}, nd4j::DataType::INT64); - NDArray exp4('c', {0}, {2}, nd4j::DataType::INT64); + NDArray exp1('c', {0}, std::vector{3}, nd4j::DataType::INT64); + NDArray exp2('c', {0}, std::vector{4}, nd4j::DataType::INT64); + NDArray exp3('c', {0}, std::vector{3}, nd4j::DataType::INT64); + NDArray exp4('c', {0}, std::vector{2}, nd4j::DataType::INT64); NDArray scalar = x1.reduceNumber(reduce::CountNonZero); ASSERT_EQ(scalar, exp1); @@ -934,9 +934,9 @@ TEST_F(MultiDataTypeTests, ndarray_indexReduceNumber_test1) { NDArray x2('c', {2,2}, {0.5, 1.5, -4.5, 3.5}, nd4j::DataType::HALF); NDArray x3('c', {2,2}, {0, -1, 0, 1}, nd4j::DataType::BOOL); - NDArray exp1('c', {0}, {3}, nd4j::DataType::INT64); - NDArray exp2('c', {0}, {2}, nd4j::DataType::INT64); - NDArray exp3('c', {0}, {1}, nd4j::DataType::INT64); + NDArray exp1('c', {0}, std::vector{3}, nd4j::DataType::INT64); + NDArray exp2('c', {0}, std::vector{2}, nd4j::DataType::INT64); + NDArray exp3('c', {0}, std::vector{1}, nd4j::DataType::INT64); NDArray scalar = x1.indexReduceNumber(nd4j::indexreduce::IndexAbsoluteMax); ASSERT_EQ(scalar, exp1); @@ -1238,15 +1238,15 @@ TEST_F(MultiDataTypeTests, ndarray_applyTrueBroadcast_test1) { NDArray x7('c', {2}, {1, 2}, nd4j::DataType::INT64); NDArray x8('c', {2,2}, nd4j::DataType::BOOL); - NDArray x13('c', {0}, {3}, nd4j::DataType::INT64); - NDArray x14('c', {0}, {1.5}, nd4j::DataType::DOUBLE); + NDArray x13('c', {0}, std::vector{3}, nd4j::DataType::INT64); + NDArray x14('c', {0}, std::vector{1.5}, nd4j::DataType::DOUBLE); NDArray x15(nd4j::DataType::DOUBLE); NDArray x16('c', {2,2}, nd4j::DataType::DOUBLE); NDArray exp1('c', {2,2}, {11, 22, 31, 42}, nd4j::DataType::HALF); NDArray exp2('c', {2,2}, {11, 22, 31, 42}, nd4j::DataType::INT32); NDArray exp3('c', {2,2}, {1, 1, 1, 1}, nd4j::DataType::BOOL); - NDArray exp4('c', {0}, {4.5}, nd4j::DataType::DOUBLE); + NDArray exp4('c', {0}, std::vector{4.5}, nd4j::DataType::DOUBLE); NDArray exp5('c', {2,2}, {11.5, 21.5, 31.5, 41.5}, nd4j::DataType::DOUBLE); x1.applyTrueBroadcast(nd4j::BroadcastOpsTuple::Add(), x2, x3); @@ -1289,13 +1289,13 @@ TEST_F(MultiDataTypeTests, ndarray_applyTrueBroadcast_test2) { NDArray x1('c', {2,2}, {10, 20, 30, 40}, nd4j::DataType::HALF); NDArray x2('c', {2}, {10, 40}, nd4j::DataType::HALF); NDArray x3('c', {2,2}, nd4j::DataType::BOOL); - NDArray x4('c', {0}, {10}, nd4j::DataType::HALF); - NDArray x5('c', {0}, {20}, nd4j::DataType::HALF); + NDArray x4('c', {0}, std::vector{10}, nd4j::DataType::HALF); + NDArray x5('c', {0}, std::vector{20}, nd4j::DataType::HALF); NDArray x6(nd4j::DataType::BOOL); NDArray exp1('c', {2,2}, {1, 0, 0, 1}, nd4j::DataType::BOOL); NDArray exp2('c', {2,2}, {1, 0, 0, 0}, nd4j::DataType::BOOL); - NDArray exp3('c', {0}, {0}, nd4j::DataType::BOOL); + NDArray exp3('c', {0}, std::vector{0}, nd4j::DataType::BOOL); x1.applyTrueBroadcast(BroadcastBoolOpsTuple(nd4j::scalar::EqualTo, nd4j::pairwise::EqualTo, nd4j::broadcast::EqualTo), x2, x3); ASSERT_EQ(x3, exp1); @@ -1459,16 +1459,16 @@ TEST_F(MultiDataTypeTests, ndarray_applyIndexedLambda_test1) { ////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_applyPairwiseLambda_test1) { - NDArray x1('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::DOUBLE); - NDArray x2('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::INT64); - NDArray x3('c', {2,2}, {0, 1.5, 2.5, 3.5}, nd4j::DataType::FLOAT32); + NDArray x1('c', {2,2}, {0., 1, 2, 3}, nd4j::DataType::DOUBLE); + NDArray x2('c', {2,2}, {0., 1, 2, 3}, nd4j::DataType::INT64); + NDArray x3('c', {2,2}, {0., 1.5, 2.5, 3.5}, nd4j::DataType::FLOAT32); NDArray x4('c', {2,2}, nd4j::DataType::DOUBLE); NDArray x5('c', {2,2}, {0, 1.5, 2.5, 3.5}, nd4j::DataType::FLOAT32); NDArray x6('c', {2,2}, {0.1, -1, -1, 0.1}, nd4j::DataType::BOOL); NDArray x7('c', {2,2}, nd4j::DataType::BOOL); NDArray other1('c', {2,2}, {0.1, 0.1, 0.1, 0.1}, nd4j::DataType::FLOAT32); NDArray other2('c', {2,2}, {0.1, 0.1, 0.1, 0.1}, nd4j::DataType::DOUBLE); - NDArray other3('c', {2,2}, {0, -1, -2, -3}, nd4j::DataType::INT64); + NDArray other3('c', {2,2}, {0., -1, -2, -3}, nd4j::DataType::INT64); NDArray other4('c', {2,2}, {1, 0, 0.1, 0}, nd4j::DataType::BOOL); auto func1 = [](float elem1, float elem2) { return elem1 + elem2; }; @@ -1478,10 +1478,10 @@ TEST_F(MultiDataTypeTests, ndarray_applyPairwiseLambda_test1) { auto func5 = [](float elem1, int elem2) { return elem1 - elem2; }; NDArray exp1('c', {2,2}, {0.1, 1.1, 2.1, 3.1}, nd4j::DataType::DOUBLE); - NDArray exp2('c', {2,2}, {0, 0, 0, 0}, nd4j::DataType::INT64); + NDArray exp2('c', {2,2}, {0., 0, 0, 0}, nd4j::DataType::INT64); NDArray exp3('c', {2,2}, {0.1, 1.1, 2.1, 3.1}, nd4j::DataType::FLOAT32); NDArray exp4('c', {2,2}, {0.1, 1.6, 2.6, 3.6}, nd4j::DataType::FLOAT32); - NDArray exp5('c', {2,2}, {0, 1, 0, 1}, nd4j::DataType::BOOL); + NDArray exp5('c', {2,2}, {0., 1, 0, 1}, nd4j::DataType::BOOL); x1.applyPairwiseLambda(other2, func1, x4); ASSERT_EQ(x4, exp1); @@ -1505,16 +1505,16 @@ TEST_F(MultiDataTypeTests, ndarray_applyPairwiseLambda_test1) { ////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_applyIndexedPairwiseLambda_test1) { - NDArray x1('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::DOUBLE); - NDArray x2('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::INT64); - NDArray x3('c', {2,2}, {0, 1.5, 2.5, 3.5}, nd4j::DataType::FLOAT32); + NDArray x1('c', {2,2}, {0., 1, 2, 3}, nd4j::DataType::DOUBLE); + NDArray x2('c', {2,2}, {0., 1, 2, 3}, nd4j::DataType::INT64); + NDArray x3('c', {2,2}, {0., 1.5, 2.5, 3.5}, nd4j::DataType::FLOAT32); NDArray x4('c', {2,2}, nd4j::DataType::DOUBLE); NDArray x5('c', {2,2}, {0, 1.5, 2.5, 3.5}, nd4j::DataType::FLOAT32); NDArray x6('c', {2,2}, {0.1, -1, -1, 0.1}, nd4j::DataType::BOOL); NDArray x7('c', {2,2}, nd4j::DataType::BOOL); NDArray other1('c', {2,2}, {0.1, 0.1, 0.1, 0.1}, nd4j::DataType::FLOAT32); NDArray other2('c', {2,2}, {0.1, 0.1, 0.1, 0.1}, nd4j::DataType::DOUBLE); - NDArray other3('c', {2,2}, {0, -1, -2, -3}, nd4j::DataType::INT64); + NDArray other3('c', {2,2}, {0., -1, -2, -3}, nd4j::DataType::INT64); NDArray other4('c', {2,2}, {1, 0, 0.1, 0}, nd4j::DataType::BOOL); auto func1 = [](Nd4jLong idx, float elem1, float elem2) { return elem1 + elem2 + idx; }; @@ -1524,10 +1524,10 @@ TEST_F(MultiDataTypeTests, ndarray_applyIndexedPairwiseLambda_test1) { auto func5 = [](Nd4jLong idx, float elem1, int elem2) { return elem1 - elem2 + idx; }; NDArray exp1('c', {2,2}, {0.1, 2.1, 4.1, 6.1}, nd4j::DataType::DOUBLE); - NDArray exp2('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::INT64); + NDArray exp2('c', {2,2}, {0., 1, 2, 3}, nd4j::DataType::INT64); NDArray exp3('c', {2,2}, {0.1, 2.1, 4.1, 6.1}, nd4j::DataType::FLOAT32); NDArray exp4('c', {2,2}, {0.1, 2.6, 4.6, 6.6}, nd4j::DataType::FLOAT32); - NDArray exp5('c', {2,2}, {0, 1, 1, 1}, nd4j::DataType::BOOL); + NDArray exp5('c', {2,2}, {0., 1, 1, 1}, nd4j::DataType::BOOL); x1.applyIndexedPairwiseLambda(other2, func1, x4); ASSERT_EQ(x4, exp1); @@ -1551,25 +1551,25 @@ TEST_F(MultiDataTypeTests, ndarray_applyIndexedPairwiseLambda_test1) { ////////////////////////////////////////////////////////////////////////////// TEST_F(MultiDataTypeTests, ndarray_applyTriplewiseLambda_test1) { - NDArray x1('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::DOUBLE); - NDArray x2('c', {2,2}, {0, -1, -2, -3}, nd4j::DataType::DOUBLE); + NDArray x1('c', {2,2}, {0., 1, 2, 3}, nd4j::DataType::DOUBLE); + NDArray x2('c', {2,2}, {0., -1, -2, -3}, nd4j::DataType::DOUBLE); NDArray x3('c', {2,2}, {0, -1.5, -2.5, -3.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {2,2}, nd4j::DataType::DOUBLE); - NDArray x5('c', {2,2}, {0, 1, 2, 3}, nd4j::DataType::INT32); - NDArray x6('c', {2,2}, {0, -1, -2, -3}, nd4j::DataType::INT32); - NDArray x7('c', {2,2}, {0, 10, 20, 30}, nd4j::DataType::INT32); + NDArray x5('c', {2,2}, {0., 1, 2, 3}, nd4j::DataType::INT32); + NDArray x6('c', {2,2}, {0., -1, -2, -3}, nd4j::DataType::INT32); + NDArray x7('c', {2,2}, {0., 10, 20, 30}, nd4j::DataType::INT32); - NDArray x8('c', {2,2}, {0, 1, 0, 1}, nd4j::DataType::BOOL); - NDArray x9('c', {2,2}, {1, 1, 0, 1}, nd4j::DataType::BOOL); - NDArray x10('c', {2,2}, {0, 0, 0, 0}, nd4j::DataType::BOOL); + NDArray x8('c', {2,2}, {0., 1, 0, 1}, nd4j::DataType::BOOL); + NDArray x9('c', {2,2}, {1., 1, 0, 1}, nd4j::DataType::BOOL); + NDArray x10('c', {2,2}, {0., 0, 0, 0}, nd4j::DataType::BOOL); auto func1 = [](double elem1, float elem2, int elem3) { return elem1 + elem2 + elem3; }; auto func2 = [](float elem1, float elem2, float elem3) { return elem1 + elem2 + elem3; }; auto func3 = [](int elem1, int elem2, int elem3) { return elem1 + elem2 + elem3; }; auto func4 = [](bool elem1, bool elem2, bool elem3) { return elem1 + elem2 + elem3; }; - NDArray exp('c', {2,2}, {1, 1, 0, 1}, nd4j::DataType::BOOL); + NDArray exp('c', {2,2}, {1., 1, 0, 1}, nd4j::DataType::BOOL); x1.applyTriplewiseLambda(x2, x3, func1, x4); ASSERT_EQ(x4, x2); @@ -1590,7 +1590,7 @@ TEST_F(MultiDataTypeTests, ndarray_applyTriplewiseLambda_test1) { TEST_F(MultiDataTypeTests, ndarray_applyIndexReduce_test1) { NDArray x1('c', {2,3}, {0, 1, 2, 3, 4, 5}, nd4j::DataType::DOUBLE); - NDArray exp1('c', {}, {5}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{5}, nd4j::DataType::INT64); NDArray exp2('c', {2}, {2,2}, nd4j::DataType::INT64); NDArray exp3('c', {3}, {1,1,1}, nd4j::DataType::INT64); @@ -1608,10 +1608,10 @@ TEST_F(MultiDataTypeTests, ndarray_applyIndexReduce_test1) { TEST_F(MultiDataTypeTests, ndarray_applyIndexReduce_test2) { NDArray x1('c', {2,3}, {0, 1, 2, 3, 4, 5}, nd4j::DataType::DOUBLE); - NDArray scalar('c', {}, {5}, nd4j::DataType::INT64); + NDArray scalar('c', {}, std::vector{5}, nd4j::DataType::INT64); NDArray vec1('c', {2}, {2,2}, nd4j::DataType::INT64); NDArray vec2('c', {3}, {1,1,1}, nd4j::DataType::INT64); - NDArray exp1('c', {}, {5}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{5}, nd4j::DataType::INT64); NDArray exp2('c', {2}, {2,2}, nd4j::DataType::INT64); NDArray exp3('c', {3}, {1,1,1}, nd4j::DataType::INT64); @@ -1632,8 +1632,8 @@ TEST_F(MultiDataTypeTests, applyReduce3_test1) { NDArray x2('c', {2,2}, {-1,-2,-3,-4}, nd4j::DataType::INT32); NDArray x3('c', {2,2}, {1.5,1.5,1.5,1.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {2,2}, {1,2,3,4}, nd4j::DataType::DOUBLE); - NDArray exp1('c', {}, {-30}, nd4j::DataType::FLOAT32); - NDArray exp2('c', {}, {15}, nd4j::DataType::DOUBLE); + NDArray exp1('c', {}, std::vector{-30}, nd4j::DataType::FLOAT32); + NDArray exp2('c', {}, std::vector{15}, nd4j::DataType::DOUBLE); auto result = x1.applyReduce3(reduce3::Dot, x2); ASSERT_EQ(result, exp1); @@ -1654,8 +1654,8 @@ TEST_F(MultiDataTypeTests, applyReduce3_test2) { NDArray x7('c', {2,3}, {1.5,1.5,1.5,1.5,1.5,1.5}, nd4j::DataType::DOUBLE); NDArray x8('c', {2,3}, {1,2,3,4,5,6}, nd4j::DataType::DOUBLE); - NDArray exp1('c', {}, {-30}, nd4j::DataType::FLOAT32); - NDArray exp2('c', {}, {15}, nd4j::DataType::DOUBLE); + NDArray exp1('c', {}, std::vector{-30}, nd4j::DataType::FLOAT32); + NDArray exp2('c', {}, std::vector{15}, nd4j::DataType::DOUBLE); NDArray exp3('c', {3}, {-18,-20,-18}, nd4j::DataType::FLOAT32); NDArray exp4('c', {2}, {-28,-28}, nd4j::DataType::FLOAT32); NDArray exp5('c', {3}, {7.5,10.5,13.5}, nd4j::DataType::DOUBLE); diff --git a/libnd4j/tests_cpu/layers_tests/NDArrayConstructorsTests.cu b/libnd4j/tests_cpu/layers_tests/NDArrayConstructorsTests.cu index 33384d4d8..0c0c102ac 100644 --- a/libnd4j/tests_cpu/layers_tests/NDArrayConstructorsTests.cu +++ b/libnd4j/tests_cpu/layers_tests/NDArrayConstructorsTests.cu @@ -184,7 +184,7 @@ TEST_F(NDArrayConstructorsTests, test_linspace_1) { TEST_F(NDArrayConstructorsTests, test_constructor_10) { NDArray scalar1(nd4j::DataType::DOUBLE); // scalar1 = 0 - NDArray scalar2('c', {}, {0}); + NDArray scalar2('c', {}, std::vector{0}); ASSERT_TRUE(scalar1.isActualOnDeviceSide()); ASSERT_TRUE(!scalar1.isActualOnHostSide()); diff --git a/libnd4j/tests_cpu/layers_tests/NDArrayCudaBasicsTests.cu b/libnd4j/tests_cpu/layers_tests/NDArrayCudaBasicsTests.cu index c6c0a1bd8..46f962dda 100644 --- a/libnd4j/tests_cpu/layers_tests/NDArrayCudaBasicsTests.cu +++ b/libnd4j/tests_cpu/layers_tests/NDArrayCudaBasicsTests.cu @@ -1226,8 +1226,8 @@ TEST_F(NDArrayCudaBasicsTests, applyReduce3_3) { NDArray x3('c', {3,2}, {1.5,1.5,1.5,1.5,1.5,1.5}, nd4j::DataType::DOUBLE); NDArray x4('c', {3,2}, {1,2,3,4,5,6}, nd4j::DataType::DOUBLE); - NDArray exp1('c', {}, {-204}, nd4j::DataType::FLOAT32); - NDArray exp2('c', {}, {31.5}, nd4j::DataType::DOUBLE); + NDArray exp1('c', {}, std::vector{-204}, nd4j::DataType::FLOAT32); + NDArray exp2('c', {}, std::vector{31.5}, nd4j::DataType::DOUBLE); auto z = x1.applyReduce3(reduce3::Dot, x2); @@ -1260,7 +1260,7 @@ TEST_F(NDArrayCudaBasicsTests, applyAllReduce3_1) { NDArray exp2('c', {6,4}, {-36.f, -44.f, -52.f, -60.f,-42.f, -52.f, -62.f, -72.f, 2.f, 0.f, -2.f, -4.f, 6.f, 4.f, 2.f, 0.f, 10.f, 8.f, 6.f, 4.f, 14.f, 12.f, 10.f, 8.f}, nd4j::DataType::FLOAT32); - NDArray exp3('c', {1,1}, {31.5}, nd4j::DataType::DOUBLE); + NDArray exp3('c', {1,1}, std::vector{31.5}, nd4j::DataType::DOUBLE); NDArray exp4('c', {3,3}, {4.5, 10.5, 16.5,4.5, 10.5, 16.5,4.5, 10.5, 16.5}, nd4j::DataType::DOUBLE); auto z = x1.applyAllReduce3(reduce3::Dot, x2, {0,2}); @@ -1292,15 +1292,15 @@ TEST_F(NDArrayCudaBasicsTests, applyIndexReduce_test1) { NDArray x('c', {2,3}, {0, 10, 1, 2, 2.5,-4}, nd4j::DataType::DOUBLE); - NDArray scalar('c', {}, {100}, nd4j::DataType::INT64); + NDArray scalar('c', {}, std::vector{100}, nd4j::DataType::INT64); NDArray vec1('c', {2}, {100,100}, nd4j::DataType::INT64); NDArray vec2('c', {3}, {100,100,100}, nd4j::DataType::INT64); - NDArray exp1('c', {}, {1}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{1}, nd4j::DataType::INT64); NDArray exp2('c', {2}, {1,1}, nd4j::DataType::INT64); NDArray exp3('c', {3}, {1,0,0}, nd4j::DataType::INT64); - NDArray exp4('c', {}, {2}, nd4j::DataType::INT64); + NDArray exp4('c', {}, std::vector{2}, nd4j::DataType::INT64); NDArray exp5('c', {2}, {1,1}, nd4j::DataType::INT64); NDArray exp6('c', {3}, {1,0,0}, nd4j::DataType::INT64); @@ -1331,11 +1331,11 @@ TEST_F(NDArrayCudaBasicsTests, applyIndexReduce_test2) { NDArray x('c', {2,3}, {0, 10, 1, 2, 2.5,-4}, nd4j::DataType::DOUBLE); - NDArray exp1('c', {}, {1}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{1}, nd4j::DataType::INT64); NDArray exp2('c', {2}, {1,1}, nd4j::DataType::INT64); NDArray exp3('c', {3}, {1,0,0}, nd4j::DataType::INT64); - NDArray exp4('c', {}, {2}, nd4j::DataType::INT64); + NDArray exp4('c', {}, std::vector{2}, nd4j::DataType::INT64); NDArray exp5('c', {2}, {1,1}, nd4j::DataType::INT64); NDArray exp6('c', {3}, {1,0,0}, nd4j::DataType::INT64); @@ -1365,13 +1365,13 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_float_test1) { NDArray x('c', {2,3,2}, {1,2,3,4,5,6,7,8,-1,-2,-3,-4,}, nd4j::DataType::INT32); - NDArray z1('c', {}, {100}, nd4j::DataType::DOUBLE); + NDArray z1('c', {}, std::vector{100}, nd4j::DataType::DOUBLE); NDArray z2('c', {2,2}, {100,100,100,100}, nd4j::DataType::FLOAT32); NDArray z3('c', {3}, {100,100,100}, nd4j::DataType::DOUBLE); NDArray z4('c', {3,2}, {100,100,100,100,100,100}, nd4j::DataType::FLOAT32); NDArray z5('c', {2}, {100,100}, nd4j::DataType::FLOAT32); - NDArray exp1('c', {}, {2.166667}, nd4j::DataType::DOUBLE); + NDArray exp1('c', {}, std::vector{2.166667}, nd4j::DataType::DOUBLE); NDArray exp2('c', {2,2}, {3.f,4.f,1.f,0.666667f}, nd4j::DataType::FLOAT32); NDArray exp3('c', {3}, {4.5,1,1}, nd4j::DataType::DOUBLE); NDArray exp4('c', {3,2}, {4,5,1,1,1,1}, nd4j::DataType::FLOAT32); @@ -1403,7 +1403,7 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_float_test2) { NDArray x('c', {2,3,2}, {1,2,3,4,5,6,7,8,-1,-2,-3,-4,}, nd4j::DataType::DOUBLE); - NDArray exp1('c', {}, {2.166667}, nd4j::DataType::DOUBLE); + NDArray exp1('c', {}, std::vector{2.166667}, nd4j::DataType::DOUBLE); NDArray exp2('c', {2,2}, {3,4,1,0.666667}, nd4j::DataType::DOUBLE); NDArray exp3('c', {3}, {4.5,1,1}, nd4j::DataType::DOUBLE); NDArray exp4('c', {3,2}, {4,5,1,1,1,1}, nd4j::DataType::DOUBLE); @@ -1477,13 +1477,13 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_same_test1) { NDArray x('c', {2,3,2}, {1.5f,2.f,3.f,4.f,5.f,6.f,7.5f,8.f,-1.f,-2.f,-3.5f,-4.f}, nd4j::DataType::FLOAT32); - NDArray z1('c', {}, {100}, nd4j::DataType::FLOAT32); + NDArray z1('c', {}, std::vector{100}, nd4j::DataType::FLOAT32); NDArray z2('c', {2,2}, {100,100,100,100}, nd4j::DataType::FLOAT32); NDArray z3('c', {3}, {100,100,100}, nd4j::DataType::FLOAT32); NDArray z4('c', {3,2}, {100,100,100,100,100,100}, nd4j::DataType::FLOAT32); NDArray z5('c', {2}, {100,100}, nd4j::DataType::FLOAT32); - NDArray exp1('c', {}, {26.5f}, nd4j::DataType::FLOAT32); + NDArray exp1('c', {}, std::vector{26.5f}, nd4j::DataType::FLOAT32); NDArray exp2('c', {2,2}, {9.5f,12.f,3.f,2.f}, nd4j::DataType::FLOAT32); NDArray exp3('c', {3}, {19.f,4.f,3.5f}, nd4j::DataType::FLOAT32); NDArray exp4('c', {3,2}, {9.f,10.f,2.f,2.f,1.5f,2.f}, nd4j::DataType::FLOAT32); @@ -1515,7 +1515,7 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_same_test2) { NDArray x('c', {2,3,2}, {1.5,2,3,4,5,6,7.5,8,-1,-2,-3.5,-4,}, nd4j::DataType::INT64); - NDArray exp1('c', {}, {26}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{26}, nd4j::DataType::INT64); NDArray exp2('c', {2,2}, {9,12,3,2}, nd4j::DataType::INT64); NDArray exp3('c', {3}, {18,4,4}, nd4j::DataType::INT64); NDArray exp4('c', {3,2}, {8,10,2,2,2,2}, nd4j::DataType::INT64); @@ -1547,13 +1547,13 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_bool_test1) { NDArray x('c', {2,3,2}, {0.5,2,3,-4,5,6,-7.5,8,-1,-0.5,-3.5,4}, nd4j::DataType::DOUBLE); - NDArray z1('c', {}, {true}, nd4j::DataType::BOOL); + NDArray z1('c', {}, std::vector{true}, nd4j::DataType::BOOL); NDArray z2('c', {2,2}, {true,true,true,true}, nd4j::DataType::BOOL); NDArray z3('c', {3}, {true,true,true}, nd4j::DataType::BOOL); NDArray z4('c', {3,2}, {true,true,true,true,true,true}, nd4j::DataType::BOOL); NDArray z5('c', {2}, {true,true}, nd4j::DataType::BOOL); - NDArray exp1('c', {}, {true}, nd4j::DataType::BOOL); + NDArray exp1('c', {}, std::vector{true}, nd4j::DataType::BOOL); NDArray exp2('c', {2,2}, {true,true,false,true}, nd4j::DataType::BOOL); NDArray exp3('c', {3}, {true,true,true}, nd4j::DataType::BOOL); NDArray exp4('c', {3,2}, {true,true,true,false,true,true}, nd4j::DataType::BOOL); @@ -1585,7 +1585,7 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_bool_test2) { NDArray x('c', {2,3,2}, {0.5,2,3,-4,5,6,-7.5,8,-1,-0.5,-3.5,4}, nd4j::DataType::INT32); - NDArray exp1('c', {}, {1}, nd4j::DataType::BOOL); + NDArray exp1('c', {}, std::vector{1}, nd4j::DataType::BOOL); NDArray exp2('c', {2,2}, {1,1,0,1}, nd4j::DataType::BOOL); NDArray exp3('c', {3}, {1,1,1}, nd4j::DataType::BOOL); NDArray exp4('c', {3,2}, {0,1,1,0,1,1}, nd4j::DataType::BOOL); @@ -1617,13 +1617,13 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_long_test1) { NDArray x('c', {2,3,2}, {0.5f,2.f,3.f,-0.f,5.f,6.f,-7.5f,0.f,-1.f,-0.5f,-3.5f,4.f}, nd4j::DataType::FLOAT32); - NDArray z1('c', {}, {100}, nd4j::DataType::INT64); + NDArray z1('c', {}, std::vector{100}, nd4j::DataType::INT64); NDArray z2('c', {2,2}, {100,100,100,100}, nd4j::DataType::INT64); NDArray z3('c', {3}, {100,100,100}, nd4j::DataType::INT64); NDArray z4('c', {3,2}, {100,100,100,100,100,100}, nd4j::DataType::INT64); NDArray z5('c', {2}, {100,100}, nd4j::DataType::INT64); - NDArray exp1('c', {}, {2}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{2}, nd4j::DataType::INT64); NDArray exp2('c', {2,2}, {0,1,0,1}, nd4j::DataType::INT64); NDArray exp3('c', {3}, {1,1,0}, nd4j::DataType::INT64); NDArray exp4('c', {3,2}, {0,1,0,1,0,0}, nd4j::DataType::INT64); @@ -1655,7 +1655,7 @@ TEST_F(NDArrayCudaBasicsTests, reduceAlongDimension_long_test2) { NDArray x('c', {2,3,2}, {0.5,2,3,-0,5,6,-7.5,0,-1,-0.5,-3.5,4}, nd4j::DataType::INT32); - NDArray exp1('c', {}, {4}, nd4j::DataType::INT64); + NDArray exp1('c', {}, std::vector{4}, nd4j::DataType::INT64); NDArray exp2('c', {2,2}, {1,1,0,2}, nd4j::DataType::INT64); NDArray exp3('c', {3}, {2,2,0}, nd4j::DataType::INT64); NDArray exp4('c', {3,2}, {1,1,0,2,0,0}, nd4j::DataType::INT64); diff --git a/libnd4j/tests_cpu/layers_tests/ParityOpsTests.cpp b/libnd4j/tests_cpu/layers_tests/ParityOpsTests.cpp index 987817136..a53d71a65 100644 --- a/libnd4j/tests_cpu/layers_tests/ParityOpsTests.cpp +++ b/libnd4j/tests_cpu/layers_tests/ParityOpsTests.cpp @@ -692,7 +692,7 @@ TEST_F(ParityOpsTests, Test_Bias_Add_1) { TEST_F(ParityOpsTests, Test_Scatter_Add_1) { auto matrix = NDArrayFactory::create('c', {2, 2}, {1, 2, 3, 4}); - NDArray idc('c', {1}, {0}, nd4j::DataType::INT64); + NDArray idc('c', {1}, std::vector({0}), nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2}, {1, 1}); auto exp = NDArrayFactory::create('c', {2, 2}, {2, 3, 3, 4}); @@ -710,7 +710,7 @@ TEST_F(ParityOpsTests, Test_Scatter_Add_1) { TEST_F(ParityOpsTests, Test_Scatter_Add_2) { auto vec = NDArrayFactory::create('c', {4}, {1, 2, 3, 4}); - NDArray idc('c', {1, 4}, {0, 1, 2, 3}, nd4j::DataType::INT64); + NDArray idc('c', {1, 4}, {0., 1, 2, 3}, nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 4}, {1, 1, 1, 1}); auto exp = NDArrayFactory::create('c', {1, 4}, {2, 3, 4, 5}); @@ -727,7 +727,7 @@ TEST_F(ParityOpsTests, Test_Scatter_Add_2) { TEST_F(ParityOpsTests, Test_Scatter_Add_3) { auto matrix = NDArrayFactory::create('c', {2, 2, 2}, {1, 2, 3, 4, 5, 6, 7, 8}); - NDArray idc('c', {1}, {0}, nd4j::DataType::INT64); + NDArray idc('c', {1}, std::vector({0}), nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2, 2}, {1, 1, 1, 1}); auto exp = NDArrayFactory::create('c', {2, 2, 2}, {2, 3, 4, 5, 5, 6, 7, 8}); @@ -744,7 +744,7 @@ TEST_F(ParityOpsTests, Test_Scatter_Add_3) { TEST_F(ParityOpsTests, Test_Scatter_Add_4) { auto matrix = NDArrayFactory::create('c', {2, 2, 2}, {1, 2, 3, 4, 5, 6, 7, 8}); - NDArray idc('c', {1, 2}, {0, 0}, nd4j::DataType::INT64); + NDArray idc('c', {1, 2}, std::vector{0, 0}, nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2, 2, 2}, {1, 1, 1, 1, 1, 1, 1, 1}); auto exp = NDArrayFactory::create('c', {2, 2, 2}, {3, 4, 5, 6, 5, 6, 7, 8}); @@ -761,7 +761,7 @@ TEST_F(ParityOpsTests, Test_Scatter_Add_4) { TEST_F(ParityOpsTests, Test_Scatter_Add_5) { auto matrix = NDArrayFactory::create('c', {2, 2, 3}, {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); - NDArray idc('c', {2, 2}, {1, 1, 0, 0}, nd4j::DataType::INT64); + NDArray idc('c', {2, 2}, {1., 1, 0, 0}, nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {2, 2, 2, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}); auto exp = NDArrayFactory::create('c', {2, 2, 3}, {9., 11., 13.,15., 17., 19., 9., 11., 13.,15., 17., 19.}); @@ -796,7 +796,7 @@ TEST_F(ParityOpsTests, Test_Scatter_Add_6) { TEST_F(ParityOpsTests, Test_Scatter_Add_7) { auto matrix = NDArrayFactory::create('c', {10, 3}, {1.f,2.f,3.f,4.f,5.f,6.f,7.f,8.f,9.f,10.f,11.f,12.f,13.f,14.f,15.f,16.f,17.f,18.f,19.f,20.f,21.f,22.f,23.f,24.f,25.f,26.f,27.f,28.f,29.f,30.f}); - NDArray idc('c', {}, {5}, nd4j::DataType::INT64); + NDArray idc('c', {}, std::vector{5}, nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {3}, {10.f, 20.f, 30.f}); auto exp = NDArrayFactory::create('c', {10, 3}, {1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f,11.f,12.f, 13.f,14.f,15.f, 26.f,37.f,48.f, 19.f,20.f,21.f, 22.f,23.f,24.f, 25.f,26.f,27.f, 28.f,29.f,30.f}); @@ -845,7 +845,7 @@ TEST_F(ParityOpsTests, Test_Scatter_Add_9) { //////////////////////////////////////////////////////////////////// TEST_F(ParityOpsTests, scatterMax_test1) { auto matrix = NDArrayFactory::create('c', {2, 2}, {1, 2, 3, 4}); - NDArray idc('c', {1}, {0.}, nd4j::DataType::INT64); + NDArray idc('c', {1}, std::vector{0.}, nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2}, {10, 1}); auto exp = NDArrayFactory::create('c', {2, 2}, {10, 2, 3, 4}); @@ -879,7 +879,7 @@ TEST_F(ParityOpsTests, scatterMax_test2) { TEST_F(ParityOpsTests, scatterMax_test3) { auto matrix = NDArrayFactory::create('c', {2, 2, 2}, {1, 2, 3, 4, 5, 6, 7, 8}); - NDArray idc('c', {1}, {0}, nd4j::DataType::INT64); + NDArray idc('c', {1}, std::vector({0}), nd4j::DataType::INT64); auto updates = NDArrayFactory::create('c', {1, 2, 2}, {10, 1, 30, 1}); auto exp = NDArrayFactory::create('c', {2, 2, 2}, {10, 2, 30, 4, 5, 6, 7, 8}); @@ -896,7 +896,7 @@ TEST_F(ParityOpsTests, scatterMax_test3) { TEST_F(ParityOpsTests, scatterMax_test4) { auto matrix = NDArrayFactory::create('c', {2, 2, 2}, {1, 2, 3, 4, 5, 6, 7, 8}); - NDArray idc('c', {1,2}, {0,0}, nd4j::DataType::INT32); + NDArray idc('c', {1,2}, std::vector{0.,0}, nd4j::DataType::INT32); auto updates = NDArrayFactory::create('c', {1, 2, 2, 2}, {1,10,1,10, 1,1,10,1.}); auto exp = NDArrayFactory::create('c', {2, 2, 2}, {1, 10, 10, 10, 5, 6, 7, 8}); @@ -948,7 +948,7 @@ TEST_F(ParityOpsTests, scatterMax_test6) { TEST_F(ParityOpsTests, scatterMin_test1) { auto matrix = NDArrayFactory::create('c', {2, 2}, {1, 2, 3, 4}); - NDArray idc('c', {1}, {0}, nd4j::DataType::INT32); + NDArray idc('c', {1}, std::vector({0}), nd4j::DataType::INT32); auto updates = NDArrayFactory::create('c', {1, 2}, {-1, 1}); auto exp = NDArrayFactory::create('c', {2, 2}, {-1, 1, 3, 4}); @@ -982,7 +982,7 @@ TEST_F(ParityOpsTests, scatterMin_test2) { TEST_F(ParityOpsTests, scatterMin_test3) { auto matrix = NDArrayFactory::create('c', {2, 2, 2}, {1, 2, 3, 4, 5, 6, 7, 8}); - NDArray idc('c', {1}, {0}, nd4j::DataType::INT32); + NDArray idc('c', {1}, std::vector({0}), nd4j::DataType::INT32); auto updates = NDArrayFactory::create('c', {1, 2, 2}, {10, 1, 30, 2}); auto exp = NDArrayFactory::create('c', {2, 2, 2}, {1, 1, 3, 2, 5, 6, 7, 8}); @@ -999,7 +999,7 @@ TEST_F(ParityOpsTests, scatterMin_test3) { TEST_F(ParityOpsTests, scatterMin_test4) { auto matrix = NDArrayFactory::create('c', {2, 2, 2}, {1, 2, 3, 4, 5, 6, 7, 8}); - NDArray idc('c', {1,2}, {0,0}, nd4j::DataType::INT32); + NDArray idc('c', {1,2}, std::vector{0.,0}, nd4j::DataType::INT32); auto updates = NDArrayFactory::create('c', {1, 2, 2, 2}, {1,10,1,10, 1,1,10,1.}); auto exp = NDArrayFactory::create('c', {2, 2, 2}, {1, 1, 1, 1, 5, 6, 7, 8}); diff --git a/libnd4j/tests_cpu/layers_tests/RNGTests.cpp b/libnd4j/tests_cpu/layers_tests/RNGTests.cpp index e6cf01521..b40b74939 100644 --- a/libnd4j/tests_cpu/layers_tests/RNGTests.cpp +++ b/libnd4j/tests_cpu/layers_tests/RNGTests.cpp @@ -1005,24 +1005,24 @@ TEST_F(RNGTests, test_uniform_119) { } TEST_F(RNGTests, test_multinomial_1) { - + NDArray probs('f', { 3, 3 }, { 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3 }, nd4j::DataType::FLOAT32); - NDArray expected('f', { 3, 3 }, { 0, 1, 2, 2, 0, 0, 1, 2, 1 }, nd4j::DataType::INT64); + NDArray expected('f', { 3, 3 }, { 0., 1, 2, 2, 0, 0, 1, 2, 1 }, nd4j::DataType::INT64); NDArray output('f', { 3, 3 }, nd4j::DataType::INT64); - NDArray samples('f', { 1 }, { 3 }, nd4j::DataType::INT32); - + NDArray samples('f', { 1 }, std::vector({3}), nd4j::DataType::INT32); + nd4j::ops::random_multinomial op; RandomGenerator rng(1234, 1234); ASSERT_EQ(Status::OK(), op.execute(rng, { &probs, &samples }, { &output }, {}, { 0, INT64}, {}, {}, false) ); ASSERT_TRUE(expected.isSameShape(output)); ASSERT_TRUE(expected.equalsTo(output)); - + NDArray probsZ('c', { 1, 3 }, { 0.3, 0.3, 0.3 }, nd4j::DataType::FLOAT32); - NDArray expectedZ('c', { 3, 3 }, { 0, 0, 0, 0, 0, 0, 0, 0, 0 }, nd4j::DataType::INT64); + NDArray expectedZ('c', { 3, 3 }, { 0., 0, 0, 0, 0, 0, 0, 0, 0 }, nd4j::DataType::INT64); auto result = op.evaluate({ &probsZ, &samples }, { }, { 1, INT64 }); auto outputZ = result->at(0); - + ASSERT_EQ(Status::OK(), result->status()); ASSERT_TRUE(expectedZ.isSameShape(outputZ)); ASSERT_TRUE(expectedZ.equalsTo(outputZ)); @@ -1031,7 +1031,7 @@ TEST_F(RNGTests, test_multinomial_1) { TEST_F(RNGTests, test_multinomial_2) { - NDArray samples('c', { 1 }, { 20 }, nd4j::DataType::INT32); + NDArray samples('c', { 1 }, std::vector{ 20 }, nd4j::DataType::INT32); NDArray probs('c', { 3, 5 }, { 0.2, 0.3, 0.5, 0.3, 0.5, 0.2, 0.5, 0.2, 0.3, 0.35, 0.25, 0.3, 0.25, 0.25, 0.5 }, nd4j::DataType::FLOAT32); NDArray expected('c', { 3, 20 }, { 0, 2, 0, 2, 0, 4, 2, 0, 1, 2, 0, 2, 3, 0, 0, 2, 4, 4, 1, 0, 2, 3, 2, 3, 0, 1, 3, 1, 1, 1, 2, 4, 3, 3, 1, 4, 4, 2, 0, 0, 3, 3, 3, 0, 0, 2, 2, 3, 3, 0, 0, 2, 3, 4, 2, 2, 3, 2, 1, 2 }, nd4j::DataType::INT64); NDArray output('c', { 3, 20 }, nd4j::DataType::INT64); @@ -1041,11 +1041,11 @@ TEST_F(RNGTests, test_multinomial_2) { ASSERT_EQ(Status::OK(), op.execute(rng, { &probs, &samples }, { &output }, {}, { 0, INT64 }, {}, {}, false)); ASSERT_TRUE(expected.isSameShape(output)); ASSERT_TRUE(expected.equalsTo(output)); - + NDArray probs2('c', { 5, 3 }, { 0.2, 0.3, 0.5, 0.3, 0.5, 0.2, 0.5, 0.2, 0.3, 0.35, 0.25, 0.3, 0.25, 0.25, 0.5 }, nd4j::DataType::FLOAT32); NDArray expected2('c', { 20, 3 }, { 0, 2, 3, 2, 3, 3, 0, 2, 3, 2, 3, 0, 0, 0, 0, 4, 1, 2, 2, 3, 2, 3, 1, 3, 1, 1, 3, 2, 1, 0, 0, 2, 0, 2, 4, 2, 3, 3, 3, 0, 3, 4, 0, 1, 2, 2, 0, 2, 4, 4, 0, 4, 2, 2, 1, 0, 1, 0, 0, 2 }, nd4j::DataType::INT64); NDArray output2('c', { 20, 3 }, nd4j::DataType::INT64); - + rng.setStates(1234, 1234); ASSERT_EQ(Status::OK(), op.execute(rng, { &probs2, &samples }, { &output2 }, {}, { 1, INT64 }, {}, {}, false)); ASSERT_TRUE(expected2.isSameShape(output2)); @@ -1053,16 +1053,17 @@ TEST_F(RNGTests, test_multinomial_2) { } TEST_F(RNGTests, test_multinomial_3) { - + NDArray probs('c', { 4, 3 }, { 0.3, 0.3, 0.4, 0.3, 0.4, 0.3, 0.3, 0.3, 0.4, 0.4, 0.3, 0.3 }, nd4j::DataType::FLOAT32); NDArray expected('c', { 4, 5 }, nd4j::DataType::INT64); NDArray output('c', { 4, 5 }, nd4j::DataType::INT64); - NDArray samples('c', { 1 }, { 5 }, nd4j::DataType::INT32); + NDArray samples('c', { 1 }, std::vector{ 5 }, nd4j::DataType::INT32); RandomGenerator rng(1234, 1234); nd4j::ops::random_multinomial op; + ASSERT_EQ(Status::OK(), op.execute(rng, { &probs, &samples }, { &expected }, {}, { 0, INT64 }, {}, {}, false)); - + rng.setStates(1234, 1234); ASSERT_EQ(Status::OK(), op.execute(rng, { &probs, &samples }, { &output }, {}, { 0, INT64 }, {}, {}, false)); ASSERT_TRUE(expected.isSameShape(output)); @@ -1074,7 +1075,7 @@ TEST_F(RNGTests, test_multinomial_4) { NDArray probs('c', { 3, 4 }, { 0.3, 0.3, 0.4, 0.3, 0.4, 0.3, 0.3, 0.3, 0.4, 0.4, 0.3, 0.3 }, nd4j::DataType::FLOAT32); NDArray expected('c', { 5, 4 }, nd4j::DataType::INT64); NDArray output('c', { 5, 4 }, nd4j::DataType::INT64); - NDArray samples('c', { 1 }, { 5 }, nd4j::DataType::INT32); + NDArray samples('c', { 1 }, std::vector{ 5 }, nd4j::DataType::INT32); RandomGenerator rng(1234, 1234); nd4j::ops::random_multinomial op; @@ -1092,15 +1093,15 @@ TEST_F(RNGTests, test_multinomial_5) { int ClassValue = 2; int Samples = 100000; - NDArray samples('c', { 1 }, { 1.*Samples }, nd4j::DataType::INT32); - + NDArray samples('c', { 1 }, std::vector{ 1.*Samples }, nd4j::DataType::INT32); + NDArray probs('c', { ClassValue, batchValue }, { 1.0, 1.0 }, nd4j::DataType::FLOAT32); - + nd4j::ops::random_multinomial op; NDArray output('c', { Samples, batchValue }, nd4j::DataType::INT64); RandomGenerator rng(1234, 1234); - + ASSERT_EQ(Status::OK(), op.execute(rng, { &probs, &samples }, { &output }, {}, { 1 }, {}, {}, false)); auto deviation = output.varianceNumber(variance::SummaryStatsStandardDeviation, false); @@ -1109,7 +1110,7 @@ TEST_F(RNGTests, test_multinomial_5) { // theoretical values for binomial ASSERT_NEAR(0.5, deviation.e(0), 4e-3); // 1000000 3e-3); ASSERT_NEAR(0.5, mean.e(0), 4e-3); // 1000000 3e-3); - + for (int i = 0; i < output.lengthOf(); i++) { auto value = output.e(i); ASSERT_TRUE(value >= 0 && value < ClassValue); @@ -1139,8 +1140,8 @@ TEST_F(RNGTests, test_multinomial_6) { int batchValue = 1; int ClassValue = 5; int Samples = 100000; - - NDArray samples('c', { 1 }, { 1. * Samples }, nd4j::DataType::INT32); + + NDArray samples('c', { 1 }, std::vector{ 1. * Samples }, nd4j::DataType::INT32); nd4j::ops::random_multinomial op; NDArray probExpect('c', { ClassValue }, { 0.058, 0.096, 0.1576, 0.2598, 0.4287 }, nd4j::DataType::DOUBLE); @@ -1152,8 +1153,8 @@ TEST_F(RNGTests, test_multinomial_6) { auto outputR = resultR->at(0); ASSERT_EQ(Status::OK(), resultR->status()); - NDArray countsR('c', { ClassValue }, { 0, 0, 0, 0, 0 }, nd4j::DataType::DOUBLE); - + NDArray countsR('c', { ClassValue }, { 0., 0, 0, 0, 0 }, nd4j::DataType::DOUBLE); + for (int i = 0; i < outputR->lengthOf(); i++) { auto value = outputR->e(i); ASSERT_TRUE(value >= 0 && value < ClassValue); @@ -1179,11 +1180,11 @@ TEST_F(RNGTests, test_multinomial_6) { RandomGenerator rng(1234, 1234); NDArray probs('c', { batchValue, ClassValue }, { 1., 1.5, 2., 2.5, 3. }, nd4j::DataType::FLOAT32); NDArray output('c', { batchValue, Samples }, nd4j::DataType::INT64); - + ASSERT_EQ(Status::OK(), op.execute(rng, { &probs, &samples }, { &output }, {}, { 0, INT64 }, {}, {}, false)); - NDArray counts('c', { ClassValue }, { 0, 0, 0, 0, 0 }, nd4j::DataType::DOUBLE); - + NDArray counts('c', { ClassValue }, { 0., 0, 0, 0, 0 }, nd4j::DataType::DOUBLE); + for (int i = 0; i < output.lengthOf(); i++) { auto value = output.e(i); ASSERT_TRUE(value >= 0 && value < ClassValue); diff --git a/libnd4j/tests_cpu/layers_tests/StringTests.cpp b/libnd4j/tests_cpu/layers_tests/StringTests.cpp index ec7821f21..8b9d92f2f 100644 --- a/libnd4j/tests_cpu/layers_tests/StringTests.cpp +++ b/libnd4j/tests_cpu/layers_tests/StringTests.cpp @@ -1,5 +1,6 @@ -/******************************************************************************* +/******************************************************************************* * Copyright (c) 2015-2018 Skymind, Inc. + * Copyright (c) 2019-2020 Konduit K.K. * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -16,6 +17,7 @@ // // @author raver119@gmail.com +// @author Oleg Semeniv // @@ -30,7 +32,7 @@ class StringTests : public testing::Test { public: }; - +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, Basic_Test_1) { std::string f("alpha"); auto array = NDArrayFactory::string(f); @@ -43,7 +45,7 @@ TEST_F(StringTests, Basic_Test_1) { ASSERT_EQ(f, z); } - +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, Basic_Test_2) { std::string f("alpha"); auto array = NDArrayFactory::string(f.c_str()); @@ -56,23 +58,213 @@ TEST_F(StringTests, Basic_Test_2) { ASSERT_EQ(f, z); } - +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, Basic_Test_3) { - auto array = NDArrayFactory::string('c', {3, 2}, {"alpha", "beta", "gamma", "phi", "theta", "omega"}); + + auto array = NDArrayFactory::string({3, 2}, {"alpha", "beta", "gamma", "phi", "theta", "omega"}); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_4) { + + NDArray array( { 3, 2 }, std::vector{ U"alpha", U"beta", U"gamma€한", U"pÿqwe", U"ß水𝄋", U"omega" }); + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_5) { + + NDArray array( { 3, 2 }, std::vector{ u"alpha", u"beta", u"gamma€한", u"pÿqwe", u"ß水𝄋", u"omega" }); + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_6) { + + NDArray array( { 3, 2 }, std::vector{ "alpha", "beta", "gamma€한", "pÿqwe", "ß水𝄋", "omega" }); + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_7) { + + NDArray array( { 3, 2 }, std::vector{ U"alpha", U"beta", U"gamma€한", U"pÿqwe", U"ß水𝄋", U"omega" }); + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_8) { + + NDArray array( { 3, 2 }, std::vector{ u"alpha", u"beta", u"gamma€한", u"pÿqwe", u"ß水𝄋", u"omega" }); + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_9) { + + NDArray array( { 3, 2 }, std::vector{ "alpha", "beta", "gamma€한", "pÿqwe", "ß水𝄋", "omega" }); + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_10) { + + NDArray array(std::u32string(U"gamma€한")); + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_11) { + + NDArray array(U"gamma€한"); + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_12) { + + NDArray array(std::u16string(u"gamma€한")); + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_13) { + + NDArray array(u"gamma€한"); + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_14) { + + NDArray array(std::string("gamma€한")); + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_15) { + + NDArray array("gamma€한"); + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_16) { + + auto array = NDArrayFactory::string( { 3, 2 }, std::vector{ "alpha", "beta", "gamma", "phi", "theta", "omega" }); ASSERT_EQ(6, array.lengthOf()); ASSERT_EQ(2, array.rankOf()); array.printIndexedBuffer("String array"); } +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_17) { + auto array = NDArrayFactory::string({ 3, 2 }, std::vector{ "alpha", "beta", "gamma", "phi", "theta", "omega" }); + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_18) { + + auto array = NDArrayFactory::string({ 3, 2 }, std::vector{ u"alpha", u"beta", u"gamma", u"phi", u"theta", u"omega" }); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_19) { + + auto array = NDArrayFactory::string( { 3, 2 }, std::vector{ u"alpha", u"beta", u"gamma", u"phi", u"theta", u"omega" }); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_20) { + + auto array = NDArrayFactory::string( { 3, 2 }, std::vector{ U"alpha", U"beta", U"gamma", U"phi", U"theta", U"omega" }); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_21) { + + auto array = NDArrayFactory::string( { 3, 2 }, std::vector{ U"alpha", U"òèçùà12345¤z", U"ß水𝄋ÿ€한𐍈®кею90ощъ]ї", U"phi", U"theta", U"omega" }); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_22) { + std::u16string f(u"ß水𝄋ÿ€한𐍈®кею90ощъ]ї"); + auto array = NDArrayFactory::string(f.c_str()); + ASSERT_EQ(nd4j::DataType::UTF16, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto z = array.e(0); + + ASSERT_EQ(f, z); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_23) { + std::u32string f(U"ß水𝄋ÿ€한𐍈®кею90ощъ]ї"); + auto array = NDArrayFactory::string(f.c_str()); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto z = array.e(0); + + ASSERT_EQ(f, z); +} +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, Export_Test_1) { - auto array = NDArrayFactory::string('c', {3}, {"alpha", "beta", "gamma"}); - + auto array = NDArrayFactory::string( {3}, {"alpha", "beta", "gamma"}); auto vector = array.asByteVector(); } - +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, Basic_dup_1) { std::string f("alpha"); auto array = NDArrayFactory::string(f); @@ -91,20 +283,20 @@ TEST_F(StringTests, Basic_dup_1) { delete dup; } - +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, byte_length_test_1) { std::string f("alpha"); auto array = NDArrayFactory::string(f); ASSERT_EQ(f.length(), StringUtils::byteLength(array)); } - +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, byte_length_test_2) { - auto array = NDArrayFactory::string('c', {2}, {"alpha", "beta"}); + auto array = NDArrayFactory::string( {2}, {"alpha", "beta"}); ASSERT_EQ(9, StringUtils::byteLength(array)); } - +///////////////////////////////////////////////////////////////////////// TEST_F(StringTests, test_split_1) { auto split = StringUtils::split("alpha beta gamma", " "); @@ -112,4 +304,562 @@ TEST_F(StringTests, test_split_1) { ASSERT_EQ(std::string("alpha"), split[0]); ASSERT_EQ(std::string("beta"), split[1]); ASSERT_EQ(std::string("gamma"), split[2]); -} \ No newline at end of file +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, test_unicode_utf8_utf16) { + + std::string utf8 = u8"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + std::u16string utf16Exp = u"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + + std::u16string utf16Res; + ASSERT_TRUE(StringUtils::u8StringToU16String(utf8, utf16Res)); + + ASSERT_EQ(utf16Res.size(), utf16Exp.size()); + for (auto i = 0; i < utf16Exp.size(); i++) { + ASSERT_EQ(utf16Exp[i], utf16Res[i]); + } +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, test_unicode_utf8_utf32) { + + std::string utf8 = u8"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + std::u32string utf32Exp = U"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + + std::u32string utf32Res; + ASSERT_TRUE(StringUtils::u8StringToU32String(utf8, utf32Res)); + + ASSERT_EQ(utf32Res.size(), utf32Exp.size()); + for (auto i = 0; i < utf32Exp.size(); i++) { + ASSERT_EQ(utf32Exp[i], utf32Res[i]); + } +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, test_unicode_utf16_utf8) { + + std::string utf8Exp = u8"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + std::u16string utf16 = u"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + + std::string utf8Res; + ASSERT_TRUE(StringUtils::u16StringToU8String(utf16, utf8Res)); + + ASSERT_EQ(utf8Res.size(), utf8Exp.size()); + for (auto i = 0; i < utf8Exp.size(); i++) { + ASSERT_EQ(utf8Exp[i], utf8Res[i]); + } +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, test_unicode_utf32_utf8) { + + std::string utf8Exp = u8"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею 90ощъ]їїщkk1q\n\t\rop~"; + std::u32string utf32 = U"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею 90ощъ]їїщkk1q\n\t\rop~"; + + std::string utf8Res; + ASSERT_TRUE(StringUtils::u32StringToU8String(utf32, utf8Res)); + + ASSERT_EQ(utf8Res.size(), utf8Exp.size()); + for (auto i = 0; i < utf8Exp.size(); i++) { + ASSERT_EQ(utf8Exp[i], utf8Res[i]); + } +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, test_unicode_utf16_utf32) { + + std::u32string utf32Exp = U"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + std::u16string utf16 = u"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + + std::u32string utf32Res; + ASSERT_TRUE(StringUtils::u16StringToU32String(utf16, utf32Res)); + + ASSERT_EQ(utf32Res.size(), utf32Exp.size()); + for (auto i = 0; i < utf32Exp.size(); i++) { + ASSERT_EQ(utf32Exp[i], utf32Res[i]); + } +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, test_unicode_utf32_utf16) { + + std::u16string utf16Exp = u"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + std::u32string utf32 = U"\nòèçùà12345¤zß水𝄋ÿ€한𐍈®кею90ощъ]їїщkk1q\n\t\rop~"; + + std::u16string utf16Res; + ASSERT_TRUE(StringUtils::u32StringToU16String(utf32, utf16Res)); + + ASSERT_EQ(utf16Res.size(), utf16Exp.size()); + for (auto i = 0; i < utf16Exp.size(); i++) { + ASSERT_EQ(utf16Exp[i], utf16Res[i]); + } +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, byte_length_test_Default) { + + std::string f("alpha"); + auto array = NDArrayFactory::string(f); + + ASSERT_EQ(f.length(), StringUtils::byteLength(array)); + + std::u16string f16(u"alpha"); + auto array16 = NDArrayFactory::string(f16); + + ASSERT_EQ(sizeof(char16_t)*f16.length(), StringUtils::byteLength(array16)); + + std::u32string f32(U"alpha"); + auto array32 = NDArrayFactory::string(f32); + + ASSERT_EQ(sizeof(char32_t) * f32.length(), StringUtils::byteLength(array32)); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, byte_length_test_UTF16) { + std::string f(u8"alpha"); + auto array = NDArrayFactory::string(f, nd4j::DataType::UTF16); + + ASSERT_EQ(sizeof(char16_t) * f.length(), StringUtils::byteLength(array)); + + std::u16string f16(u"alpha"); + auto array16 = NDArrayFactory::string(f16, nd4j::DataType::UTF16); + + ASSERT_EQ(sizeof(char16_t) * f16.length(), StringUtils::byteLength(array16)); + + std::u32string f32(U"alpha"); + auto array32 = NDArrayFactory::string(f32, nd4j::DataType::UTF16); + + ASSERT_EQ(sizeof(char16_t) * f32.length(), StringUtils::byteLength(array32)); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_UTF16toU8) { + + std::u16string f16(u"alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f16, nd4j::DataType::UTF8); + ASSERT_EQ(nd4j::DataType::UTF8, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto z = array.e(0); + + std::string f(u8"alpha水𝄋ÿ€한𐍈®кею"); + ASSERT_EQ(f, z); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_UTF32toU8) { + std::u32string f32(U"alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f32.c_str(), nd4j::DataType::UTF8); + ASSERT_EQ(nd4j::DataType::UTF8, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto z = array.e(0); + std::string f(u8"alpha水𝄋ÿ€한𐍈®кею"); + ASSERT_EQ(f, z); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_UTF16toU16) { + + std::u16string f16(u"€alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f16, nd4j::DataType::UTF16); + ASSERT_EQ(nd4j::DataType::UTF16, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + auto z = array.e(0); + + ASSERT_EQ(z, f16); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_UTF32toU16) { + + std::u32string f32(U"€alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f32, nd4j::DataType::UTF16); + ASSERT_EQ(nd4j::DataType::UTF16, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + auto z = array.e(0); + std::u16string f16(u"€alpha水𝄋ÿ€한𐍈®кею"); + ASSERT_EQ(z, f16); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_UTF16toU32) { + + std::u16string f16(u"€alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f16, nd4j::DataType::UTF32); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto z = array.e(0); + std::u32string fres(U"€alpha水𝄋ÿ€한𐍈®кею"); + ASSERT_EQ(z, fres); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_UTF32toU32) { + + std::u32string f32(U"€alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f32); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + auto z = array.e(0); + ASSERT_EQ(f32, z); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_UTF8toU32) { + + std::string f(u8"€alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f, nd4j::DataType::UTF32); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + std::u32string f32(U"€alpha水𝄋ÿ€한𐍈®кею"); + auto z = array.e(0); + ASSERT_EQ(f32, z); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU8toUTF16) { + auto array = NDArrayFactory::string({ 3, 2 }, { "alpha€", "beta", "gamma水", "phi", "theta", "omega水" }, nd4j::DataType::UTF16); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU8toUTF32) { + auto array = NDArrayFactory::string( { 3, 2 }, { "alpha€", "beta水", "gamma", "phi", "theta", "omega" }, nd4j::DataType::UTF32); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U8toUTF16) { + auto array = NDArrayFactory::string({ 3 }, { "alpha", "beta", "gamma" }, nd4j::DataType::UTF16); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U8toUTF32) { + auto array = NDArrayFactory::string({ 3 }, { "alpha", "beta", "gamma" }, nd4j::DataType::UTF32); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU16toUTF16) { + auto array = NDArrayFactory::string({ 3, 2 }, { u"alpha水", u"beta", u"gamma", u"phi", u"theta水", u"omega" }, nd4j::DataType::UTF16); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU16toUTF32) { + auto array = NDArrayFactory::string( { 3, 2 }, { u"alpha水", u"beta", u"gamma水", u"phi", u"theta", u"omega" }, nd4j::DataType::UTF32); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU16toUTF8) { + auto array = NDArrayFactory::string( { 3, 2 }, { u"alpha€", u"beta水", u"gamma", u"phi水", u"theta", u"omega" }, nd4j::DataType::UTF8); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U16toUTF8) { + auto array = NDArrayFactory::string( { 3 }, { u"alpha", u"beta", u"gamma" }, nd4j::DataType::UTF8); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U16toUTF16) { + auto array = NDArrayFactory::string( { 3 }, { u"alpha", u"beta", u"gamma" }, nd4j::DataType::UTF16); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U16toUTF32) { + auto array = NDArrayFactory::string( { 3 }, { u"alpha水", u"beta", u"gamma水" }, nd4j::DataType::UTF32); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU32toUTF32) { + auto array = NDArrayFactory::string( { 3, 2 }, { U"alpha€", U"beta水", U"gamma", U"phi", U"theta", U"omega水" }, nd4j::DataType::UTF32); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU32toUTF16) { + auto array = NDArrayFactory::string({ 3, 2 }, { U"alpha水", U"水beta", U"gamma", U"phi水", U"theta", U"omega" }, nd4j::DataType::UTF16); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); + + printf("Array elements size: \n"); + for (int e = 0; e < array.lengthOf(); e++) { + printf("Element %d size: %d\n", e, static_cast(array.e(e).size())); + } +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_Test_StringVecU32toUTF8) { + auto array = NDArrayFactory::string( { 3, 2 }, { U"alpha水", U"beta", U"gamma水", U"phi", U"theta", U"omega" }, nd4j::DataType::UTF8); + + ASSERT_EQ(6, array.lengthOf()); + ASSERT_EQ(2, array.rankOf()); + + array.printIndexedBuffer("String array"); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U32toUTF32) { + auto array = NDArrayFactory::string( { 3 }, { U"alpha", U"beta", U"gamma" }, nd4j::DataType::UTF32); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U32toUTF16) { + auto array = NDArrayFactory::string( { 3 }, { U"alpha", U"beta水", U"gamma水" }, nd4j::DataType::UTF16); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Export_Test_U32toUTF8) { + auto array = NDArrayFactory::string( { 3 }, { U"alpha", U"beta", U"gamma水" }, nd4j::DataType::UTF8); + + auto vector = array.asByteVector(); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_dup_UTF16) { + std::u16string f(u"€alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f); + ASSERT_EQ(nd4j::DataType::UTF16, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto dup = new NDArray(array.dup()); + + auto z0 = array.e(0); + auto z1 = dup->e(0); + + ASSERT_EQ(f, z0); + ASSERT_EQ(f, z1); + + delete dup; +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_dup_UTF32) { + std::u32string f(U"€alpha水𝄋ÿ€한𐍈®кею"); + auto array = NDArrayFactory::string(f); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto dup = new NDArray(array.dup()); + + auto z0 = array.e(0); + auto z1 = dup->e(0); + + ASSERT_EQ(f, z0); + ASSERT_EQ(f, z1); + + delete dup; +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF32toUTF8) { + + std::u32string u32(U"€alpha水𝄋ÿ€한𐍈®кею"); + + std::string u8(u8"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u32); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF8); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u32, z0); + ASSERT_EQ(u8, z1); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF32toUTF16) { + + std::u32string u32(U"€alpha水𝄋ÿ€한𐍈®кею"); + + std::u16string u16(u"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u32); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF16); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u32, z0); + ASSERT_EQ(u16, z1); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF32toUTF32) { + + std::u32string u32(U"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u32); + ASSERT_EQ(nd4j::DataType::UTF32, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF32); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u32, z0); + ASSERT_EQ(u32, z1); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF16toUTF16) { + + std::u16string u16(u"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u16); + ASSERT_EQ(nd4j::DataType::UTF16, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF16); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u16, z0); + ASSERT_EQ(u16, z1); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF16toUTF32) { + + std::u32string u32(U"€alpha水𝄋ÿ€한𐍈®кею"); + + std::u16string u16(u"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u16); + ASSERT_EQ(nd4j::DataType::UTF16, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF32); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u32, z1); + ASSERT_EQ(u16, z0); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF16toUTF8) { + + std::string u8(u8"€alpha水𝄋ÿ€한𐍈®кею"); + + std::u16string u16(u"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u16); + ASSERT_EQ(nd4j::DataType::UTF16, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF8); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u8, z1); + ASSERT_EQ(u16, z0); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF8toUTF8) { + + std::string u8("€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u8); + ASSERT_EQ(nd4j::DataType::UTF8, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF8); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u8, z1); + ASSERT_EQ(u8, z0); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF8toUTF16) { + + std::string u8(u8"€alpha水𝄋ÿ€한𐍈®кею"); + + std::u16string u16(u"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u8); + ASSERT_EQ(nd4j::DataType::UTF8, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF16); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u8, z0); + ASSERT_EQ(u16, z1); +} +///////////////////////////////////////////////////////////////////////// +TEST_F(StringTests, Basic_cast_UTF8toUTF32) { + + std::string u8(u8"€alpha水𝄋ÿ€한𐍈®кею"); + + std::u32string u32(U"€alpha水𝄋ÿ€한𐍈®кею"); + + auto array = NDArrayFactory::string(u8); + ASSERT_EQ(nd4j::DataType::UTF8, array.dataType()); + + ASSERT_EQ(1, array.lengthOf()); + ASSERT_EQ(0, array.rankOf()); + + auto aCast = array.cast(nd4j::DataType::UTF32); + + auto z0 = array.e(0); + auto z1 = aCast.e(0); + + ASSERT_EQ(u8, z0); + ASSERT_EQ(u32, z1); +}