Support python 3.7, cherry-pick of: https://github.com/tensorflow/tensorflow/commit/abb903df7a5998b33547c02e95f9fa47c00f31f4 https://github.com/tensorflow/tensorflow/commit/2b0805301e4531dd7c2ed677d932f6408675460e https://github.com/tensorflow/tensorflow/pull/21202 https://github.com/tensorflow/tensorflow/pull/23453 --- diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 81221c4078..50873a07d8 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -46,6 +46,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/refcount.h" +#include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/gtl/flatmap.h" #include "tensorflow/core/lib/gtl/map_util.h" @@ -200,8 +201,8 @@ void TFE_ContextOptionsSetConfig(TFE_ContextOptions* options, const void* proto, } void TFE_ContextOptionsSetAsync(TFE_ContextOptions* options, - unsigned char async) { - options->async = async; + unsigned char enable) { + options->async = enable; } void TFE_ContextOptionsSetDevicePlacementPolicy( TFE_ContextOptions* options, TFE_ContextDevicePlacementPolicy policy) { @@ -218,9 +219,9 @@ TF_CAPI_EXPORT extern void TFE_ContextOptionsSetServerDef( } TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx, - unsigned char async, + unsigned char enable, TF_Status* status) { - status->status = ctx->context.SetAsyncForThread(async); + status->status = ctx->context.SetAsyncForThread(enable); } void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; } @@ -421,8 +422,11 @@ TF_AttrType TFE_OpNameGetAttrType(TFE_Context* ctx, return ret; } -void TFE_OpSetAttrString(TFE_Op* op, const char* attr_name, const char* value) { - op->operation.MutableAttrs()->Set(attr_name, value); +void TFE_OpSetAttrString(TFE_Op* op, const char* attr_name, const void* value, + size_t length) { + op->operation.MutableAttrs()->Set( + attr_name, + tensorflow::StringPiece(static_cast(value), length)); } void TFE_OpSetAttrInt(TFE_Op* op, const char* attr_name, int64_t value) { @@ -473,16 +477,22 @@ void TFE_OpSetAttrFunction(TFE_Op* op, const char* attr_name, op->operation.MutableAttrs()->Set(attr_name, attr_value); } -#define TFE_OP_SET_ATTR_LIST(fn, type) \ - void fn(TFE_Op* op, const char* attr_name, const type* values, \ - int num_values) { \ - op->operation.MutableAttrs()->Set( \ - attr_name, \ - tensorflow::gtl::ArraySlice(values, num_values)); \ +void TFE_OpSetAttrStringList(TFE_Op* op, const char* attr_name, + const void* const* values, const size_t* lengths, + int num_values) { + std::vector v(num_values); + for (int i = 0; i < num_values; ++i) { + v[i] = tensorflow::StringPiece(static_cast(values[i]), + lengths[i]); } -TFE_OP_SET_ATTR_LIST(TFE_OpSetAttrStringList, char*) -TFE_OP_SET_ATTR_LIST(TFE_OpSetAttrFloatList, float) -#undef TFE_OP_SET_ATTR_LIST + op->operation.MutableAttrs()->Set(attr_name, v); +} + +void TFE_OpSetAttrFloatList(TFE_Op* op, const char* attr_name, + const float* values, int num_values) { + op->operation.MutableAttrs()->Set( + attr_name, tensorflow::gtl::ArraySlice(values, num_values)); +} void TFE_OpSetAttrIntList(TFE_Op* op, const char* attr_name, const int64_t* values, int num_values) { @@ -655,9 +665,11 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op, const tensorflow::AttrValue& default_value, const char* attr_name, TF_Status* status) { switch (default_value.value_case()) { - case tensorflow::AttrValue::kS: - TFE_OpSetAttrString(op, attr_name, default_value.s().data()); + case tensorflow::AttrValue::kS: { + const string& v = default_value.s(); + TFE_OpSetAttrString(op, attr_name, v.data(), v.size()); break; + } case tensorflow::AttrValue::kI: TFE_OpSetAttrInt(op, attr_name, static_cast(default_value.i())); break; diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h index 1862af3ce2..8c6bc63c9b 100644 --- a/tensorflow/c/eager/c_api.h +++ b/tensorflow/c/eager/c_api.h @@ -76,7 +76,7 @@ typedef enum TFE_ContextDevicePlacementPolicy { // Sets the default execution mode (sync/async). Note that this can be // overridden per thread using TFE_ContextSetAsyncForThread. TF_CAPI_EXPORT extern void TFE_ContextOptionsSetAsync(TFE_ContextOptions*, - unsigned char async); + unsigned char enable); TF_CAPI_EXPORT extern void TFE_ContextOptionsSetDevicePlacementPolicy( TFE_ContextOptions*, TFE_ContextDevicePlacementPolicy); @@ -125,7 +125,7 @@ TFE_ContextGetDevicePlacementPolicy(TFE_Context*); // Overrides the execution mode (sync/async) for the current thread. TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context*, - unsigned char async, + unsigned char enable, TF_Status* status); // Causes the calling thread to block till all ops dispatched in async mode @@ -278,7 +278,8 @@ TF_CAPI_EXPORT extern TF_AttrType TFE_OpNameGetAttrType( TF_CAPI_EXPORT extern void TFE_OpSetAttrString(TFE_Op* op, const char* attr_name, - const char* value); + const void* value, + size_t length); TF_CAPI_EXPORT extern void TFE_OpSetAttrInt(TFE_Op* op, const char* attr_name, int64_t value); TF_CAPI_EXPORT extern void TFE_OpSetAttrFloat(TFE_Op* op, const char* attr_name, @@ -305,7 +306,8 @@ TF_CAPI_EXPORT extern void TFE_OpSetAttrFunction(TFE_Op* op, TF_CAPI_EXPORT extern void TFE_OpSetAttrStringList(TFE_Op* op, const char* attr_name, - const char** value, + const void* const* values, + const size_t* lengths, int num_values); TF_CAPI_EXPORT extern void TFE_OpSetAttrIntList(TFE_Op* op, const char* attr_name, diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc index 27ff5f7211..b796246747 100644 --- a/tensorflow/c/eager/c_api_test.cc +++ b/tensorflow/c/eager/c_api_test.cc @@ -1083,8 +1083,8 @@ TFE_TensorHandle* CreateVariable(TFE_Context* ctx, float value, if (TF_GetCode(status) != TF_OK) return nullptr; TFE_OpSetAttrType(op, "dtype", TF_FLOAT); TFE_OpSetAttrShape(op, "shape", {}, 0, status); - TFE_OpSetAttrString(op, "container", ""); - TFE_OpSetAttrString(op, "shared_name", ""); + TFE_OpSetAttrString(op, "container", "", 0); + TFE_OpSetAttrString(op, "shared_name", "", 0); if (TF_GetCode(status) != TF_OK) return nullptr; TFE_TensorHandle* var_handle = nullptr; int num_retvals = 1; diff --git a/tensorflow/python/eager/pywrap_tfe.h b/tensorflow/python/eager/pywrap_tfe.h index a916a75f00..823c4078b8 100644 --- a/tensorflow/python/eager/pywrap_tfe.h +++ b/tensorflow/python/eager/pywrap_tfe.h @@ -89,7 +89,7 @@ int MaybeRaiseExceptionFromStatus(const tensorflow::Status& status, PyObject* exception); // Returns the string associated with the passed-in python object. -char* TFE_GetPythonString(PyObject* o); +const char* TFE_GetPythonString(PyObject* o); // Returns a unique id on each call. int64_t get_uid(); diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 6c9481c3af..9cf428da99 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -205,14 +205,20 @@ bool ParseDimensionValue(const string& key, PyObject* py_value, } bool ParseStringValue(const string& key, PyObject* py_value, TF_Status* status, - const char** value) { + tensorflow::StringPiece* value) { if (PyBytes_Check(py_value)) { - *value = PyBytes_AsString(py_value); + Py_ssize_t size = 0; + char* buf = nullptr; + if (PyBytes_AsStringAndSize(py_value, &buf, &size) < 0) return false; + *value = tensorflow::StringPiece(buf, size); return true; } #if PY_MAJOR_VERSION >= 3 if (PyUnicode_Check(py_value)) { - *value = PyUnicode_AsUTF8(py_value); + Py_ssize_t size = 0; + const char* buf = PyUnicode_AsUTF8AndSize(py_value, &size); + if (buf == nullptr) return false; + *value = tensorflow::StringPiece(buf, size); return true; } #endif @@ -275,8 +281,16 @@ bool SetOpAttrList( } if (type == TF_ATTR_STRING) { - PARSE_LIST(const char*, ParseStringValue); - TFE_OpSetAttrStringList(op, key, values.get(), num_values); + std::unique_ptr values(new const void*[num_values]); + std::unique_ptr lengths(new size_t[num_values]); + for (int i = 0; i < num_values; ++i) { + tensorflow::StringPiece value; + tensorflow::Safe_PyObjectPtr py_value(PySequence_ITEM(py_list, i)); + if (!ParseStringValue(key, py_value.get(), status, &value)) return false; + values[i] = value.data(); + lengths[i] = value.size(); + } + TFE_OpSetAttrStringList(op, key, values.get(), lengths.get(), num_values); } else if (type == TF_ATTR_INT) { PARSE_LIST(int64_t, ParseInt64Value); TFE_OpSetAttrIntList(op, key, values.get(), num_values); @@ -379,12 +393,15 @@ void SetOpAttrListDefault( TF_Status* status) { if (type == TF_ATTR_STRING) { int num_values = attr.default_value().list().s_size(); - std::unique_ptr values(new const char*[num_values]); + std::unique_ptr values(new const void*[num_values]); + std::unique_ptr lengths(new size_t[num_values]); (*attr_list_sizes)[key] = num_values; for (int i = 0; i < num_values; i++) { - values[i] = attr.default_value().list().s(i).data(); + const string& v = attr.default_value().list().s(i); + values[i] = v.data(); + lengths[i] = v.size(); } - TFE_OpSetAttrStringList(op, key, values.get(), num_values); + TFE_OpSetAttrStringList(op, key, values.get(), lengths.get(), num_values); } else if (type == TF_ATTR_INT) { int num_values = attr.default_value().list().i_size(); std::unique_ptr values(new int64_t[num_values]); @@ -470,9 +487,9 @@ bool SetOpAttrScalar( tensorflow::gtl::FlatMap* attr_list_sizes, TF_Status* status) { if (type == TF_ATTR_STRING) { - const char* value; + tensorflow::StringPiece value; if (!ParseStringValue(key, py_value, status, &value)) return false; - TFE_OpSetAttrString(op, key, value); + TFE_OpSetAttrString(op, key, value.data(), value.size()); } else if (type == TF_ATTR_INT) { int64_t value; if (!ParseInt64Value(key, py_value, status, &value)) return false; @@ -533,7 +550,7 @@ bool SetOpAttrScalar( // (which is what the various "defun" or "Defun" decorators do). // And in the future also allow an object that can encapsulate // the function name and its attribute values. - const char* func_name = nullptr; + tensorflow::StringPiece func_name; if (!ParseStringValue(key, py_value, status, &func_name)) { PyObject* name_attr = PyObject_GetAttrString(py_value, "name"); if (name_attr == nullptr || @@ -549,7 +566,8 @@ bool SetOpAttrScalar( return false; } } - TFE_Op* func = TFE_NewOp(ctx, func_name, status); + TFE_Op* func = TFE_NewOp( + ctx, string(func_name.data(), func_name.size()).c_str(), status); if (TF_GetCode(status) != TF_OK) return false; TFE_OpSetAttrFunction(op, key, func); TFE_DeleteOp(func); @@ -807,7 +825,7 @@ int MaybeRaiseExceptionFromStatus(const tensorflow::Status& status, return -1; } -char* TFE_GetPythonString(PyObject* o) { +const char* TFE_GetPythonString(PyObject* o) { if (PyBytes_Check(o)) { return PyBytes_AsString(o); } diff --git a/tensorflow/python/lib/core/ndarray_tensor.cc b/tensorflow/python/lib/core/ndarray_tensor.cc index 9df38d464c..f87674a524 100644 --- a/tensorflow/python/lib/core/ndarray_tensor.cc +++ b/tensorflow/python/lib/core/ndarray_tensor.cc @@ -136,6 +136,31 @@ Status PyArray_TYPE_to_TF_DataType(PyArrayObject* array, return Status::OK(); } +Status PyObjectToString(PyObject* obj, const char** ptr, Py_ssize_t* len) { + if (!PyUnicode_Check(obj)) { + char* buf; + if (PyBytes_AsStringAndSize(obj, &buf, len) != 0) { + return errors::Internal("Unable to get element as bytes."); + } + *ptr = buf; + return Status::OK(); + } +#if (PY_MAJOR_VERSION > 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 3)) + *ptr = PyUnicode_AsUTF8AndSize(obj, len); + if (*ptr != nullptr) return Status::OK(); +#else + PyObject* utemp = PyUnicode_AsUTF8String(obj); + char* buf; + if (utemp != nullptr && PyBytes_AsStringAndSize(utemp, &buf, len) != -1) { + *ptr = buf; + Py_DECREF(utemp); + return Status::OK(); + } + Py_XDECREF(utemp); +#endif + return errors::Internal("Unable to convert element to UTF-8."); +} + // Iterate over the string array 'array', extract the ptr and len of each string // element and call f(ptr, len). template @@ -148,33 +173,10 @@ Status PyBytesArrayMap(PyArrayObject* array, F f) { if (!item) { return errors::Internal("Unable to get element from the feed - no item."); } - char* ptr; Py_ssize_t len; - - if (PyUnicode_Check(item.get())) { -#if PY_VERSION_HEX >= 0x03030000 - // Accept unicode by converting to UTF-8 bytes. - ptr = PyUnicode_AsUTF8AndSize(item.get(), &len); - if (!ptr) { - return errors::Internal("Unable to get element as UTF-8."); - } - f(ptr, len); -#else - PyObject* utemp = PyUnicode_AsUTF8String(item.get()); - if (!utemp || PyBytes_AsStringAndSize(utemp, &ptr, &len) == -1) { - Py_XDECREF(utemp); - return errors::Internal("Unable to convert element to UTF-8."); - } - f(ptr, len); - Py_DECREF(utemp); -#endif - } else { - int success = PyBytes_AsStringAndSize(item.get(), &ptr, &len); - if (success != 0) { - return errors::Internal("Unable to get element as bytes."); - } - f(ptr, len); - } + const char* ptr; + TF_RETURN_IF_ERROR(PyObjectToString(item.get(), &ptr, &len)); + f(ptr, len); PyArray_ITER_NEXT(iter.get()); } return Status::OK(); @@ -186,10 +188,11 @@ Status EncodePyBytesArray(PyArrayObject* array, tensorflow::int64 nelems, size_t* size, void** buffer) { // Compute bytes needed for encoding. *size = 0; - TF_RETURN_IF_ERROR(PyBytesArrayMap(array, [&size](char* ptr, Py_ssize_t len) { - *size += - sizeof(tensorflow::uint64) + tensorflow::core::VarintLength(len) + len; - })); + TF_RETURN_IF_ERROR( + PyBytesArrayMap(array, [&size](const char* ptr, Py_ssize_t len) { + *size += sizeof(tensorflow::uint64) + + tensorflow::core::VarintLength(len) + len; + })); // Encode all strings. std::unique_ptr base_ptr(new char[*size]); char* base = base_ptr.get(); @@ -198,7 +201,7 @@ Status EncodePyBytesArray(PyArrayObject* array, tensorflow::int64 nelems, tensorflow::uint64* offsets = reinterpret_cast(base); TF_RETURN_IF_ERROR(PyBytesArrayMap( - array, [&base, &data_start, &dst, &offsets](char* ptr, Py_ssize_t len) { + array, [&data_start, &dst, &offsets](const char* ptr, Py_ssize_t len) { *offsets = (dst - data_start); offsets++; dst = tensorflow::core::EncodeVarint64(dst, len); diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc index a13fdbb57c..a7f8065fb3 100644 --- a/tensorflow/python/lib/core/py_func.cc +++ b/tensorflow/python/lib/core/py_func.cc @@ -303,6 +303,35 @@ class NumpyTensorBuffer : public TensorBuffer { void* data_; }; +Status PyObjectToString(PyObject* obj, string* str) { + char* py_bytes; + Py_ssize_t size; + if (PyBytes_AsStringAndSize(obj, &py_bytes, &size) != -1) { + str->assign(py_bytes, size); + return Status::OK(); + } +#if PY_MAJOR_VERSION >= 3 + const char* ptr = PyUnicode_AsUTF8AndSize(obj, &size); + if (ptr != nullptr) { + str->assign(ptr, size); + return Status::OK(); + } +#else + if (PyUnicode_Check(obj)) { + PyObject* unicode = PyUnicode_AsUTF8String(obj); + char* ptr; + if (unicode && PyString_AsStringAndSize(unicode, &ptr, &size) != -1) { + str->assign(ptr, size); + Py_DECREF(unicode); + return Status::OK(); + } + Py_XDECREF(unicode); + } +#endif + return errors::Unimplemented("Unsupported object type ", + obj->ob_type->tp_name); +} + Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) { PyArrayObject* input = reinterpret_cast(obj); DataType dtype = DT_INVALID; @@ -318,29 +347,7 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) { auto tflat = t.flat(); PyObject** input_data = reinterpret_cast(PyArray_DATA(input)); for (int i = 0; i < tflat.dimension(0); ++i) { - char* el; - Py_ssize_t el_size; - if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) { -#if PY_MAJOR_VERSION >= 3 - el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size); -#else - el = nullptr; - if (PyUnicode_Check(input_data[i])) { - PyObject* unicode = PyUnicode_AsUTF8String(input_data[i]); - if (unicode) { - if (PyString_AsStringAndSize(unicode, &el, &el_size) == -1) { - Py_DECREF(unicode); - el = nullptr; - } - } - } -#endif - if (!el) { - return errors::Unimplemented("Unsupported object type ", - input_data[i]->ob_type->tp_name); - } - } - tflat(i) = string(el, el_size); + TF_RETURN_IF_ERROR(PyObjectToString(input_data[i], &tflat(i))); } *ret = t; break; diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 500dc30cc3..eb627d3ecc 100644 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -102,20 +102,29 @@ limitations under the License. } } +// For const parameters in a function, SWIG pretty much ignores the const. +// See: http://www.swig.org/Doc2.0/SWIG.html#SWIG_nn13 +// Hence the 'const_cast'. %typemap(in) const char* serialized_function_def { - $1 = TFE_GetPythonString($input); + $1 = const_cast(TFE_GetPythonString($input)); } +// For const parameters in a function, SWIG pretty much ignores the const. +// See: http://www.swig.org/Doc2.0/SWIG.html#SWIG_nn13 +// Hence the 'const_cast'. %typemap(in) const char* device_name { if ($input == Py_None) { $1 = nullptr; } else { - $1 = TFE_GetPythonString($input); + $1 = const_cast(TFE_GetPythonString($input)); } } +// For const parameters in a function, SWIG pretty much ignores the const. +// See: http://www.swig.org/Doc2.0/SWIG.html#SWIG_nn13 +// Hence the 'const_cast'. %typemap(in) const char* op_name { - $1 = TFE_GetPythonString($input); + $1 = const_cast(TFE_GetPythonString($input)); } %typemap(in) (TFE_Context*) { diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index ea8c727e4d..44a95d6bad 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -358,14 +358,18 @@ def tf_workspace(path_prefix="", tf_repo_name=""): }, ) + PROTOBUF_urls =[ + "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v3.6.1.2.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.2.tar.gz", + ] + PROTOBUF_sha256 = "2244b0308846bb22b4ff0bcc675e99290ff9f1115553ae9671eba1030af31bc0" + PROTOBUF_strip_prefix = "protobuf-3.6.1.2" + tf_http_archive( name = "protobuf_archive", - urls = [ - "https://mirror.bazel.build/github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz", - "https://github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz", - ], - sha256 = "846d907acf472ae233ec0882ef3a2d24edbbe834b80c305e867ac65a1f2c59e3", - strip_prefix = "protobuf-396336eb961b75f03b25824fe86cf6490fb75e3a", + urls = PROTOBUF_urls, + sha256 = PROTOBUF_sha256, + strip_prefix = PROTOBUF_strip_prefix, ) # We need to import the protobuf library under the names com_google_protobuf @@ -373,22 +377,16 @@ def tf_workspace(path_prefix="", tf_repo_name=""): # Unfortunately there is no way to alias http_archives at the moment. tf_http_archive( name = "com_google_protobuf", - urls = [ - "https://mirror.bazel.build/github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz", - "https://github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz", - ], - sha256 = "846d907acf472ae233ec0882ef3a2d24edbbe834b80c305e867ac65a1f2c59e3", - strip_prefix = "protobuf-396336eb961b75f03b25824fe86cf6490fb75e3a", + urls = PROTOBUF_urls, + sha256 = PROTOBUF_sha256, + strip_prefix = PROTOBUF_strip_prefix, ) tf_http_archive( name = "com_google_protobuf_cc", - urls = [ - "https://mirror.bazel.build/github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz", - "https://github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz", - ], - sha256 = "846d907acf472ae233ec0882ef3a2d24edbbe834b80c305e867ac65a1f2c59e3", - strip_prefix = "protobuf-396336eb961b75f03b25824fe86cf6490fb75e3a", + urls = PROTOBUF_urls, + sha256 = PROTOBUF_sha256, + strip_prefix = PROTOBUF_strip_prefix, ) tf_http_archive(