From 280ec145e0944019f31faba42e8c1cac57e761ba Mon Sep 17 00:00:00 2001 From: liqun fu Date: Wed, 25 Mar 2020 11:20:15 -0700 Subject: [PATCH 1/6] support Pooling ops with Sequence axis --- .../proto/onnx/CNTKToONNX.cpp | 62 ++++++++++++++++++- bindings/python/cntk/tests/onnx_op_test.py | 26 +++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp b/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp index 25647576bb38..38fabf61fae7 100644 --- a/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp +++ b/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp @@ -845,6 +845,12 @@ class CNTKToONNXHelper std::unordered_map& variableNodes, std::vector& scanLoops, int createLoopIndex); + static onnxruntime::Node* CreatePoolingNode(const FunctionPtr& src, + onnxruntime::Graph* graph, + std::unordered_map& functionNodes, + std::unordered_map& variableNodes, + std::vector& scanLoops, int createLoopIndex); + static onnxruntime::Node* CreateConvolutionNode(const FunctionPtr& src, onnxruntime::Graph* graph, std::unordered_map& functionNodes, @@ -5396,6 +5402,9 @@ onnxruntime::Node* CNTKToONNXHelper::CreateNode(const FunctionPtr& src, std::string cntkOpName = ToLegacyString(ToUTF8(src->OpName())); std::string onnxOpName = ToOPName(src); + if (src->OpName() == L"Pooling") + std::cout << ""; + // TODO: uncomment this code once bidirectional LSTM is supprted. //if (cntkOpName == "Splice") //{ @@ -5629,7 +5638,10 @@ onnxruntime::Node* CNTKToONNXHelper::CreateNode(const FunctionPtr& src, else return CreateConvolutionNode(src, graph, functionNodes, variableNodes, scanLoops, createLoopIndex); } - + else if (src->OpName() == L"Pooling" && src->Inputs()[0].HasBatchAxis() && src->Inputs()[0].HasSequenceAxis()) + { + return CreatePoolingNode(src, graph, functionNodes, variableNodes, scanLoops, createLoopIndex); + } // // If this block node equivalent to a primitive ONNX OP, then treated as such. // And just maps its argument to ONNX node. @@ -7087,7 +7099,7 @@ void CNTKToONNXHelper::CopyAttributes(const FunctionPtr& src, onnxruntime::Node* auto lowerPad = ToINTS(src->Attributes()[L"lowerPad"].Value()); auto upperPad = ToINTS(src->Attributes()[L"upperPad"].Value()); - if (IsPadValueValid(lowerPad, upperPad, autoPadding, ceilOutDim)) + if (IsPadValueValid(lowerPad, upperPad, autoPadding, ceilOutDim) && !(src->Inputs()[0].HasBatchAxis() && src->Inputs()[0].HasSequenceAxis())) { if (ceilOutDim) ValidatePadValueForCeilOutDim(lowerPad, upperPad, autoPadding, kernelShape, inputShape, strides, @@ -8605,6 +8617,52 @@ onnxruntime::Node* ApplyActivationToSequenceConvolution(Node* convNode, const Fu return activationNode; } +onnxruntime::Node* CNTKToONNXHelper::CreatePoolingNode(const FunctionPtr& src, + onnxruntime::Graph* graph, + std::unordered_map& functionNodes, + std::unordered_map& variableNodes, + std::vector& scanLoops, int createLoopIndex) +{ + if (!src->Inputs()[0].HasBatchAxis() || !src->Inputs()[0].HasSequenceAxis()) + LogicError("CreatePoolingNode is only to handle MaxPool with batch and sequence dimensions."); + + std::vector inputs; + ProcessInputs(src, graph, functionNodes, variableNodes, inputs, + scanLoops, createLoopIndex); + + std::vector outputs; + ProcessOutputs(src, inputs, outputs, graph); + + // Max/AveragePool takes input of shape [N, C, H, W] or [N, C, D1, D2, ..., Dn]. CNTK input needs to be reshaped to match it. + // reshape [#, *][C, H, W] to [-1, C, H, W] + // onnx Max/AveragePool + // reshape [-1, C_out, H_out, W_out] to [#, *][C_out, H_out, W_out] + vector newDimInputToPooling; + // collapse extra dims into one axis as N for ONNX Conv + newDimInputToPooling.push_back(-1); + for (int i = 2; i < inputs[0]->Shape()->dim_size(); i++) + { + // copy C, H, W + if (!inputs[0]->Shape()->dim(i).has_dim_value()) + LogicError("Max/AveragePool: feature dimensions need to have dim value."); + newDimInputToPooling.push_back(inputs[0]->Shape()->dim(i).dim_value()); + } + + onnxruntime::Node* preReshape = AddReshapeNode(*inputs[0], newDimInputToPooling, inputs[0]->Name() + "_reshaped_for_max_pool", graph); + const std::vector pooling_inputs({const_cast(preReshape->OutputDefs()[0])}); + TypeProto poolingOutputTypeProto; + UpdateONNXType(src->Outputs()[0].GetDataType(), poolingOutputTypeProto); + + NodeArg *poolingOutputArg = &graph->GetOrCreateNodeArg(outputs[0]->Name() + "_pooling_of_reshaped", &poolingOutputTypeProto); + + onnxruntime::Node* poolingNode = AddNode(src, graph, pooling_inputs, { poolingOutputArg }); + + vector newDimOutputFromPooling = ToINTS(*outputs[0]->TypeAsProto()); + onnxruntime::Node* postReshape = AddReshapeNode(*poolingOutputArg, newDimOutputFromPooling, outputs[0]->Name(), graph); + + return poolingNode; +} + onnxruntime::Node* CNTKToONNXHelper::CreateConvolutionNode(const FunctionPtr& src, onnxruntime::Graph* graph, std::unordered_map& functionNodes, diff --git a/bindings/python/cntk/tests/onnx_op_test.py b/bindings/python/cntk/tests/onnx_op_test.py index 432dd28ae43b..46853d4ff3a9 100644 --- a/bindings/python/cntk/tests/onnx_op_test.py +++ b/bindings/python/cntk/tests/onnx_op_test.py @@ -423,6 +423,18 @@ def test_AveragePool(tmpdir, dtype, device_id): verify_one_input(model, img, tmpdir, 'AveragePool_2', device) +#AveragePool +@pytest.mark.parametrize("dtype", DType_Config) +def test_AvergaePoolWithSequenceAxis(tmpdir, dtype, device_id): + if device_id == -1 and dtype == np.float16: + pytest.skip('Test is skipped on CPU with float16 data') + device = cntk_device(device_id) + with C.default_options(dtype=dtype): + img = np.reshape(np.arange(16, dtype = dtype), [1, 4, 4]) + x = C.sequence.input_variable(img.shape) + model = C.pooling(x, C.AVG_POOLING, (2,2), (2,2)) + verify_sequence_model(model, np.reshape(img, [1, 1, 1, 4, 4]), tmpdir, "AveragePoolWithSeq_1", resave = False, bypass_load_into_cntk = True) + #BatchNormalization def verify_BN(x, init_scale, init_bias, mean, var, epsilon, spatial, tmpdir, dtype): with C.default_options(dtype = dtype): @@ -1311,7 +1323,7 @@ def test_Max(tmpdir, dtype): #MaxPool @pytest.mark.parametrize("dtype", DType_Config) -def test_MaxPool(tmpdir, dtype, device_id): +def test_MaxPool(tmpdir, dtype, device_id): if device_id == -1 and dtype == np.float16: pytest.skip('Test is skipped on CPU with float16 data') device = cntk_device(device_id) @@ -1327,6 +1339,18 @@ def test_MaxPool(tmpdir, dtype, device_id): model = C.pooling(x, C.MAX_POOLING, (3, 3), (2, 2), auto_padding=[False, False, False], ceil_out_dim=True) verify_one_input(model, img, tmpdir, 'MaxPool_2', device) +#MaxPool +@pytest.mark.parametrize("dtype", DType_Config) +def test_MaxPoolWithSequenceAxis(tmpdir, dtype, device_id): + if device_id == -1 and dtype == np.float16: + pytest.skip('Test is skipped on CPU with float16 data') + device = cntk_device(device_id) + with C.default_options(dtype=dtype): + img = np.reshape(np.arange(16, dtype = dtype), [1, 4, 4]) + x = C.sequence.input_variable(img.shape) + model = C.pooling(x, C.MAX_POOLING, (2,2), (2,2)) + verify_sequence_model(model, np.reshape(img, [1, 1, 1, 4, 4]), tmpdir, "MaxPoolWithSeq_1", resave = False, bypass_load_into_cntk = True) + #MaxRoiPool @pytest.mark.parametrize("dtype", DType_Config) def test_MaxRoiPool(tmpdir, dtype): From 9a7dd4c82bf50ac52dd3fb007eb4baa6cabf7858 Mon Sep 17 00:00:00 2001 From: liqun fu Date: Wed, 25 Mar 2020 12:37:46 -0700 Subject: [PATCH 2/6] add comments --- Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp b/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp index 38fabf61fae7..f3f5623ee557 100644 --- a/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp +++ b/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp @@ -5402,9 +5402,6 @@ onnxruntime::Node* CNTKToONNXHelper::CreateNode(const FunctionPtr& src, std::string cntkOpName = ToLegacyString(ToUTF8(src->OpName())); std::string onnxOpName = ToOPName(src); - if (src->OpName() == L"Pooling") - std::cout << ""; - // TODO: uncomment this code once bidirectional LSTM is supprted. //if (cntkOpName == "Splice") //{ @@ -5640,6 +5637,8 @@ onnxruntime::Node* CNTKToONNXHelper::CreateNode(const FunctionPtr& src, } else if (src->OpName() == L"Pooling" && src->Inputs()[0].HasBatchAxis() && src->Inputs()[0].HasSequenceAxis()) { + // in case a Pooling op is created with bother batch and sequence axes, we need to reshape its input and output to match + // ONNX spec of [N, C, H, W] shape requirement. return CreatePoolingNode(src, graph, functionNodes, variableNodes, scanLoops, createLoopIndex); } // @@ -7099,6 +7098,7 @@ void CNTKToONNXHelper::CopyAttributes(const FunctionPtr& src, onnxruntime::Node* auto lowerPad = ToINTS(src->Attributes()[L"lowerPad"].Value()); auto upperPad = ToINTS(src->Attributes()[L"upperPad"].Value()); + // lowerPad and upperPad have incorrect dimension when the op has both batch and sequence axes. if (IsPadValueValid(lowerPad, upperPad, autoPadding, ceilOutDim) && !(src->Inputs()[0].HasBatchAxis() && src->Inputs()[0].HasSequenceAxis())) { if (ceilOutDim) @@ -8617,6 +8617,7 @@ onnxruntime::Node* ApplyActivationToSequenceConvolution(Node* convNode, const Fu return activationNode; } +// insert reshape before and after a Pooling op when the CNTK op has both sequence and batch axes. onnxruntime::Node* CNTKToONNXHelper::CreatePoolingNode(const FunctionPtr& src, onnxruntime::Graph* graph, std::unordered_map& functionNodes, From 80e9f79ae6df0a322c924d71fa1357dfbe7e712c Mon Sep 17 00:00:00 2001 From: liqun fu Date: Fri, 27 Mar 2020 14:35:31 -0700 Subject: [PATCH 3/6] update with reviewers' comments --- Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp b/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp index f3f5623ee557..a5a94901f72c 100644 --- a/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp +++ b/Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp @@ -5637,7 +5637,7 @@ onnxruntime::Node* CNTKToONNXHelper::CreateNode(const FunctionPtr& src, } else if (src->OpName() == L"Pooling" && src->Inputs()[0].HasBatchAxis() && src->Inputs()[0].HasSequenceAxis()) { - // in case a Pooling op is created with bother batch and sequence axes, we need to reshape its input and output to match + // in case a Pooling op is created with both batch and sequence axes, we need to reshape its input and output to match // ONNX spec of [N, C, H, W] shape requirement. return CreatePoolingNode(src, graph, functionNodes, variableNodes, scanLoops, createLoopIndex); } @@ -7109,6 +7109,14 @@ void CNTKToONNXHelper::CopyAttributes(const FunctionPtr& src, onnxruntime::Node* } else { + if (src->Inputs()[0].HasBatchAxis() && src->Inputs()[0].HasSequenceAxis()) + { + if (!std::all_of(lowerPad.begin(), lowerPad.end(), [](int64_t pad) {return pad == 0; }) || + !std::all_of(upperPad.begin(), upperPad.end(), [](int64_t pad) {return pad == 0; })) + { + fprintf(stderr, "Warning: Cannot set upperPad and lowerPad with pooling ops. Padding values will be computed according to kernel and input shapes."); + } + } if (isPooling) PutPadAttrInNode(node, autoPadding, kernelShape, inputShape, strides, /*dilation=*/std::vector(kernelShape.Rank(), 1), ceilOutDim, /*transpose=*/!isPooling); @@ -8661,7 +8669,8 @@ onnxruntime::Node* CNTKToONNXHelper::CreatePoolingNode(const FunctionPtr& src, vector newDimOutputFromPooling = ToINTS(*outputs[0]->TypeAsProto()); onnxruntime::Node* postReshape = AddReshapeNode(*poolingOutputArg, newDimOutputFromPooling, outputs[0]->Name(), graph); - return poolingNode; + functionNodes.emplace(src, poolingNode); + return postReshape; } onnxruntime::Node* CNTKToONNXHelper::CreateConvolutionNode(const FunctionPtr& src, From a2055f6e3d624543e654c4a1565e91557dc9bf47 Mon Sep 17 00:00:00 2001 From: liqun fu Date: Fri, 27 Mar 2020 14:40:05 -0700 Subject: [PATCH 4/6] fix a typo --- bindings/python/cntk/tests/onnx_op_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/cntk/tests/onnx_op_test.py b/bindings/python/cntk/tests/onnx_op_test.py index 46853d4ff3a9..ff3f0bc544c8 100644 --- a/bindings/python/cntk/tests/onnx_op_test.py +++ b/bindings/python/cntk/tests/onnx_op_test.py @@ -425,7 +425,7 @@ def test_AveragePool(tmpdir, dtype, device_id): #AveragePool @pytest.mark.parametrize("dtype", DType_Config) -def test_AvergaePoolWithSequenceAxis(tmpdir, dtype, device_id): +def test_AveragePoolWithSequenceAxis(tmpdir, dtype, device_id): if device_id == -1 and dtype == np.float16: pytest.skip('Test is skipped on CPU with float16 data') device = cntk_device(device_id) From e9396480025b9ca457d26b6f33dd07c474c6aa04 Mon Sep 17 00:00:00 2001 From: liqunfu Date: Tue, 31 Mar 2020 08:55:14 -0700 Subject: [PATCH 5/6] disable fp16 test for poolWithSequenceAxis (#3810) --- bindings/python/cntk/tests/onnx_op_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bindings/python/cntk/tests/onnx_op_test.py b/bindings/python/cntk/tests/onnx_op_test.py index ff3f0bc544c8..ad4503e0b8df 100644 --- a/bindings/python/cntk/tests/onnx_op_test.py +++ b/bindings/python/cntk/tests/onnx_op_test.py @@ -426,8 +426,9 @@ def test_AveragePool(tmpdir, dtype, device_id): #AveragePool @pytest.mark.parametrize("dtype", DType_Config) def test_AveragePoolWithSequenceAxis(tmpdir, dtype, device_id): - if device_id == -1 and dtype == np.float16: - pytest.skip('Test is skipped on CPU with float16 data') + if dtype == np.float16: + # CI reporting "FP16 convolution is only supported via cuDNN." on GPU with float16 data + pytest.skip('Test is skipped with float16 data') device = cntk_device(device_id) with C.default_options(dtype=dtype): img = np.reshape(np.arange(16, dtype = dtype), [1, 4, 4]) @@ -1342,8 +1343,9 @@ def test_MaxPool(tmpdir, dtype, device_id): #MaxPool @pytest.mark.parametrize("dtype", DType_Config) def test_MaxPoolWithSequenceAxis(tmpdir, dtype, device_id): - if device_id == -1 and dtype == np.float16: - pytest.skip('Test is skipped on CPU with float16 data') + if dtype == np.float16: + # CI reporting "FP16 convolution is only supported via cuDNN." on GPU with float16 data + pytest.skip('Test is skipped with float16 data') device = cntk_device(device_id) with C.default_options(dtype=dtype): img = np.reshape(np.arange(16, dtype = dtype), [1, 4, 4]) From 10a8ffcf50d7b9225f3236ffcfdc422b2014fb92 Mon Sep 17 00:00:00 2001 From: "microsoft-github-policy-service[bot]" <77245923+microsoft-github-policy-service[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 16:06:50 +0200 Subject: [PATCH 6/6] Microsoft mandatory file (#3870) Co-authored-by: microsoft-github-policy-service[bot] <77245923+microsoft-github-policy-service[bot]@users.noreply.github.com> --- SECURITY.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000000..e138ec5d6a77 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). + +