diff --git a/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx b/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx index 4fd149f747e06..cc31addf82dd4 100644 --- a/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx @@ -73,13 +73,21 @@ public: std::vector kernelShape, std::vector outputPadding, std::vector outputShape, std::vector pads, std::vector strides, std::string nameX, std::string nameW, std::string nameB, std::string nameY) - : fAttrAutopad(autopad), fAttrDilations(dilations), fAttrGroup(group), fAttrKernelShape(kernelShape), - fAttrOutputPadding(outputPadding), fAttrOutputShape(outputShape), fAttrPads(pads), fAttrStrides(strides), - fNX(UTILITY::Clean_name(nameX)), fNW(UTILITY::Clean_name(nameW)), fNB(UTILITY::Clean_name(nameB)), + : fAttrAutopad(autopad), + fAttrDilations(dilations), + fAttrGroup(group), + fAttrKernelShape(kernelShape), + fAttrOutputPadding(outputPadding), + fAttrOutputShape(outputShape), + fAttrPads(pads), + fAttrStrides(strides), + fNX(UTILITY::Clean_name(nameX)), + fNW(UTILITY::Clean_name(nameW)), + fNB(UTILITY::Clean_name(nameB)), fNY(UTILITY::Clean_name(nameY)) { - fInputTensorNames = { fNX, fNW }; - fOutputTensorNames = { fNY }; + fInputTensorNames = {fNX, fNW}; + fOutputTensorNames = {fNY}; if (!fNB.empty()) { fInputTensorNames.emplace_back(fNB); } @@ -121,7 +129,7 @@ public: /*! \brief Returns the blas routines needed to compile the generated code */ - std::vector GetBlasRoutines() override { return { std::string("Gemm"), std::string("Axpy") }; } + std::vector GetBlasRoutines() override { return {std::string("Gemm"), std::string("Axpy")}; } }; template @@ -162,30 +170,7 @@ auto ROperator_ConvTranspose::ShapeInference(std::vector> // Generate the padding if (fAttrPads.empty()) { fAttrPads = std::vector(2 * fDim, 0); - if (fAttrOutputShape.size() == fDim) { - // LM: to be checked... - // for time being not support - throw std::runtime_error("ConvTranspose with output_shape explicitly set not yet supported."); - /* - std::vector totalPadding(fDim, 1); - for (size_t i = 0; i < fDim; i++) { - size_t j = i + 2; - totalPadding[i] = - fAttrStrides[i] * (fAttrOutputShape[i] - 1) + fAttrOutputPadding[i] + fAttrKernelShape[i] - fShapeX[j]; - } - for (size_t i = 0; i < fDim; i++) { - size_t end_i = i + fDim; - if (fAttrAutopad == "SAME_UPPER") { - fAttrPads[i] = totalPadding[i] / 2; - fAttrPads[end_i] = totalPadding[i] - fAttrPads[i]; - } else { - fAttrPads[end_i] = totalPadding[i] / 2; - fAttrPads[i] = totalPadding[i] - fAttrPads[end_i]; - } - } - */ - } if (fAttrAutopad != "NOTSET") { throw std::runtime_error("ConvTranspose with padding SAME_UPPER or SMAE_LOWER not supported"); } @@ -199,8 +184,29 @@ auto ROperator_ConvTranspose::ShapeInference(std::vector> } } else { // The shape of the output is explicitly set - // TODO Generate the padding from the output shape and the input shape - throw std::runtime_error("ConvTranspose with output_shape explicitly set not yet supported."); + fAttrPads = std::vector(2 * fDim, 0); + for (size_t i = 0; i < fDim; ++i) { + size_t input_shape = inputShape[i + 2]; + size_t output_shape = fAttrOutputShape[i]; + size_t kernel_shape = weightShape[i + 2]; + + size_t stride = fAttrStrides[i]; + size_t dilation = fAttrDilations[i]; + size_t output_padding = fAttrOutputPadding[i]; + + size_t effective_kernel_shape = (kernel_shape - 1) * dilation + 1; + size_t expected_shape_without_pad = (input_shape - 1) * stride + output_padding + effective_kernel_shape; + + if (expected_shape_without_pad < output_shape) { + throw std::runtime_error("ConvTranspose: explicitly set output_shape is too large for " + "the given input and kernel shapes."); + } + + size_t total_padding = expected_shape_without_pad - output_shape; + + fAttrPads[i + fDim] = total_padding / 2; + fAttrPads[i] = total_padding - fAttrPads[i + fDim]; + } } for (size_t i = 0; i < fDim; i++) @@ -319,8 +325,8 @@ std::string ROperator_ConvTranspose::GenerateInitCode() if (bsize != ysize && !fNBroadcastedB.empty()) { // include a separate scope to avoid defining unique operator temp variables out << SP << "{\n"; - out << SP << SP << "float * data = UTILITY::BroadcastConvBias(tensor_" << fNB - << ", " << bsize << ", " << ConvertShapeToString(fShapeY) << ");\n"; + out << SP << SP << "float * data = UTILITY::BroadcastConvBias(tensor_" << fNB << ", " << bsize << ", " + << ConvertShapeToString(fShapeY) << ");\n"; out << SP << SP << "std::copy(data, data + " << ConvertShapeToLength(fShapeY) << ", tensor_" << fNBroadcastedB << ");\n"; out << SP << SP << "delete[] data;\n"; @@ -447,31 +453,6 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) // trick for speed is using caffe im2col and output a matrix which contains filtered values as rows. // By doing this one has consecutive memory reads and writes // Resulting matrix op_xcol is (output channels * filter_h * filter_w , output_h * output_w) - if (fDim == 1) { - if (fAttrPads[0] != fAttrPads[1]) { - std::cout << "TMVA SOFIE Operator Conv: asymmetric padding not supported. Assume an average padding " - << std::endl; - fAttrPads[0] = (fAttrPads[0] + fAttrPads[1]) / 2; - } - fAttrPads[1] = 0; - } - if (fDim == 2) { - if (fAttrPads[0] != fAttrPads[2] || fAttrPads[1] != fAttrPads[3]) { - std::cout << "TMVA SOFIE Operator ConvTranspose: asymmetric padding not supported. Assume an average padding " - << std::endl; - fAttrPads[0] = (fAttrPads[0] + fAttrPads[2]) / 2; - fAttrPads[1] = (fAttrPads[1] + fAttrPads[3]) / 2; - } - } - if (fDim == 3) { - if (fAttrPads[0] != fAttrPads[3] || fAttrPads[1] != fAttrPads[4] || fAttrPads[2] != fAttrPads[5]) { - std::cout << "TMVA SOFIE Operator ConvTranspose: asymmetric padding not supported. Assume an average padding " - << std::endl; - fAttrPads[0] = (fAttrPads[0] + fAttrPads[3]) / 2; - fAttrPads[1] = (fAttrPads[1] + fAttrPads[4]) / 2; - fAttrPads[2] = (fAttrPads[2] + fAttrPads[5]) / 2; - } - } if (fAttrGroup == 1) { out << SP << SP << "size_t x_offset = n * " << fShapeX[1] * iDepth * iHeight * iWidth << ";\n"; @@ -492,16 +473,16 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) if (fDim < 3) { out << SP << SP << "UTILITY::col2im(tensor_" << fNX << "_xcol," - // channels, height, width, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, dilation_h, - // dilation_w, + // channels, height, width, kernel_h, kernel_w, pad_h_begin, pad_h_end, pad_w_begin, pad_w_end, + // stride_h, stride_w, dilation_h, dilation_w, << fShapeY[1] << "," << oHeight << "," << oWidth << ","; if (fDim == 1) - out << "1, " << fAttrKernelShape[0] << ",0," << fAttrPads[0] << ",1," << fAttrStrides[0] << ",1," - << fAttrDilations[0]; + out << "1, " << fAttrKernelShape[0] << ",0,0," << fAttrPads[0] << "," << fAttrPads[1] << ",1," + << fAttrStrides[0] << ",1," << fAttrDilations[0]; else // dim ==2 - out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[1] - << "," << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrDilations[0] << "," - << fAttrDilations[1]; + out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[2] + << "," << fAttrPads[1] << "," << fAttrPads[3] << "," << fAttrStrides[0] << "," << fAttrStrides[1] << "," + << fAttrDilations[0] << "," << fAttrDilations[1]; out << ", tensor_" << fNY << " + out_offset);\n\n "; } else { // 3d : needs a col2im for 3d @@ -514,9 +495,8 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) << fShapeX[1] << "," << oDepth << "," << oHeight << "," << oWidth << "," << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrKernelShape[2] << "," << fAttrPads[0] << "," << fAttrPads[3] << "," << fAttrPads[1] << "," << fAttrPads[4] << "," << fAttrPads[2] << "," << fAttrPads[5] << "," - << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << "," - << fAttrDilations[0] << "," << fAttrDilations[1] << "," << fAttrDilations[2] << ",tensor_" << fNX - << "_xcol);\n\n "; + << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << "," << fAttrDilations[0] << "," + << fAttrDilations[1] << "," << fAttrDilations[2] << ",tensor_" << fNX << "_xcol);\n\n "; } // // BLAS // out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &" @@ -545,16 +525,16 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) if (fDim < 3) { out << SP << SP << "UTILITY::col2im(tensor_" << fNX << "_xcol," - // channels, height, width, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, dilation_h, - // dilation_w, + // channels, height, width, kernel_h, kernel_w, pad_h_begin, pad_h_end, pad_w_begin, pad_w_end, + // stride_h, stride_w, dilation_h, dilation_w, << fShapeY[1] << "," << oHeight << "," << oWidth << ","; if (fDim == 1) - out << "1, " << fAttrKernelShape[0] << ",0," << fAttrPads[0] << ",1," << fAttrStrides[0] << ",1," - << fAttrDilations[0]; + out << "1, " << fAttrKernelShape[0] << ",0,0," << fAttrPads[0] << "," << fAttrPads[1] << ",1," + << fAttrStrides[0] << ",1," << fAttrDilations[0]; else // dim ==2 - out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[1] - << "," << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrDilations[0] << "," - << fAttrDilations[1]; + out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[2] + << "," << fAttrPads[1] << "," << fAttrPads[3] << "," << fAttrStrides[0] << "," << fAttrStrides[1] << "," + << fAttrDilations[0] << "," << fAttrDilations[1]; out << ", tensor_" << fNY << " + out_offset);\n\n "; } else { // 3d im2col @@ -568,9 +548,8 @@ std::string ROperator_ConvTranspose::Generate(std::string OpName) << fShapeX[1] << "," << oDepth << "," << oHeight << "," << oWidth << "," << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrKernelShape[2] << "," << fAttrPads[0] << "," << fAttrPads[3] << "," << fAttrPads[1] << "," << fAttrPads[4] << "," << fAttrPads[2] << "," << fAttrPads[5] << "," - << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << "," - << fAttrDilations[0] << "," << fAttrDilations[1] << "," << fAttrDilations[2] << "," << "tensor_" << fNX - << "_xcol);\n\n "; + << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << "," << fAttrDilations[0] << "," + << fAttrDilations[1] << "," << fAttrDilations[2] << "," << "tensor_" << fNX << "_xcol);\n\n "; } // // BLAS diff --git a/tmva/sofie/src/SOFIE_common_helpers.cxx b/tmva/sofie/src/SOFIE_common_helpers.cxx index 0cd550180eaa1..86f35602ad542 100644 --- a/tmva/sofie/src/SOFIE_common_helpers.cxx +++ b/tmva/sofie/src/SOFIE_common_helpers.cxx @@ -174,25 +174,25 @@ constexpr const char *kCol2im = R"SOFIE( template void col2im(const Dtype *data_col, const int channels, const int height, const int width, const int kernel_h, const int kernel_w, - const int pad_h, const int pad_w, + const int pad_h_begin, const int pad_h_end, const int pad_w_begin, const int pad_w_end, const int stride_h, const int stride_w, const int dilation_h, const int dilation_w, Dtype *data_im) { // output must start zeroed: col2im scatters with += so overlapping columns accumulate std::fill(data_im, data_im + height * width * channels, 0.); - const int output_h = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; - const int output_w = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + const int output_h = (height + pad_h_begin + pad_h_end - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int output_w = (width + pad_w_begin + pad_w_end - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; const int channel_size = height * width; for (int channel = channels; channel--; data_im += channel_size) { for (int kernel_row = 0; kernel_row < kernel_h; kernel_row++) { for (int kernel_col = 0; kernel_col < kernel_w; kernel_col++) { - int input_row = -pad_h + kernel_row * dilation_h; + int input_row = -pad_h_begin + kernel_row * dilation_h; for (int output_rows = output_h; output_rows; output_rows--) { if (!is_a_ge_zero_and_a_lt_b(input_row, height)) { data_col += output_w; } else { - int input_col = -pad_w + kernel_col * dilation_w; + int input_col = -pad_w_begin + kernel_col * dilation_w; for (int output_col = output_w; output_col; output_col--) { if (is_a_ge_zero_and_a_lt_b(input_col, width)) { data_im[input_row * width + input_col] += *data_col; diff --git a/tmva/sofie/test/TestCustomModelsFromONNX.cxx b/tmva/sofie/test/TestCustomModelsFromONNX.cxx index 43a90481b62dd..f807d2ae8a641 100644 --- a/tmva/sofie/test/TestCustomModelsFromONNX.cxx +++ b/tmva/sofie/test/TestCustomModelsFromONNX.cxx @@ -901,6 +901,15 @@ TEST(ONNX, ConvTranspose2d) expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE); } +TEST(ONNX, ConvTranspose2dOutputShape) +{ + SofieReference ref = readReference("ConvTranspose2dOutputShape"); + + ASSERT_INCLUDE_AND_RUN(std::vector, "ConvTranspose2dOutputShape", ref.f32("input0")); + + expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE); +} + /* ConvTranspose3d is not supported yet; a ConvTranspose3d model would have to be added to generate_input_models.py to enable this test. TEST(ONNX, ConvTranspose3d) diff --git a/tmva/sofie/test/generate_input_models.py b/tmva/sofie/test/generate_input_models.py index 5b7def0ec56df..59b4516d8da57 100644 --- a/tmva/sofie/test/generate_input_models.py +++ b/tmva/sofie/test/generate_input_models.py @@ -695,6 +695,35 @@ def make_ConvTranspose2d(): return _model(graph, opset=17, ir_version=8) +def make_ConvTranspose2dOutputShape(): + """Ops: ConvTranspose""" + nodes = [ + helper.make_node( + 'ConvTranspose', + ['X', 'W'], + ['Y'], + kernel_shape=[3, 3], + strides=[2, 2], + output_shape=[6, 6] + ), + ] + graph = helper.make_graph( + nodes, + 'ConvTranspose2dOutputShape', + inputs=[ + _vi('X', FLOAT, [1, 1, 3, 3]), + _vi('W', FLOAT, [1, 1, 3, 3]), + ], + outputs=[ + _vi('Y', FLOAT, [1, 1, 6, 6]), + ], + initializer=[ + _tensor('W', FLOAT, [1, 1, 3, 3], [1.0] * 9), + ], + ) + return _model(graph, opset=17, ir_version=8) + + def make_ConvTransposeBias2d(): """Ops: ConvTranspose""" nodes = [ @@ -5092,6 +5121,7 @@ def make_Where(): 'ConvAddRelu': make_ConvAddRelu, 'ConvTranspose1d': make_ConvTranspose1d, 'ConvTranspose2d': make_ConvTranspose2d, + 'ConvTranspose2dOutputShape': make_ConvTranspose2dOutputShape, 'ConvTransposeBias2d': make_ConvTransposeBias2d, 'ConvTransposeBias2dBatched': make_ConvTransposeBias2dBatched, 'ConvWithAsymmetricPadding': make_ConvWithAsymmetricPadding, @@ -5307,6 +5337,7 @@ def rand_f32(seed, shape): 'ConvAddRelu': [f32(np.arange(-7.0, 9.0), (1, 1, 4, 4))], 'ConvTranspose1d': [f32(np.arange(0.0, 3.0), (1, 1, 3))], 'ConvTranspose2d': [f32(np.arange(0.0, 9.0), (1, 1, 3, 3))], + 'ConvTranspose2dOutputShape': [f32(np.arange(0.0, 9.0), (1, 1, 3, 3))], 'ConvTransposeBias2d': [f32(np.arange(0.0, 9.0), (1, 1, 3, 3))], 'ConvTransposeBias2dBatched': [f32(np.arange(0.0, 18.0), (2, 1, 3, 3))], 'ConvWithAsymmetricPadding': [f32(np.arange(0.0, 35.0), (1, 1, 7, 5))], @@ -5631,6 +5662,29 @@ def _mean_reference(model, feeds): return [np.mean(np.broadcast_arrays(*feeds.values()), axis=0, dtype=np.float32)] +def _convtranspose_outputshape_reference(model, feeds): + """The ReferenceEvaluator crashes if output_shape is set but pads is not. + We temporarily add the correct inferred pads to evaluate it, then remove + them so the saved model strictly tests SOFIE's inference logic.""" + from onnx import helper + + for node in model.graph.node: + if node.op_type == "ConvTranspose" and "pads" not in [a.name for a in node.attribute]: + node.attribute.extend([helper.make_attribute("pads", [1, 1, 0, 0])]) + + from onnx.reference import ReferenceEvaluator + + outputs = ReferenceEvaluator(model).run(None, feeds) + + for node in model.graph.node: + if node.op_type == "ConvTranspose": + for i, attr in enumerate(node.attribute): + if attr.name == "pads": + del node.attribute[i] + break + return outputs + + # Models whose expected outputs the ReferenceEvaluator cannot compute. EXPECTED_OVERRIDES = { "GRUBidirectional": _recurrent_reference, @@ -5641,6 +5695,7 @@ def _mean_reference(model, feeds): "RNNSequenceBatchwise": _recurrent_reference, "MaxPool2d_AsymPad": _maxpool2d_reference, "MeanMultidirectionalBroadcast": _mean_reference, + "ConvTranspose2dOutputShape": _convtranspose_outputshape_reference, }