Skip to content

[tmva][sofie] Implement padding generation for ConvTranspose with explicitly set output_shape #23088

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Sainava wants to merge 1 commit into
base: master
Choose a base branch
from
+126 −83
Open

[tmva][sofie] Implement padding generation for ConvTranspose with explicitly set output_shape #23088

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 57 additions & 78 deletions tmva/sofie/inc/TMVA/ROperator_ConvTranspose.hxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,21 @@ public:
std::vector<size_t> kernelShape, std::vector<size_t> outputPadding,
std::vector<size_t> outputShape, std::vector<size_t> pads, std::vector<size_t> strides,
std::string nameX, std::string nameW, std::string nameB, std::string nameY)
: fAttrAutopad(autopad), fAttrDilations(dilations), fAttrGroup(group), fAttrKernelShape(kernelShape),
fAttrOutputPadding(outputPadding), fAttrOutputShape(outputShape), fAttrPads(pads), fAttrStrides(strides),
fNX(UTILITY::Clean_name(nameX)), fNW(UTILITY::Clean_name(nameW)), fNB(UTILITY::Clean_name(nameB)),
: fAttrAutopad(autopad),
fAttrDilations(dilations),
fAttrGroup(group),
fAttrKernelShape(kernelShape),
fAttrOutputPadding(outputPadding),
fAttrOutputShape(outputShape),
fAttrPads(pads),
fAttrStrides(strides),
fNX(UTILITY::Clean_name(nameX)),
fNW(UTILITY::Clean_name(nameW)),
fNB(UTILITY::Clean_name(nameB)),
fNY(UTILITY::Clean_name(nameY))
{
fInputTensorNames = { fNX, fNW };
fOutputTensorNames = { fNY };
fInputTensorNames = {fNX, fNW};
fOutputTensorNames = {fNY};
if (!fNB.empty()) {
fInputTensorNames.emplace_back(fNB);
}
Expand Down Expand Up @@ -121,7 +129,7 @@ public:

/*! \brief Returns the blas routines needed to compile the generated code
*/
std::vector<std::string> GetBlasRoutines() override { return { std::string("Gemm"), std::string("Axpy") }; }
std::vector<std::string> GetBlasRoutines() override { return {std::string("Gemm"), std::string("Axpy")}; }
};

template <typename T>
Expand Down Expand Up @@ -162,30 +170,7 @@ auto ROperator_ConvTranspose<T>::ShapeInference(std::vector<std::vector<size_t>>
// Generate the padding
if (fAttrPads.empty()) {
fAttrPads = std::vector<size_t>(2 * fDim, 0);
if (fAttrOutputShape.size() == fDim) {
// LM: to be checked...
// for time being not support
throw std::runtime_error("ConvTranspose with output_shape explicitly set not yet supported.");
/*
std::vector<size_t> totalPadding(fDim, 1);
for (size_t i = 0; i < fDim; i++) {
size_t j = i + 2;
totalPadding[i] =
fAttrStrides[i] * (fAttrOutputShape[i] - 1) + fAttrOutputPadding[i] + fAttrKernelShape[i] - fShapeX[j];
}

for (size_t i = 0; i < fDim; i++) {
size_t end_i = i + fDim;
if (fAttrAutopad == "SAME_UPPER") {
fAttrPads[i] = totalPadding[i] / 2;
fAttrPads[end_i] = totalPadding[i] - fAttrPads[i];
} else {
fAttrPads[end_i] = totalPadding[i] / 2;
fAttrPads[i] = totalPadding[i] - fAttrPads[end_i];
}
}
*/
}
if (fAttrAutopad != "NOTSET") {
throw std::runtime_error("ConvTranspose with padding SAME_UPPER or SMAE_LOWER not supported");
}
Expand All @@ -199,8 +184,29 @@ auto ROperator_ConvTranspose<T>::ShapeInference(std::vector<std::vector<size_t>>
}
} else {
// The shape of the output is explicitly set
// TODO Generate the padding from the output shape and the input shape
throw std::runtime_error("ConvTranspose with output_shape explicitly set not yet supported.");
fAttrPads = std::vector<size_t>(2 * fDim, 0);
for (size_t i = 0; i < fDim; ++i) {
size_t input_shape = inputShape[i + 2];
size_t output_shape = fAttrOutputShape[i];
size_t kernel_shape = weightShape[i + 2];

size_t stride = fAttrStrides[i];
size_t dilation = fAttrDilations[i];
size_t output_padding = fAttrOutputPadding[i];

size_t effective_kernel_shape = (kernel_shape - 1) * dilation + 1;
size_t expected_shape_without_pad = (input_shape - 1) * stride + output_padding + effective_kernel_shape;

if (expected_shape_without_pad < output_shape) {
throw std::runtime_error("ConvTranspose: explicitly set output_shape is too large for "
"the given input and kernel shapes.");
}

size_t total_padding = expected_shape_without_pad - output_shape;

fAttrPads[i + fDim] = total_padding / 2;
fAttrPads[i] = total_padding - fAttrPads[i + fDim];
}
}

for (size_t i = 0; i < fDim; i++)
Expand Down Expand Up @@ -319,8 +325,8 @@ std::string ROperator_ConvTranspose<T>::GenerateInitCode()
if (bsize != ysize && !fNBroadcastedB.empty()) {
// include a separate scope to avoid defining unique operator temp variables
out << SP << "{\n";
out << SP << SP << "float * data = UTILITY::BroadcastConvBias<float>(tensor_" << fNB
<< ", " << bsize << ", " << ConvertShapeToString(fShapeY) << ");\n";
out << SP << SP << "float * data = UTILITY::BroadcastConvBias<float>(tensor_" << fNB << ", " << bsize << ", "
<< ConvertShapeToString(fShapeY) << ");\n";
out << SP << SP << "std::copy(data, data + " << ConvertShapeToLength(fShapeY) << ", tensor_" << fNBroadcastedB
<< ");\n";
out << SP << SP << "delete[] data;\n";
Expand Down Expand Up @@ -447,31 +453,6 @@ std::string ROperator_ConvTranspose<T>::Generate(std::string OpName)
// trick for speed is using caffe im2col and output a matrix which contains filtered values as rows.
// By doing this one has consecutive memory reads and writes
// Resulting matrix op_xcol is (output channels * filter_h * filter_w , output_h * output_w)
if (fDim == 1) {
if (fAttrPads[0] != fAttrPads[1]) {
std::cout << "TMVA SOFIE Operator Conv: asymmetric padding not supported. Assume an average padding "
<< std::endl;
fAttrPads[0] = (fAttrPads[0] + fAttrPads[1]) / 2;
}
fAttrPads[1] = 0;
}
if (fDim == 2) {
if (fAttrPads[0] != fAttrPads[2] || fAttrPads[1] != fAttrPads[3]) {
std::cout << "TMVA SOFIE Operator ConvTranspose: asymmetric padding not supported. Assume an average padding "
<< std::endl;
fAttrPads[0] = (fAttrPads[0] + fAttrPads[2]) / 2;
fAttrPads[1] = (fAttrPads[1] + fAttrPads[3]) / 2;
}
}
if (fDim == 3) {
if (fAttrPads[0] != fAttrPads[3] || fAttrPads[1] != fAttrPads[4] || fAttrPads[2] != fAttrPads[5]) {
std::cout << "TMVA SOFIE Operator ConvTranspose: asymmetric padding not supported. Assume an average padding "
<< std::endl;
fAttrPads[0] = (fAttrPads[0] + fAttrPads[3]) / 2;
fAttrPads[1] = (fAttrPads[1] + fAttrPads[4]) / 2;
fAttrPads[2] = (fAttrPads[2] + fAttrPads[5]) / 2;
}
}

if (fAttrGroup == 1) {
out << SP << SP << "size_t x_offset = n * " << fShapeX[1] * iDepth * iHeight * iWidth << ";\n";
Expand All @@ -492,16 +473,16 @@ std::string ROperator_ConvTranspose<T>::Generate(std::string OpName)
if (fDim < 3) {
out << SP << SP << "UTILITY::col2im<float>(tensor_" << fNX
<< "_xcol,"
// channels, height, width, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, dilation_h,
// dilation_w,
// channels, height, width, kernel_h, kernel_w, pad_h_begin, pad_h_end, pad_w_begin, pad_w_end,
// stride_h, stride_w, dilation_h, dilation_w,
<< fShapeY[1] << "," << oHeight << "," << oWidth << ",";
if (fDim == 1)
out << "1, " << fAttrKernelShape[0] << ",0," << fAttrPads[0] << ",1," << fAttrStrides[0] << ",1,"
<< fAttrDilations[0];
out << "1, " << fAttrKernelShape[0] << ",0,0," << fAttrPads[0] << "," << fAttrPads[1] << ",1,"
<< fAttrStrides[0] << ",1," << fAttrDilations[0];
else // dim ==2
out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[1]
<< "," << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrDilations[0] << ","
<< fAttrDilations[1];
out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[2]
<< "," << fAttrPads[1] << "," << fAttrPads[3] << "," << fAttrStrides[0] << "," << fAttrStrides[1] << ","
<< fAttrDilations[0] << "," << fAttrDilations[1];
out << ", tensor_" << fNY << " + out_offset);\n\n ";
} else {
// 3d : needs a col2im for 3d
Expand All @@ -514,9 +495,8 @@ std::string ROperator_ConvTranspose<T>::Generate(std::string OpName)
<< fShapeX[1] << "," << oDepth << "," << oHeight << "," << oWidth << "," << fAttrKernelShape[0] << ","
<< fAttrKernelShape[1] << "," << fAttrKernelShape[2] << "," << fAttrPads[0] << "," << fAttrPads[3] << ","
<< fAttrPads[1] << "," << fAttrPads[4] << "," << fAttrPads[2] << "," << fAttrPads[5] << ","
<< fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << ","
<< fAttrDilations[0] << "," << fAttrDilations[1] << "," << fAttrDilations[2] << ",tensor_" << fNX
<< "_xcol);\n\n ";
<< fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << "," << fAttrDilations[0] << ","
<< fAttrDilations[1] << "," << fAttrDilations[2] << ",tensor_" << fNX << "_xcol);\n\n ";
}
// // BLAS
// out << SP << SP << "BLAS::sgemm_(&" << OpName << "_transA, &" << OpName << "_transB, &" << OpName << "_m, &"
Expand Down Expand Up @@ -545,16 +525,16 @@ std::string ROperator_ConvTranspose<T>::Generate(std::string OpName)
if (fDim < 3) {
out << SP << SP << "UTILITY::col2im<float>(tensor_" << fNX
<< "_xcol,"
// channels, height, width, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, dilation_h,
// dilation_w,
// channels, height, width, kernel_h, kernel_w, pad_h_begin, pad_h_end, pad_w_begin, pad_w_end,
// stride_h, stride_w, dilation_h, dilation_w,
<< fShapeY[1] << "," << oHeight << "," << oWidth << ",";
if (fDim == 1)
out << "1, " << fAttrKernelShape[0] << ",0," << fAttrPads[0] << ",1," << fAttrStrides[0] << ",1,"
<< fAttrDilations[0];
out << "1, " << fAttrKernelShape[0] << ",0,0," << fAttrPads[0] << "," << fAttrPads[1] << ",1,"
<< fAttrStrides[0] << ",1," << fAttrDilations[0];
else // dim ==2
out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[1]
<< "," << fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrDilations[0] << ","
<< fAttrDilations[1];
out << fAttrKernelShape[0] << "," << fAttrKernelShape[1] << "," << fAttrPads[0] << "," << fAttrPads[2]
<< "," << fAttrPads[1] << "," << fAttrPads[3] << "," << fAttrStrides[0] << "," << fAttrStrides[1] << ","
<< fAttrDilations[0] << "," << fAttrDilations[1];
out << ", tensor_" << fNY << " + out_offset);\n\n ";
} else {
// 3d im2col
Expand All @@ -568,9 +548,8 @@ std::string ROperator_ConvTranspose<T>::Generate(std::string OpName)
<< fShapeX[1] << "," << oDepth << "," << oHeight << "," << oWidth << "," << fAttrKernelShape[0] << ","
<< fAttrKernelShape[1] << "," << fAttrKernelShape[2] << "," << fAttrPads[0] << "," << fAttrPads[3] << ","
<< fAttrPads[1] << "," << fAttrPads[4] << "," << fAttrPads[2] << "," << fAttrPads[5] << ","
<< fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << ","
<< fAttrDilations[0] << "," << fAttrDilations[1] << "," << fAttrDilations[2] << "," << "tensor_" << fNX
<< "_xcol);\n\n ";
<< fAttrStrides[0] << "," << fAttrStrides[1] << "," << fAttrStrides[2] << "," << fAttrDilations[0] << ","
<< fAttrDilations[1] << "," << fAttrDilations[2] << "," << "tensor_" << fNX << "_xcol);\n\n ";
}

// // BLAS
Expand Down
10 changes: 5 additions & 5 deletions tmva/sofie/src/SOFIE_common_helpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,25 +174,25 @@ constexpr const char *kCol2im = R"SOFIE(
template <typename Dtype>
void col2im(const Dtype *data_col, const int channels,
const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w,
const int pad_h_begin, const int pad_h_end, const int pad_w_begin, const int pad_w_end,
const int stride_h, const int stride_w,
const int dilation_h, const int dilation_w,
Dtype *data_im)
{
// output must start zeroed: col2im scatters with += so overlapping columns accumulate
std::fill(data_im, data_im + height * width * channels, 0.);
const int output_h = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1;
const int output_w = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1;
const int output_h = (height + pad_h_begin + pad_h_end - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1;
const int output_w = (width + pad_w_begin + pad_w_end - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1;
const int channel_size = height * width;
for (int channel = channels; channel--; data_im += channel_size) {
for (int kernel_row = 0; kernel_row < kernel_h; kernel_row++) {
for (int kernel_col = 0; kernel_col < kernel_w; kernel_col++) {
int input_row = -pad_h + kernel_row * dilation_h;
int input_row = -pad_h_begin + kernel_row * dilation_h;
for (int output_rows = output_h; output_rows; output_rows--) {
if (!is_a_ge_zero_and_a_lt_b(input_row, height)) {
data_col += output_w;
} else {
int input_col = -pad_w + kernel_col * dilation_w;
int input_col = -pad_w_begin + kernel_col * dilation_w;
for (int output_col = output_w; output_col; output_col--) {
if (is_a_ge_zero_and_a_lt_b(input_col, width)) {
data_im[input_row * width + input_col] += *data_col;
Expand Down
9 changes: 9 additions & 0 deletions tmva/sofie/test/TestCustomModelsFromONNX.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -901,6 +901,15 @@ TEST(ONNX, ConvTranspose2d)
expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE);
}

TEST(ONNX, ConvTranspose2dOutputShape)
{
SofieReference ref = readReference("ConvTranspose2dOutputShape");

ASSERT_INCLUDE_AND_RUN(std::vector<float>, "ConvTranspose2dOutputShape", ref.f32("input0"));

expectNear(output, ref.f32("output0"), DEFAULT_TOLERANCE);
}

/* ConvTranspose3d is not supported yet; a ConvTranspose3d model would have
to be added to generate_input_models.py to enable this test.
TEST(ONNX, ConvTranspose3d)
Expand Down
55 changes: 55 additions & 0 deletions tmva/sofie/test/generate_input_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,35 @@ def make_ConvTranspose2d():
return _model(graph, opset=17, ir_version=8)


def make_ConvTranspose2dOutputShape():
"""Ops: ConvTranspose"""
nodes = [
helper.make_node(
'ConvTranspose',
['X', 'W'],
['Y'],
kernel_shape=[3, 3],
strides=[2, 2],
output_shape=[6, 6]
),
]
graph = helper.make_graph(
nodes,
'ConvTranspose2dOutputShape',
inputs=[
_vi('X', FLOAT, [1, 1, 3, 3]),
_vi('W', FLOAT, [1, 1, 3, 3]),
],
outputs=[
_vi('Y', FLOAT, [1, 1, 6, 6]),
],
initializer=[
_tensor('W', FLOAT, [1, 1, 3, 3], [1.0] * 9),
],
)
return _model(graph, opset=17, ir_version=8)


def make_ConvTransposeBias2d():
"""Ops: ConvTranspose"""
nodes = [
Expand Down Expand Up @@ -5092,6 +5121,7 @@ def make_Where():
'ConvAddRelu': make_ConvAddRelu,
'ConvTranspose1d': make_ConvTranspose1d,
'ConvTranspose2d': make_ConvTranspose2d,
'ConvTranspose2dOutputShape': make_ConvTranspose2dOutputShape,
'ConvTransposeBias2d': make_ConvTransposeBias2d,
'ConvTransposeBias2dBatched': make_ConvTransposeBias2dBatched,
'ConvWithAsymmetricPadding': make_ConvWithAsymmetricPadding,
Expand Down Expand Up @@ -5307,6 +5337,7 @@ def rand_f32(seed, shape):
'ConvAddRelu': [f32(np.arange(-7.0, 9.0), (1, 1, 4, 4))],
'ConvTranspose1d': [f32(np.arange(0.0, 3.0), (1, 1, 3))],
'ConvTranspose2d': [f32(np.arange(0.0, 9.0), (1, 1, 3, 3))],
'ConvTranspose2dOutputShape': [f32(np.arange(0.0, 9.0), (1, 1, 3, 3))],
'ConvTransposeBias2d': [f32(np.arange(0.0, 9.0), (1, 1, 3, 3))],
'ConvTransposeBias2dBatched': [f32(np.arange(0.0, 18.0), (2, 1, 3, 3))],
'ConvWithAsymmetricPadding': [f32(np.arange(0.0, 35.0), (1, 1, 7, 5))],
Expand Down Expand Up @@ -5631,6 +5662,29 @@ def _mean_reference(model, feeds):
return [np.mean(np.broadcast_arrays(*feeds.values()), axis=0, dtype=np.float32)]


def _convtranspose_outputshape_reference(model, feeds):
"""The ReferenceEvaluator crashes if output_shape is set but pads is not.
We temporarily add the correct inferred pads to evaluate it, then remove
them so the saved model strictly tests SOFIE's inference logic."""
from onnx import helper

for node in model.graph.node:
if node.op_type == "ConvTranspose" and "pads" not in [a.name for a in node.attribute]:
node.attribute.extend([helper.make_attribute("pads", [1, 1, 0, 0])])

from onnx.reference import ReferenceEvaluator

outputs = ReferenceEvaluator(model).run(None, feeds)

for node in model.graph.node:
if node.op_type == "ConvTranspose":
for i, attr in enumerate(node.attribute):
if attr.name == "pads":
del node.attribute[i]
break
return outputs


# Models whose expected outputs the ReferenceEvaluator cannot compute.
EXPECTED_OVERRIDES = {
"GRUBidirectional": _recurrent_reference,
Expand All @@ -5641,6 +5695,7 @@ def _mean_reference(model, feeds):
"RNNSequenceBatchwise": _recurrent_reference,
"MaxPool2d_AsymPad": _maxpool2d_reference,
"MeanMultidirectionalBroadcast": _mean_reference,
"ConvTranspose2dOutputShape": _convtranspose_outputshape_reference,
}


Expand Down
Loading