From cb2b12a97940d413475f8e89354fe783f224dc48 Mon Sep 17 00:00:00 2001 From: Man Wang Date: Thu, 11 Oct 2018 15:16:34 -0700 Subject: [PATCH] [Importer] Support arg "order" in Caffe2 Conv, MaxPool and AveragePool ops. --- lib/Importer/Caffe2ModelLoader.cpp | 39 +++- .../avgpool_nhwc_predict_net.pbtxt | 23 ++ .../caffe2Models/avgpool_predict_net.pbtxt | 19 ++ .../caffe2Models/conv_nhwc_init_net.pbtxt | 32 +++ .../caffe2Models/conv_nhwc_predict_net.pbtxt | 29 +++ .../maxpool_nhwc_predict_net.pbtxt | 23 ++ .../caffe2Models/maxpool_predict_net.pbtxt | 19 ++ tests/unittests/caffe2ImporterTest.cpp | 201 ++++++++++++++++++ 8 files changed, 374 insertions(+), 11 deletions(-) create mode 100644 tests/models/caffe2Models/avgpool_nhwc_predict_net.pbtxt create mode 100644 tests/models/caffe2Models/avgpool_predict_net.pbtxt create mode 100644 tests/models/caffe2Models/conv_nhwc_init_net.pbtxt create mode 100644 tests/models/caffe2Models/conv_nhwc_predict_net.pbtxt create mode 100644 tests/models/caffe2Models/maxpool_nhwc_predict_net.pbtxt create mode 100644 tests/models/caffe2Models/maxpool_predict_net.pbtxt diff --git a/lib/Importer/Caffe2ModelLoader.cpp b/lib/Importer/Caffe2ModelLoader.cpp index fa459a49f5..8b3bc7abb8 100644 --- a/lib/Importer/Caffe2ModelLoader.cpp +++ b/lib/Importer/Caffe2ModelLoader.cpp @@ -153,6 +153,7 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) { std::vector pads = getPads(dict); std::vector kernels = getSizeHW(dict, "kernel", 0); unsigned_t group = dict.count("group") ? loadInt(dict["group"]) : 1; + std::string order = dict.count("order") ? loadStr(dict["order"]) : "NCHW"; auto in = getNodeValueOrCreateVariableByName(op.input(0)); Tensor *w = getTensorByName(op.input(1)); @@ -186,23 +187,30 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) { } auto *bias = G_.getParent()->createConstant("conv.bias", biasTensor); - // Caffe passes the input as NCHW, and we expect the input to be NHWC. - auto *tr = G_.createTranspose(opName, in, NCHW2NHWC); + // We expect the input to be NHWC. + Node *tr; + if (order == "NCHW") { + tr = G_.createTranspose(opName, in, NCHW2NHWC); + } else { + tr = in; + } // Calculate the size and allocate the output buffer. - ShapeNHWC idim = ShapeNHWC(tr->getResult().dims()); + ShapeNHWC idim = ShapeNHWC(tr->getType(0)->dims()); auto outSz = calculateConvPoolOutputDims(idim.h, idim.w, kernels, strides, pads); std::array outDims = { {idim.n, outSz.first, outSz.second, depth}}; auto outTy = G_.getParent()->uniqueType(ElemKind::FloatTy, outDims); - auto *node = G_.createConv(opName, tr, filter, bias, outTy, kernels, + Node *node = G_.createConv(opName, tr, filter, bias, outTy, kernels, strides, pads, group); - // Transpose the output back. - auto *N = G_.createTranspose(opName, node, NHWC2NCHW); - addNodeAsOutput(op, N); + if (order == "NCHW") { + // Transpose the output back. + node = G_.createTranspose(opName, node, NHWC2NCHW); + } + addNodeAsOutput(op, node); return; } @@ -212,8 +220,14 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) { std::vector strides = getSizeHW(dict, "stride", 1); std::vector kernels = getSizeHW(dict, "kernel", 0); std::vector pads = getPads(dict); - - auto *tr = G_.createTranspose(opName, in, NCHW2NHWC); + std::string order = dict.count("order") ? loadStr(dict["order"]) : "NCHW"; + // We expect the input to be NHWC. + Node *tr; + if (order == "NCHW") { + tr = G_.createTranspose(opName, in, NCHW2NHWC); + } else { + tr = in; + } // If 'global_pooling' is set then the operation will pool over the size of // the input by doing: kernels = {height, width}. @@ -229,8 +243,11 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) { } else { node = G_.createAvgPool(opName, tr, kernels, strides, pads); } - auto *N = G_.createTranspose(opName, node, NHWC2NCHW); - addNodeAsOutput(op, N); + if (order == "NCHW") { + // Transpose the output back. + node = G_.createTranspose(opName, node, NHWC2NCHW); + } + addNodeAsOutput(op, node); return; } diff --git a/tests/models/caffe2Models/avgpool_nhwc_predict_net.pbtxt b/tests/models/caffe2Models/avgpool_nhwc_predict_net.pbtxt new file mode 100644 index 0000000000..c7f1d4d3fe --- /dev/null +++ b/tests/models/caffe2Models/avgpool_nhwc_predict_net.pbtxt @@ -0,0 +1,23 @@ +name: "averagePool_test" +op { + input: "inputs" + output: "output" + type: "AveragePool" + arg { + name: "order" + s: "NHWC" + } + arg { + name: "kernel" + i: 2 + } + arg { + name: "stride" + i: 1 + } + arg { + name: "pad" + i: 1 + } +} +external_output: "output" diff --git a/tests/models/caffe2Models/avgpool_predict_net.pbtxt b/tests/models/caffe2Models/avgpool_predict_net.pbtxt new file mode 100644 index 0000000000..c61ba2f062 --- /dev/null +++ b/tests/models/caffe2Models/avgpool_predict_net.pbtxt @@ -0,0 +1,19 @@ +name: "averagePool_test" +op { + input: "inputs" + output: "output" + type: "AveragePool" + arg { + name: "kernel" + i: 2 + } + arg { + name: "stride" + i: 1 + } + arg { + name: "pad" + i: 1 + } +} +external_output: "output" diff --git a/tests/models/caffe2Models/conv_nhwc_init_net.pbtxt b/tests/models/caffe2Models/conv_nhwc_init_net.pbtxt new file mode 100644 index 0000000000..2ff77c2903 --- /dev/null +++ b/tests/models/caffe2Models/conv_nhwc_init_net.pbtxt @@ -0,0 +1,32 @@ +name: "init" +op { + output: "conv_w" + type: "GivenTensorFill" + arg { + name: "shape" + ints: 1 + ints: 1 + ints: 2 + ints: 2 + } + arg { + name: "values" + floats: 1.0 + floats: 1.0 + floats: 1.0 + floats: 1.0 + } +} +op { + output: "conv_b" + type: "GivenTensorFill" + arg { + name: "shape" + ints: 1 + } + arg { + name: "values" + floats: 2.0 + } +} + diff --git a/tests/models/caffe2Models/conv_nhwc_predict_net.pbtxt b/tests/models/caffe2Models/conv_nhwc_predict_net.pbtxt new file mode 100644 index 0000000000..8d3e86790e --- /dev/null +++ b/tests/models/caffe2Models/conv_nhwc_predict_net.pbtxt @@ -0,0 +1,29 @@ +name: "conv_test" +op { + input: "inputs" + input: "conv_w" + input: "conv_b" + output: "conv_out" + type: "Conv" + arg { + name: "order" + s: "NHWC" + } + arg { + name: "kernel" + i: 2 + } + arg { + name: "stride" + i: 1 + } + arg { + name: "group" + i: 1 + } + arg { + name: "pad" + i: 1 + } +} +external_output: "conv_out" diff --git a/tests/models/caffe2Models/maxpool_nhwc_predict_net.pbtxt b/tests/models/caffe2Models/maxpool_nhwc_predict_net.pbtxt new file mode 100644 index 0000000000..df511c66d9 --- /dev/null +++ b/tests/models/caffe2Models/maxpool_nhwc_predict_net.pbtxt @@ -0,0 +1,23 @@ +name: "maxpool_test" +op { + input: "inputs" + output: "output" + type: "MaxPool" + arg { + name: "order" + s: "NHWC" + } + arg { + name: "kernel" + i: 2 + } + arg { + name: "stride" + i: 1 + } + arg { + name: "pad" + i: 1 + } +} +external_output: "output" diff --git a/tests/models/caffe2Models/maxpool_predict_net.pbtxt b/tests/models/caffe2Models/maxpool_predict_net.pbtxt new file mode 100644 index 0000000000..2ea2c82700 --- /dev/null +++ b/tests/models/caffe2Models/maxpool_predict_net.pbtxt @@ -0,0 +1,19 @@ +name: "maxpool_test" +op { + input: "inputs" + output: "output" + type: "MaxPool" + arg { + name: "kernel" + i: 2 + } + arg { + name: "stride" + i: 1 + } + arg { + name: "pad" + i: 1 + } +} +external_output: "output" diff --git a/tests/unittests/caffe2ImporterTest.cpp b/tests/unittests/caffe2ImporterTest.cpp index 037f828f98..2e39b285ac 100644 --- a/tests/unittests/caffe2ImporterTest.cpp +++ b/tests/unittests/caffe2ImporterTest.cpp @@ -61,6 +61,207 @@ TEST(caffe2, importConv) { EXPECT_FLOAT_EQ(result.raw(i), expectedValues[i]); } +/// Test loading conv op from a Caffe2 model. +/// The input is N*H*W*C (1*3*3*1), the kernel is 2, +/// stride is 1, pad is 1, group is 1. +TEST(caffe2, convNHWC) { + ExecutionEngine EE{BackendKind::Interpreter}; + auto &mod = EE.getModule(); + Function *F = mod.createFunction("main"); + + std::string NetDescFilename( + "tests/models/caffe2Models/conv_nhwc_predict_net.pbtxt"); + std::string NetWeightFilename( + "tests/models/caffe2Models/conv_nhwc_init_net.pbtxt"); + + Placeholder *output; + Context ctx; + + Tensor inputs(ElemKind::FloatTy, {1, 3, 3, 1}); + + // Destroy the loader after the graph is loaded since the following execution + // will not depend on anyting from the loader. + { + Caffe2ModelLoader caffe2LD(NetDescFilename, NetWeightFilename, {"inputs"}, + {&inputs.getType()}, *F); + output = caffe2LD.getSingleOutput(); + } + + // High level check on the content of the graph. We have 1 conv and 1 save. + EXPECT_EQ(F->getNodes().size(), 2); + auto *saveNode = getSaveNodeFromDest(output); + auto *convNode = + llvm::dyn_cast(saveNode->getInput().getNode()); + ASSERT_TRUE(convNode); + + // We have 2 placeholders: 1 input and 1 output. + EXPECT_EQ(mod.getPlaceholders().size(), 2); + // We have 2 constants: Weights and bias. + EXPECT_EQ(mod.getConstants().size(), 2); +} + +/// Test loading MaxPool with NHWC order input. +TEST(caffe2, maxPoolNHWC) { + ExecutionEngine EE{BackendKind::Interpreter}; + auto &mod = EE.getModule(); + Function *F = mod.createFunction("main"); + + std::string NetDescFilename( + "tests/models/caffe2Models/maxpool_nhwc_predict_net.pbtxt"); + std::string NetWeightFilename( + "tests/models/caffe2Models/empty_init_net.pbtxt"); + + Placeholder *output; + Context ctx; + + Tensor inputs(ElemKind::FloatTy, {1, 3, 3, 1}); + + // Destroy the loader after the graph is loaded since the following execution + // will not depend on anyting from the loader. + { + Caffe2ModelLoader caffe2LD(NetDescFilename, NetWeightFilename, {"inputs"}, + {&inputs.getType()}, *F); + output = caffe2LD.getSingleOutput(); + } + + // High level check on the content of the graph. We have 1 maxpool and 1 save. + EXPECT_EQ(F->getNodes().size(), 2); + auto *saveNode = getSaveNodeFromDest(output); + auto *maxPoolNode = + llvm::dyn_cast(saveNode->getInput().getNode()); + ASSERT_TRUE(maxPoolNode); + + // We have 2 placeholders: 1 input and 1 output. + EXPECT_EQ(mod.getPlaceholders().size(), 2); + // We have 0 constants. + EXPECT_EQ(mod.getConstants().size(), 0); +} + +/// Test loading MaxPool with default NCHW order input. +TEST(caffe2, maxPool) { + ExecutionEngine EE{BackendKind::Interpreter}; + auto &mod = EE.getModule(); + Function *F = mod.createFunction("main"); + + std::string NetDescFilename( + "tests/models/caffe2Models/maxpool_predict_net.pbtxt"); + std::string NetWeightFilename( + "tests/models/caffe2Models/empty_init_net.pbtxt"); + + Placeholder *output; + Context ctx; + + Tensor inputs(ElemKind::FloatTy, {1, 3, 3, 1}); + + // Destroy the loader after the graph is loaded since the following execution + // will not depend on anyting from the loader. + { + Caffe2ModelLoader caffe2LD(NetDescFilename, NetWeightFilename, {"inputs"}, + {&inputs.getType()}, *F); + output = caffe2LD.getSingleOutput(); + } + + // High level check on the content of the graph. We have 1 maxpool, 1 save + // and 2 transpose. + EXPECT_EQ(F->getNodes().size(), 4); + auto *saveNode = getSaveNodeFromDest(output); + auto *transNode1 = + llvm::dyn_cast(saveNode->getInput().getNode()); + ASSERT_TRUE(transNode1); + auto *maxPoolNode = + llvm::dyn_cast(transNode1->getInput().getNode()); + ASSERT_TRUE(maxPoolNode); + auto *transNode2 = + llvm::dyn_cast(maxPoolNode->getInput().getNode()); + ASSERT_TRUE(transNode2); + + // We have 2 placeholders: 1 input and 1 output. + EXPECT_EQ(mod.getPlaceholders().size(), 2); + // We have 0 constants. + EXPECT_EQ(mod.getConstants().size(), 0); +} + +/// Test loading AvgPool with NHWC order input. +TEST(caffe2, avgPoolNHWC) { + ExecutionEngine EE{BackendKind::Interpreter}; + auto &mod = EE.getModule(); + Function *F = mod.createFunction("main"); + + std::string NetDescFilename( + "tests/models/caffe2Models/avgpool_nhwc_predict_net.pbtxt"); + std::string NetWeightFilename( + "tests/models/caffe2Models/empty_init_net.pbtxt"); + + Placeholder *output; + Context ctx; + + Tensor inputs(ElemKind::FloatTy, {1, 3, 3, 1}); + + // Destroy the loader after the graph is loaded since the following execution + // will not depend on anyting from the loader. + { + Caffe2ModelLoader caffe2LD(NetDescFilename, NetWeightFilename, {"inputs"}, + {&inputs.getType()}, *F); + output = caffe2LD.getSingleOutput(); + } + + // High level check on the content of the graph. We have 1 maxpool and 1 save. + EXPECT_EQ(F->getNodes().size(), 2); + auto *saveNode = getSaveNodeFromDest(output); + auto *avgPoolNode = + llvm::dyn_cast(saveNode->getInput().getNode()); + ASSERT_TRUE(avgPoolNode); + + // We have 2 placeholders: 1 input and 1 output. + EXPECT_EQ(mod.getPlaceholders().size(), 2); + // We have 0 constants. + EXPECT_EQ(mod.getConstants().size(), 0); +} + +/// Test loading AveragePool with default NCHW order input. +TEST(caffe2, avgPool) { + ExecutionEngine EE{BackendKind::Interpreter}; + auto &mod = EE.getModule(); + Function *F = mod.createFunction("main"); + + std::string NetDescFilename( + "tests/models/caffe2Models/avgpool_predict_net.pbtxt"); + std::string NetWeightFilename( + "tests/models/caffe2Models/empty_init_net.pbtxt"); + + Placeholder *output; + Context ctx; + + Tensor inputs(ElemKind::FloatTy, {1, 3, 3, 1}); + + // Destroy the loader after the graph is loaded since the following execution + // will not depend on anyting from the loader. + { + Caffe2ModelLoader caffe2LD(NetDescFilename, NetWeightFilename, {"inputs"}, + {&inputs.getType()}, *F); + output = caffe2LD.getSingleOutput(); + } + + // High level check on the content of the graph. We have 1 maxpool, 1 save + // and 2 transpose. + EXPECT_EQ(F->getNodes().size(), 4); + auto *saveNode = getSaveNodeFromDest(output); + auto *transNode1 = + llvm::dyn_cast(saveNode->getInput().getNode()); + ASSERT_TRUE(transNode1); + auto *avgPoolNode = + llvm::dyn_cast(transNode1->getInput().getNode()); + ASSERT_TRUE(avgPoolNode); + auto *transNode2 = + llvm::dyn_cast(avgPoolNode->getInput().getNode()); + ASSERT_TRUE(transNode2); + + // We have 2 placeholders: 1 input and 1 output. + EXPECT_EQ(mod.getPlaceholders().size(), 2); + // We have 0 constants. + EXPECT_EQ(mod.getConstants().size(), 0); +} + /// Test loading a concat node with add_axis. /// Concat nodes with add_axis have a different semantic /// than the plain glow concat.