fix: no resize when training with images

Bycob · mergify[bot] · commit e84c6161aa75 · 2023-05-15T07:13:08.000Z
diff --git a/src/backends/torch/torchdataaug.cc b/src/backends/torch/torchdataaug.cc
@@ -395,17 +395,24 @@ namespace dd
 
     if (sample)
       {
+        int img_width = src.cols;
+        int img_height = src.rows;
+        std::uniform_int_distribution<int> uniform_int_crop_x(
+            0, img_width - cp._crop_size);
+        std::uniform_int_distribution<int> uniform_int_crop_y(
+            0, img_height - cp._crop_size);
+
 #pragma omp critical
         {
           if (test)
             {
-              crop_x = cp._uniform_int_crop_x(_rnd_test_gen);
-              crop_y = cp._uniform_int_crop_y(_rnd_test_gen);
+              crop_x = uniform_int_crop_x(_rnd_test_gen);
+              crop_y = uniform_int_crop_y(_rnd_test_gen);
             }
           else
             {
-              crop_x = cp._uniform_int_crop_x(_rnd_gen);
-              crop_y = cp._uniform_int_crop_y(_rnd_gen);
+              crop_x = uniform_int_crop_x(_rnd_gen);
+              crop_y = uniform_int_crop_y(_rnd_gen);
             }
         }
       }
@@ -464,20 +471,22 @@ namespace dd
 
 #pragma omp critical
     {
+      int img_width = src.cols;
+      int img_height = src.rows;
       // get shape and area to erase
       int w = 0, h = 0, rect_x = 0, rect_y = 0;
       if (cp._w == 0 && cp._h == 0)
         {
-          float s = cp._uniform_real_cutout_s(_rnd_gen) * cp._img_width
-                    * cp._img_height;                    // area
+          float s = cp._uniform_real_cutout_s(_rnd_gen) * img_width
+                    * img_height;                        // area
           float r = cp._uniform_real_cutout_r(_rnd_gen); // aspect ratio
 
-          w = std::min(cp._img_width,
+          w = std::min(img_width,
                        static_cast<int>(std::floor(std::sqrt(s / r))));
-          h = std::min(cp._img_height,
+          h = std::min(img_height,
                        static_cast<int>(std::floor(std::sqrt(s * r))));
-          std::uniform_int_distribution<int> distx(0, cp._img_width - w);
-          std::uniform_int_distribution<int> disty(0, cp._img_height - h);
+          std::uniform_int_distribution<int> distx(0, img_width - w);
+          std::uniform_int_distribution<int> disty(0, img_height - h);
           rect_x = distx(_rnd_gen);
           rect_y = disty(_rnd_gen);
         }
diff --git a/src/backends/torch/torchdataaug.h b/src/backends/torch/torchdataaug.h
@@ -33,67 +33,34 @@
 
 namespace dd
 {
-  class ImgAugParams
+  class CropParams
   {
   public:
-    ImgAugParams() : _img_width(224), _img_height(224)
+    CropParams()
     {
     }
 
-    ImgAugParams(const int &img_width, const int &img_height)
-        : _img_width(img_width), _img_height(img_height)
+    CropParams(const int &crop_size) : _crop_size(crop_size)
     {
     }
 
-    ~ImgAugParams()
-    {
-    }
-
-    int _img_width;
-    int _img_height;
-  };
-
-  class CropParams : public ImgAugParams
-  {
-  public:
-    CropParams() : ImgAugParams()
-    {
-    }
-
-    CropParams(const int &crop_size, const int &img_width,
-               const int &img_height)
-        : ImgAugParams(img_width, img_height), _crop_size(crop_size)
-    {
-      if (_crop_size > 0)
-        {
-          _uniform_int_crop_x
-              = std::uniform_int_distribution<int>(0, _img_width - _crop_size);
-          _uniform_int_crop_y = std::uniform_int_distribution<int>(
-              0, _img_height - _crop_size);
-        }
-    }
-
     ~CropParams()
     {
     }
 
     // default params
     int _crop_size = -1;
-    std::uniform_int_distribution<int> _uniform_int_crop_x;
-    std::uniform_int_distribution<int> _uniform_int_crop_y;
     int _test_crop_samples = 1; /**< number of sampled crops (at test time). */
   };
 
-  class CutoutParams : public ImgAugParams
+  class CutoutParams
   {
   public:
-    CutoutParams() : ImgAugParams()
+    CutoutParams()
     {
     }
 
-    CutoutParams(const float &prob, const int &img_width,
-                 const int &img_height)
-        : ImgAugParams(img_width, img_height), _prob(prob)
+    CutoutParams(const float &prob) : _prob(prob)
     {
       _uniform_real_cutout_s
           = std::uniform_real_distribution<float>(_cutout_sl, _cutout_sh);
@@ -287,11 +254,6 @@ namespace dd
           _uniform_real_1(0.0, 1.0), _bernouilli(0.5),
           _uniform_int_rotate(0, 3)
     {
-      if (_crop_params._crop_size > 0)
-        {
-          _cutout_params._img_width = _crop_params._crop_size;
-          _cutout_params._img_height = _crop_params._crop_size;
-        }
       reset_rnd_test_gen();
     }
 
diff --git a/src/backends/torch/torchdataset.cc b/src/backends/torch/torchdataset.cc
@@ -227,7 +227,7 @@ namespace dd
         torch::load(targett, targetstream);
       }
 
-    if (bgr.cols != width || bgr.rows != height)
+    if (width > 0 && height > 0 && (bgr.cols != width || bgr.rows != height))
       {
         cv::resize(bgr, bgr, cv::Size(width, height), 0, 0, cv::INTER_CUBIC);
 
@@ -860,10 +860,13 @@ namespace dd
 
     std::ifstream infile(bboxfname);
     std::string line;
-    double wfactor = static_cast<double>(inputc->_width)
-                     / static_cast<double>(orig_width);
-    double hfactor = static_cast<double>(inputc->_height)
-                     / static_cast<double>(orig_height);
+    double wfactor = inputc->_width > 0 ? static_cast<double>(inputc->_width)
+                                              / static_cast<double>(orig_width)
+                                        : 1;
+    double hfactor = inputc->_height > 0
+                         ? static_cast<double>(inputc->_height)
+                               / static_cast<double>(orig_height)
+                         : 1;
 
     while (std::getline(infile, line))
       {
diff --git a/src/backends/torch/torchlib.cc b/src/backends/torch/torchlib.cc
@@ -701,8 +701,7 @@ namespace dd
             if (ad_mllib.has("crop_size"))
               {
                 int crop_size = ad_mllib.get("crop_size").get<int>();
-                crop_params
-                    = CropParams(crop_size, inputc.width(), inputc.height());
+                crop_params = CropParams(crop_size);
                 if (ad_mllib.has("test_crop_samples"))
                   crop_params._test_crop_samples
                       = ad_mllib.get("test_crop_samples").get<int>();
@@ -712,8 +711,7 @@ namespace dd
             if (ad_mllib.has("cutout"))
               {
                 float cutout = ad_mllib.get("cutout").get<double>();
-                cutout_params
-                    = CutoutParams(cutout, inputc.width(), inputc.height());
+                cutout_params = CutoutParams(cutout);
                 this->_logger->info("cutout: {}", cutout);
               }
             GeometryParams geometry_params;
@@ -1640,6 +1638,10 @@ namespace dd
                         throw MLLibInternalException(
                             "Couldn't find original image size for " + uri);
                       }
+                    int src_width
+                        = inputc.width() > 0 ? inputc.width() : cols - 1;
+                    int src_height
+                        = inputc.height() > 0 ? inputc.height() : rows - 1;
 
                     APIData results_ad;
                     std::vector<double> probs;
@@ -1676,10 +1678,10 @@ namespace dd
                             this->_mlmodel.get_hcorresp(labels_acc[j]));
 
                         double bbox[] = {
-                          bboxes_acc[j][0] / inputc.width() * (cols - 1),
-                          bboxes_acc[j][1] / inputc.height() * (rows - 1),
-                          bboxes_acc[j][2] / inputc.width() * (cols - 1),
-                          bboxes_acc[j][3] / inputc.height() * (rows - 1),
+                          bboxes_acc[j][0] / src_width * (cols - 1),
+                          bboxes_acc[j][1] / src_height * (rows - 1),
+                          bboxes_acc[j][2] / src_width * (cols - 1),
+                          bboxes_acc[j][3] / src_height * (rows - 1),
                         };
 
                         // clamp bbox
diff --git a/src/imginputfileconn.h b/src/imginputfileconn.h
@@ -100,9 +100,9 @@ namespace dd
         {
           if (_scaled)
             scale(src, dst);
-          else if (_width == 0 || _height == 0)
+          else if (_width < 0 || _height < 0)
             {
-              if (_width == 0 && _height == 0)
+              if (_width < 0 && _height < 0)
                 {
                   // Do nothing and keep native resolution. May cause issues if
                   // batched images are different resolutions
@@ -199,9 +199,9 @@ namespace dd
         {
           if (_scaled)
             scale_cuda(src, dst);
-          else if (_width == 0 || _height == 0)
+          else if (_width < 0 || _height < 0)
             {
-              if (_width == 0 && _height == 0)
+              if (_width < 0 && _height < 0)
                 {
                   // Do nothing and keep native resolution. May cause issues if
                   // batched images are different resolutions
diff --git a/tests/ut-torchapi.cc b/tests/ut-torchapi.cc
@@ -394,6 +394,46 @@ TEST(torchapi, service_predict_object_detection)
   ASSERT_EQ(preds_best.Size(), 3);
 }
 
+TEST(torchapi, service_predict_object_detection_any_size)
+{
+  JsonAPI japi;
+  std::string sname = "detectserv";
+  std::string jstr
+      = "{\"mllib\":\"torch\",\"description\":\"fasterrcnn\",\"type\":"
+        "\"supervised\",\"model\":{\"repository\":\""
+        + detect_repo
+        + "\"},\"parameters\":{\"input\":{\"connector\":\"image\",\"height\":"
+          "-1,\"width\":-1,\"rgb\":true,\"scale\":0.0039},\"mllib\":{"
+          "\"template\":\"fasterrcnn\"}}}";
+
+  std::string joutstr = japi.jrender(japi.service_create(sname, jstr));
+  ASSERT_EQ(created_str, joutstr);
+  std::string jpredictstr
+      = "{\"service\":\"detectserv\",\"parameters\":{"
+        "\"input\":{\"height\":-1,"
+        "\"width\":-1},\"output\":{\"bbox\":true, "
+        "\"best_bbox\":1,\"confidence_threshold\":0.8}},\"data\":[\""
+        + detect_train_repo_fasterrcnn + "/imgs/000550-L.jpg\"]}";
+
+  joutstr = japi.jrender(japi.service_predict(jpredictstr));
+  JDoc jd;
+  std::cout << "joutstr=" << joutstr << std::endl;
+  jd.Parse<rapidjson::kParseNanAndInfFlag>(joutstr.c_str());
+  ASSERT_TRUE(!jd.HasParseError());
+  ASSERT_EQ(200, jd["status"]["code"]);
+  ASSERT_TRUE(jd["body"]["predictions"].IsArray());
+
+  auto &preds = jd["body"]["predictions"][0]["classes"];
+  std::string cl1 = preds[0]["cat"].GetString();
+  ASSERT_TRUE(cl1 == "car");
+  ASSERT_TRUE(preds[0]["prob"].GetDouble() > 0.9);
+  auto &bbox = preds[0]["bbox"];
+  ASSERT_NEAR(bbox["xmin"].GetDouble(), 258.0, 5.0);
+  ASSERT_NEAR(bbox["ymin"].GetDouble(), 333.0, 5.0);
+  ASSERT_NEAR(bbox["xmax"].GetDouble(), 401.0, 5.0);
+  ASSERT_NEAR(bbox["ymax"].GetDouble(), 448.0, 5.0);
+}
+
 TEST(torchapi, service_predict_segmentation)
 {
   JsonAPI japi;
@@ -2748,6 +2788,107 @@ TEST(torchapi, service_train_object_detection_translation)
   fileops::remove_dir(detect_train_repo_yolox + "test_0.lmdb");
 }
 
+TEST(torchapi, service_train_object_detection_yolox_any_size)
+{
+  // Test with arbitrary image size: width = -1, height = -1
+  setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8", true);
+  torch::manual_seed(torch_seed);
+  at::globalContext().setDeterministicCuDNN(true);
+
+  JsonAPI japi;
+  std::string sname = "detectserv";
+  std::string jstr
+      = "{\"mllib\":\"torch\",\"description\":\"yolox\",\"type\":"
+        "\"supervised\",\"model\":{\"repository\":\""
+        + detect_train_repo_yolox
+        + "\"},\"parameters\":{\"input\":{\"connector\":\"image\",\"height\":"
+          "-1,\"width\":-1,\"rgb\":true,\"bbox\":true,\"db\":true},"
+          "\"mllib\":{\"template\":\"yolox\",\"gpu\":true,"
+          "\"nclasses\":2}}}";
+
+  std::string joutstr = japi.jrender(japi.service_create(sname, jstr));
+  ASSERT_EQ(created_str, joutstr);
+
+  // Train
+  std::string jtrainstr
+      = "{\"service\":\"detectserv\",\"async\":false,\"parameters\":{"
+        "\"mllib\":{\"solver\":{\"iterations\":3"
+        + std::string("")
+        //+ iterations_detection + ",\"base_lr\":" + torch_lr
+        + ",\"iter_size\":2,\"solver_"
+          "type\":\"ADAM\",\"test_interval\":200},\"net\":{\"batch_size\":2,"
+          "\"test_batch_size\":1,\"reg_weight\":0.5},\"resume\":false,"
+          "\"mirror\":true,\"rotate\":true,\"crop_size\":512,"
+          "\"test_crop_samples\":10,"
+          "\"cutout\":0.1,\"geometry\":{\"prob\":0.1,\"persp_horizontal\":"
+          "true,\"persp_vertical\":true,\"zoom_in\":true,\"zoom_out\":true,"
+          "\"pad_mode\":\"constant\"},\"noise\":{\"prob\":0.01},\"distort\":{"
+          "\"prob\":0.01}},\"input\":{\"seed\":12347,\"db\":true,"
+          "\"shuffle\":true},\"output\":{\"measure\":[\"map-05\",\"map-50\","
+          "\"map-90\"]}},\"data\":[\""
+        + fasterrcnn_train_data + "\",\"" + fasterrcnn_test_data + "\"]}";
+
+  joutstr = japi.jrender(japi.service_train(jtrainstr));
+  JDoc jd;
+  std::cout << "joutstr=" << joutstr << std::endl;
+  jd.Parse<rapidjson::kParseNanAndInfFlag>(joutstr.c_str());
+  ASSERT_TRUE(!jd.HasParseError());
+  ASSERT_EQ(201, jd["status"]["code"]);
+
+  // ASSERT_EQ(jd["body"]["measure"]["iteration"], 200) << "iterations";
+  ASSERT_TRUE(jd["body"]["measure"]["map"].GetDouble() <= 1.0) << "map";
+  ASSERT_TRUE(jd["body"]["measure"]["map-05"].GetDouble() <= 1.0) << "map-05";
+  ASSERT_TRUE(jd["body"]["measure"]["map-50"].GetDouble() <= 1.0) << "map-50";
+  ASSERT_TRUE(jd["body"]["measure"]["map-90"].GetDouble() <= 1.0) << "map-90";
+  // ASSERT_TRUE(jd["body"]["measure"]["map"].GetDouble() > 0.0) << "map";
+
+  // check metrics
+  auto &meas = jd["body"]["measure"];
+  ASSERT_TRUE(meas.HasMember("iou_loss"));
+  ASSERT_TRUE(meas.HasMember("conf_loss"));
+  ASSERT_TRUE(meas.HasMember("cls_loss"));
+  ASSERT_TRUE(meas.HasMember("l1_loss"));
+  ASSERT_TRUE(meas.HasMember("train_loss"));
+  ASSERT_TRUE(
+      std::abs(meas["train_loss"].GetDouble()
+               - (meas["iou_loss"].GetDouble() * 0.5
+                  + meas["cls_loss"].GetDouble() + meas["l1_loss"].GetDouble()
+                  + meas["conf_loss"].GetDouble()))
+      < 0.0001);
+
+  // check that predict works fine
+  std::string jpredictstr = "{\"service\":\"detectserv\",\"parameters\":{"
+                            "\"input\":{\"height\":-1,"
+                            "\"width\":-1},\"output\":{\"bbox\":true, "
+                            "\"confidence_threshold\":0.8}},\"data\":[\""
+                            + detect_train_repo_fasterrcnn
+                            + "/imgs/000550-L.jpg\"]}";
+  joutstr = japi.jrender(japi.service_predict(jpredictstr));
+  jd = JDoc();
+  std::cout << "joutstr=" << joutstr << std::endl;
+  jd.Parse<rapidjson::kParseNanAndInfFlag>(joutstr.c_str());
+  ASSERT_TRUE(!jd.HasParseError());
+  ASSERT_EQ(200, jd["status"]["code"]);
+
+  std::unordered_set<std::string> lfiles;
+  fileops::list_directory(detect_train_repo_yolox, true, false, false, lfiles);
+  for (std::string ff : lfiles)
+    {
+      if (ff.find("checkpoint") != std::string::npos
+          || ff.find("solver") != std::string::npos)
+        remove(ff.c_str());
+    }
+  ASSERT_TRUE(!fileops::file_exists(detect_train_repo_yolox + "checkpoint-"
+                                    + iterations_detection + ".ptw"));
+  ASSERT_TRUE(!fileops::file_exists(detect_train_repo_yolox + "checkpoint-"
+                                    + iterations_detection + ".pt"));
+
+  fileops::clear_directory(detect_train_repo_yolox + "train.lmdb");
+  fileops::clear_directory(detect_train_repo_yolox + "test_0.lmdb");
+  fileops::remove_dir(detect_train_repo_yolox + "train.lmdb");
+  fileops::remove_dir(detect_train_repo_yolox + "test_0.lmdb");
+}
+
 TEST(torchapi, service_train_images_native)
 {
   setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8", true);

Original file line number	Diff line number	Diff line change
`@@ -395,17 +395,24 @@ namespace dd`
`395`	`395`
`396`	`396`	`if (sample)`
`397`	`397`	`{`
	`398`	`+ int img_width = src.cols;`
	`399`	`+ int img_height = src.rows;`
	`400`	`+ std::uniform_int_distribution<int> uniform_int_crop_x(`
	`401`	`+ 0, img_width - cp._crop_size);`
	`402`	`+ std::uniform_int_distribution<int> uniform_int_crop_y(`
	`403`	`+ 0, img_height - cp._crop_size);`
	`404`	`+`
`398`	`405`	`#pragma omp critical`
`399`	`406`	`{`
`400`	`407`	`if (test)`
`401`	`408`	`{`
`402`		`- crop_x = cp._uniform_int_crop_x(_rnd_test_gen);`
`403`		`- crop_y = cp._uniform_int_crop_y(_rnd_test_gen);`
	`409`	`+ crop_x = uniform_int_crop_x(_rnd_test_gen);`
	`410`	`+ crop_y = uniform_int_crop_y(_rnd_test_gen);`
`404`	`411`	`}`
`405`	`412`	`else`
`406`	`413`	`{`
`407`		`- crop_x = cp._uniform_int_crop_x(_rnd_gen);`
`408`		`- crop_y = cp._uniform_int_crop_y(_rnd_gen);`
	`414`	`+ crop_x = uniform_int_crop_x(_rnd_gen);`
	`415`	`+ crop_y = uniform_int_crop_y(_rnd_gen);`
`409`	`416`	`}`
`410`	`417`	`}`
`411`	`418`	`}`
`@@ -464,20 +471,22 @@ namespace dd`
`464`	`471`
`465`	`472`	`#pragma omp critical`
`466`	`473`	`{`
	`474`	`+ int img_width = src.cols;`
	`475`	`+ int img_height = src.rows;`
`467`	`476`	`// get shape and area to erase`
`468`	`477`	`int w = 0, h = 0, rect_x = 0, rect_y = 0;`
`469`	`478`	`if (cp._w == 0 && cp._h == 0)`
`470`	`479`	`{`
`471`		`- float s = cp._uniform_real_cutout_s(_rnd_gen) * cp._img_width`
`472`		`- * cp._img_height; // area`
	`480`	`+ float s = cp._uniform_real_cutout_s(_rnd_gen) * img_width`
	`481`	`+ * img_height; // area`
`473`	`482`	`float r = cp._uniform_real_cutout_r(_rnd_gen); // aspect ratio`
`474`	`483`
`475`		`- w = std::min(cp._img_width,`
	`484`	`+ w = std::min(img_width,`
`476`	`485`	`static_cast<int>(std::floor(std::sqrt(s / r))));`
`477`		`- h = std::min(cp._img_height,`
	`486`	`+ h = std::min(img_height,`
`478`	`487`	`static_cast<int>(std::floor(std::sqrt(s * r))));`
`479`		`- std::uniform_int_distribution<int> distx(0, cp._img_width - w);`
`480`		`- std::uniform_int_distribution<int> disty(0, cp._img_height - h);`
	`488`	`+ std::uniform_int_distribution<int> distx(0, img_width - w);`
	`489`	`+ std::uniform_int_distribution<int> disty(0, img_height - h);`
`481`	`490`	`rect_x = distx(_rnd_gen);`
`482`	`491`	`rect_y = disty(_rnd_gen);`
`483`	`492`	`}`
Original file line number	Diff line number	Diff line change
`@@ -100,9 +100,9 @@ namespace dd`
`100`	`100`	`{`
`101`	`101`	`if (_scaled)`
`102`	`102`	`scale(src, dst);`
`103`		`- else if (_width == 0 \|\| _height == 0)`
	`103`	`+ else if (_width < 0 \|\| _height < 0)`
`104`	`104`	`{`
`105`		`- if (_width == 0 && _height == 0)`
	`105`	`+ if (_width < 0 && _height < 0)`
`106`	`106`	`{`
`107`	`107`	`// Do nothing and keep native resolution. May cause issues if`
`108`	`108`	`// batched images are different resolutions`
`@@ -199,9 +199,9 @@ namespace dd`
`199`	`199`	`{`
`200`	`200`	`if (_scaled)`
`201`	`201`	`scale_cuda(src, dst);`
`202`		`- else if (_width == 0 \|\| _height == 0)`
	`202`	`+ else if (_width < 0 \|\| _height < 0)`
`203`	`203`	`{`
`204`		`- if (_width == 0 && _height == 0)`
	`204`	`+ if (_width < 0 && _height < 0)`
`205`	`205`	`{`
`206`	`206`	`// Do nothing and keep native resolution. May cause issues if`
`207`	`207`	`// batched images are different resolutions`