[ci] [R-package] Fix memory leaks found by valgrind (#3443)

* fix int64 write error

* attempt

* [WIP] [ci] [R-package] Add CI job that runs valgrind tests

* update all-successful

* install

* executable

* fix redirect stuff

* Apply suggestions from code review

Co-authored-by: Guolin Ke <guolin.ke@outlook.com>

* more flags

* add mc to msvc proj

* fix memory leak in mc

* Update monotone_constraints.hpp

* Update r_package.yml

* remove R_INT64_PTR

* disable openmp

* Update gbdt_model_text.cpp

* Update gbdt_model_text.cpp

* Apply suggestions from code review

* try to free vector

* free more memories.

* Update src/boosting/gbdt_model_text.cpp

* fix using

* try the UNPROTECT(1);

* fix a const pointer

* fix Common

* reduce UNPROTECT

* remove UNPROTECT(1);

* fix null handle

* fix predictor

* use NULL after free

* fix a leaking in test

* try more fixes

* test the effect of tests

* throw exception in Fatal

* add test back

* Apply suggestions from code review

* commet some tests

* Apply suggestions from code review

* Apply suggestions from code review

* trying to comment out tests

* Update openmp_wrapper.h

* Apply suggestions from code review

* Update configure

* Update configure.ac

* trying to uncomment

* more comments

* more uncommenting

* more uncommenting

* fix comment

* more uncommenting

* uncomment fully-commented out stuff

* try uncommenting more dataset tests

* uncommenting more tests

* ok getting closer

* more uncommenting

* free dataset

* skipping a test, more uncommenting

* more skipping

* re-enable OpenMP

* allow on OpenMP thing

* move valgrind to comment-only job

* Apply suggestions from code review

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

* changes from code review

* Apply suggestions from code review

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

* linting

* issue comments too

* remove issue_comment

Co-authored-by: Guolin Ke <guolin.ke@outlook.com>
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.ci/test_r_package_valgrind.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+cd R-package/tests
+
+ALL_LOGS_FILE="out.log"
+VALGRIND_LOGS_FILE="valgrind-logs.log"
+
+RDvalgrind \
+  --no-readline \
+  --vanilla \
+  -d "valgrind --tool=memcheck --leak-check=full --track-origins=yes" \
+  -f testthat.R \
+  2>&1 > ${ALL_LOGS_FILE} || exit -1
+
+cat ${ALL_LOGS_FILE}
+
+cat ${ALL_LOGS_FILE} | grep -E "^\=" > ${VALGRIND_LOGS_FILE}
+
+bytes_definitely_lost=$(
+  cat ${VALGRIND_LOGS_FILE} \
+      | grep -E "definitely lost\: .*" \
+      | sed 's/^.*definitely lost\: \(.*\) bytes.*$/\1/' \
+      | tr -d ","
+)
+if [[ ${bytes_definitely_lost} -gt 0 ]]; then
+    echo "valgrind found ${bytes_definitely_lost} bytes definitely lost"
+    exit -1
+fi
+
+bytes_indirectly_lost=$(
+    cat ${VALGRIND_LOGS_FILE} \
+    | grep -E "indirectly lost\: .*" \
+    | sed 's/^.*indirectly lost\: \(.*\) bytes.*$/\1/' \
+    | tr -d ","
+)
+if [[ ${bytes_indirectly_lost} -gt 0 ]]; then
+    echo "valgrind found ${bytes_indirectly_lost} bytes indirectly lost"
+    exit -1
+fi
+
+# one error caused by a false positive between valgrind and openmp is allowed
+# ==1312== 352 bytes in 1 blocks are possibly lost in loss record 146 of 2,458
+# ==1312==    at 0x483DD99: calloc (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_memcheck-amd64-linux.so)
+# ==1312==    by 0x40149CA: allocate_dtv (dl-tls.c:286)
+# ==1312==    by 0x40149CA: _dl_allocate_tls (dl-tls.c:532)
+# ==1312==    by 0x5702322: allocate_stack (allocatestack.c:622)
+# ==1312==    by 0x5702322: pthread_create@@GLIBC_2.2.5 (pthread_create.c:660)
+# ==1312==    by 0x56D0DDA: ??? (in /usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0)
+# ==1312==    by 0x56C88E0: GOMP_parallel (in /usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0)
+# ==1312==    by 0x154351B8: LGBM_DatasetCreateFromCSC (c_api.cpp:1286)
+# ==1312==    by 0x1545789C: LGBM_DatasetCreateFromCSC_R (lightgbm_R.cpp:91)
+# ==1312==    by 0x4941E2F: R_doDotCall (dotcode.c:634)
+# ==1312==    by 0x494CCC6: do_dotcall (dotcode.c:1281)
+# ==1312==    by 0x499FB01: bcEval (eval.c:7078)
+# ==1312==    by 0x498B67F: Rf_eval (eval.c:727)
+# ==1312==    by 0x498E414: R_execClosure (eval.c:1895)
+bytes_possibly_lost=$(
+    cat ${VALGRIND_LOGS_FILE} \
+    | grep -E "possibly lost\: .*" \
+    | sed 's/^.*possibly lost\: \(.*\) bytes.*$/\1/' \
+    | tr -d ","
+)
+if [[ ${bytes_possibly_lost} -gt 352 ]]; then
+    echo "valgrind found ${bytes_possibly_lost} bytes possibly lost"
+    exit -1
+fi

.github/workflows/r_valgrind.yml

@@ -0,0 +1,28 @@
+name: R valgrind tests
+
+on:
+  pull_request_review_comment:
+    types: [created]
+
+jobs:
+  test-r-valgrind:
+    name: r-package (ubuntu-latest, R-devel, valgrind)
+    if: github.event.comment.body == '/gha run r-valgrind' && contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association)
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    container: wch1/r-debug
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 5
+          submodules: true
+      - name: Install packages
+        shell: bash
+        run: |
+          RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')"
+          sh build-cran-package.sh
+          RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1
+      - name: Run tests with valgrind
+        shell: bash
+        run: ./.ci/test_r_package_valgrind.sh

R-package/R/lgb.Predictor.R

@@ -30,7 +30,7 @@ Predictor <- R6::R6Class(
       params <- list(...)
       private$params <- lgb.params2str(params)
       # Create new lgb handle
-      handle <- 0.0
+      handle <- lgb.null.handle()
 
       # Check if handle is a character
       if (is.character(modelfile)) {

R-package/README.md

@@ -454,13 +454,17 @@ cd R-package/tests
 RDvalgrind \
     --no-readline \
     --vanilla \
-    -d valgrind \
+    -d "valgrind --tool=memcheck --leak-check=full --track-origins=yes" \
         -f testthat.R \
 2>&1 \
 | tee out.log \
 | cat
 ```
 
+These tests can also be triggered on any pull request by leaving a "Comment" review with the following comment:
+
+> /gha run r-valgrind
+
 External (Unofficial) Repositories
 ----------------------------------
 

R-package/src/R_object_helper.h

@@ -104,8 +104,6 @@ typedef union { VECTOR_SER s; double align; } SEXPREC_ALIGN;
 
 #define R_INT_PTR(x)   (reinterpret_cast<int*> DATAPTR(x))
 
-#define R_INT64_PTR(x) (reinterpret_cast<int64_t*> DATAPTR(x))
-
 #define R_REAL_PTR(x)  (reinterpret_cast<double*> DATAPTR(x))
 
 #define R_AS_INT(x)    (*(reinterpret_cast<int*> DATAPTR(x)))

R-package/src/lightgbm_R.cpp

@@ -506,7 +506,7 @@ LGBM_SE LGBM_BoosterGetNumPredict_R(LGBM_SE handle,
   R_API_BEGIN();
   int64_t len;
   CHECK_CALL(LGBM_BoosterGetNumPredict(R_GET_PTR(handle), R_AS_INT(data_idx), &len));
-  R_INT64_PTR(out)[0] = len;
+  R_INT_PTR(out)[0] = static_cast<int>(len);
   R_API_END();
 }
 
@@ -624,7 +624,7 @@ LGBM_SE LGBM_BoosterPredictForMat_R(LGBM_SE handle,
   int32_t nrow = R_AS_INT(num_row);
   int32_t ncol = R_AS_INT(num_col);
 
-  double* p_mat = R_REAL_PTR(data);
+  const double* p_mat = R_REAL_PTR(data);
   double* ptr_ret = R_REAL_PTR(out_result);
   int64_t out_len;
   CHECK_CALL(LGBM_BoosterPredictForMat(R_GET_PTR(handle),

R-package/tests/testthat/test_dataset.R

@@ -85,6 +85,8 @@ test_that("lgb.Dataset: Dataset should be able to construct from matrix and retu
     , ref_handle
   )
   expect_false(is.na(handle))
+  lgb.call("LGBM_DatasetFree_R", ret = NULL, handle)
+  handle <- NULL
 })
 
 test_that("lgb.Dataset$setinfo() should convert 'group' to integer", {

R-package/tests/testthat/test_lgb.Booster.R

@@ -439,6 +439,23 @@ test_that("Saving a model with different feature importance types works", {
             , "spore-print-color=green=1"
         )
     )
+})
+
+test_that("Saving a model with unknown importance type fails", {
+    testthat::skip("Skipping this test because it causes issues for valgrind")
+    set.seed(708L)
+    data(agaricus.train, package = "lightgbm")
+    train <- agaricus.train
+    bst <- lightgbm(
+        data = as.matrix(train$data)
+        , label = train$label
+        , num_leaves = 4L
+        , learning_rate = 1.0
+        , nrounds = 2L
+        , objective = "binary"
+        , save_name = tempfile(fileext = ".model")
+    )
+    expect_true(lgb.is.Booster(bst))
 
     UNSUPPORTED_IMPORTANCE <- 2L
     expect_error({
@@ -446,6 +463,7 @@ test_that("Saving a model with different feature importance types works", {
     }, "Unknown importance type")
 })
 
+
 .params_from_model_string <- function(model_str) {
     file_lines <- strsplit(model_str, "\n")[[1L]]
     start_indx <- which(grepl("^parameters\\:$", file_lines)) + 1L

R-package/tests/testthat/test_utils.R

@@ -67,6 +67,10 @@ test_that("lgb.last_error() throws an error if there are no errors", {
 })
 
 test_that("lgb.last_error() correctly returns errors from the C++ side", {
+    testthat::skip(paste0(
+        "Skipping this test because it causes valgrind to think "
+        , "there is a memory leak, and needs to be rethought"
+    ))
     data(agaricus.train, package = "lightgbm")
     train <- agaricus.train
     dvalid1 <- lgb.Dataset(

include/LightGBM/utils/log.h

@@ -123,10 +123,10 @@ class Log {
 #ifndef LGB_R_BUILD
     fprintf(stderr, "[LightGBM] [Fatal] %s\n", str_buf);
     fflush(stderr);
-    throw std::runtime_error(std::string(str_buf));
 #else
     Rf_error("[LightGBM] [Fatal] %s\n", str_buf);
 #endif
+    throw std::runtime_error(std::string(str_buf));
   }
 
  private:

src/boosting/gbdt_model_text.cpp

@@ -393,7 +393,7 @@ std::string GBDT::SaveModelToString(int start_iteration, int num_iteration, int
     ss << loaded_parameter_ << "\n";
     ss << "end of parameters" << '\n';
   }
-  return ss.str();
+  return std::move(ss.str());
 }
 
 bool GBDT::SaveModelToFile(int start_iteration, int num_iteration, int feature_importance_type, const char* filename) const {
@@ -618,7 +618,7 @@ std::vector<double> GBDT::FeatureImportance(int num_iteration, int importance_ty
   } else {
     Log::Fatal("Unknown importance type: only support split=0 and gain=1");
   }
-  return feature_importances;
+  return std::move(feature_importances);
 }
 
 }  // namespace LightGBM

src/treelearner/monotone_constraints.hpp

@@ -9,6 +9,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <limits>
+#include <memory>
 #include <utility>
 #include <vector>
 
@@ -37,6 +38,7 @@ struct FeatureConstraint {
 };
 
 struct ConstraintEntry {
+  virtual ~ConstraintEntry() {}
   virtual void Reset() = 0;
   virtual void UpdateMin(double new_min) = 0;
   virtual void UpdateMax(double new_max) = 0;
@@ -462,12 +464,12 @@ class BasicLeafConstraints : public LeafConstraintsBase {
  public:
   explicit BasicLeafConstraints(int num_leaves) : num_leaves_(num_leaves) {
     for (int i = 0; i < num_leaves; ++i) {
-      entries_.push_back(new BasicConstraintEntry());
+      entries_.emplace_back(new BasicConstraintEntry());
     }
   }
 
   void Reset() override {
-    for (auto entry : entries_) {
+    for (auto& entry : entries_) {
       entry->Reset();
     }
   }
@@ -484,7 +486,7 @@ class BasicLeafConstraints : public LeafConstraintsBase {
                           int8_t monotone_type, double right_output,
                           double left_output, int, const SplitInfo& ,
                           const std::vector<SplitInfo>&) override {
-    entries_[new_leaf] = entries_[leaf]->clone();
+    entries_[new_leaf].reset(entries_[leaf]->clone());
     if (is_numerical_split) {
       double mid = (left_output + right_output) / 2.0f;
       if (monotone_type < 0) {
@@ -498,7 +500,7 @@ class BasicLeafConstraints : public LeafConstraintsBase {
     return std::vector<int>();
   }
 
-  const ConstraintEntry* Get(int leaf_idx) override { return entries_[leaf_idx]; }
+  const ConstraintEntry* Get(int leaf_idx) override { return entries_[leaf_idx].get(); }
 
   FeatureConstraint* GetFeatureConstraint(int leaf_idx, int feature_index) final {
     return entries_[leaf_idx]->GetFeatureConstraint(feature_index);
@@ -506,7 +508,7 @@ class BasicLeafConstraints : public LeafConstraintsBase {
 
  protected:
   int num_leaves_;
-  std::vector<ConstraintEntry*> entries_;
+  std::vector<std::unique_ptr<ConstraintEntry>> entries_;
 };
 
 class IntermediateLeafConstraints : public BasicLeafConstraints {
@@ -541,7 +543,7 @@ class IntermediateLeafConstraints : public BasicLeafConstraints {
   void UpdateConstraintsWithOutputs(bool is_numerical_split, int leaf,
                                     int new_leaf, int8_t monotone_type,
                                     double right_output, double left_output) {
-    entries_[new_leaf] = entries_[leaf]->clone();
+    entries_[new_leaf].reset(entries_[leaf]->clone());
     if (is_numerical_split) {
       if (monotone_type < 0) {
         entries_[leaf]->UpdateMin(right_output);
@@ -857,7 +859,7 @@ class AdvancedLeafConstraints : public IntermediateLeafConstraints {
                           int num_features)
       : IntermediateLeafConstraints(config, num_leaves) {
     for (int i = 0; i < num_leaves; ++i) {
-      entries_[i] = new AdvancedConstraintEntry(num_features);
+      entries_[i].reset(new AdvancedConstraintEntry(num_features));
     }
   }
 

windows/LightGBM.vcxproj

@@ -287,6 +287,7 @@
     <ClInclude Include="..\src\treelearner\data_partition.hpp" />
     <ClInclude Include="..\src\treelearner\feature_histogram.hpp" />
     <ClInclude Include="..\src\treelearner\leaf_splits.hpp" />
+    <ClInclude Include="..\src\treelearner\monotone_constraints.hpp" />
     <ClInclude Include="..\src\treelearner\parallel_tree_learner.h" />
     <ClInclude Include="..\src\treelearner\serial_tree_learner.h" />
     <ClInclude Include="..\src\treelearner\split_info.hpp" />

windows/LightGBM.vcxproj.filters

@@ -222,6 +222,9 @@
     <ClInclude Include="..\include\LightGBM\utils\yamc\alternate_shared_mutex.hpp">
       <Filter>include\LightGBM\utils\yamc</Filter>
     </ClInclude>
+    <ClInclude Include="..\src\treelearner\monotone_constraints.hpp">
+      <Filter>src\treelearner</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\src\application\application.cpp">