Skip to content

Commit

Permalink
Applying feedback from PR #1
Browse files Browse the repository at this point in the history
  • Loading branch information
andre.cruz committed May 2, 2022
1 parent 7f180ed commit 356664b
Show file tree
Hide file tree
Showing 14 changed files with 191 additions and 309 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -455,3 +455,6 @@ dask-worker-space/
*.pub
*.rdp
*_rsa

# Others
src/main.*.cpp
15 changes: 3 additions & 12 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ if(${MM_MALLOC})
endif()

if(UNIX OR MINGW OR CYGWIN)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -lstdc++fs -pthread -Wextra -Wall -Wno-ignored-attributes -Wno-unknown-pragmas -Wno-return-type ")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -pthread -Wextra -Wall -Wno-ignored-attributes -Wno-unknown-pragmas -Wno-return-type ")
if(USE_DEBUG)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
else()
Expand Down Expand Up @@ -338,21 +338,13 @@ file(GLOB SOURCES
src/objective/*.cpp
src/network/*.cpp
src/treelearner/*.cpp
src/application/*.cpp
if(USE_CUDA)
src/treelearner/*.cu
endif(USE_CUDA)
)

# To ease out the debugging
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0")
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0")

## NOTE: CUSTOM FLAGS
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++fs") #NOTE: may only be needed for running on the hdp

#add_executable(lightgbm src/main.cpp ${SOURCES}) # NOTE: this was the original command!
add_executable(lightgbm src/main.multiple_runs.cpp ${SOURCES}) # NOTE: changed it to this one!
add_executable(lightgbm src/main.cpp src/application/application.cpp ${SOURCES})
#add_executable(fairgbm_multiple_runs src/main.multiple_runs.cpp src/application/application.cpp ${SOURCES})
list(APPEND SOURCES "src/c_api.cpp")

# Only build the R part of the library if building for
Expand All @@ -365,7 +357,6 @@ if (BUILD_STATIC_LIB)
add_library(_lightgbm STATIC ${SOURCES})
else()
add_library(_lightgbm SHARED ${SOURCES})
# add_library(_lightgbm SHARED ${SOURCES} include/LightGBM/utils/constrained.hpp)
endif(BUILD_STATIC_LIB)

if(MSVC)
Expand Down
13 changes: 8 additions & 5 deletions include/LightGBM/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ struct Config {
// check = >0.0
// desc = learning rate for the constrained boosting
// desc = it only takes effect when using a constrained objective function
double lagrangian_learning_rate = 0.1;
double multiplier_learning_rate = 0.1;

// type = multi-double
// default = None
Expand All @@ -204,7 +204,7 @@ struct Config {
// desc = if not specified, will use 0 weight penalty for all constraints,
// desc = which is equivalent to using unconstrained version in the
// desc = first iteration.
std::vector<double> init_lagrangians;
std::vector<double> init_lagrange_multipliers;

// default = 31
// alias = num_leaf, max_leaves, max_leaf
Expand Down Expand Up @@ -823,6 +823,7 @@ struct Config {

#pragma endregion

// type = string
// desc = used only in ``training`` task
// desc = output dir of gradients and hessians per iteration
// desc = **Note**: can be used only in CLI version
Expand Down Expand Up @@ -924,7 +925,7 @@ struct Config {

// type = string
// desc = type of proxy function to use in constraints (hinge, quadratic, cross_entropy)
std::string constraint_stepwise_proxy = "quadratic";
std::string constraint_stepwise_proxy = "cross_entropy";

// type = string
// desc = type of proxy function to use as the proxy objective
Expand Down Expand Up @@ -966,15 +967,17 @@ struct Config {
std::string global_constraint_type;

// check = >=0
// check = <1.0
// check = <=1.0
// type = double
// default = 1.0
// desc = used only in ``constrained_cross_entropy`` application
// desc = target rate for the global FPR constraint
double global_target_fpr = 1.;

// check = >=0
// check = <1.0
// check = <=1.0
// type = double
// default = 1.0
// desc = used only in ``constrained_cross_entropy`` application
// desc = target rate for the global FNR constraint
double global_target_fnr = 1.;
Expand Down
8 changes: 4 additions & 4 deletions include/LightGBM/dataset.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class DatasetLoader;
* 4. Query Weights, auto calculate by weights and query_boundaries(if both of them are existed)
* the weight for i-th query is sum(query_boundaries[i] , .., query_boundaries[i+1]) / (query_boundaries[i + 1] - query_boundaries[i+1])
* 5. Initial score. optional. if existing, the model will boost from this score, otherwise will start from 0.
* 6. [FairGBM-only] Group, used for training during constrain optimization.
* 6. [FairGBM-only] Group, used for training during constrained optimization.
*/
class Metadata {
public:
Expand Down Expand Up @@ -208,8 +208,8 @@ class Metadata {
* \param idx Index of this record
* \param value Group constraint value of this record
*/
inline void SetGroupConstraintAt(data_size_t idx, data_size_t value) {
group_[idx] = static_cast<data_size_t>(value);
inline void SetGroupConstraintAt(data_size_t idx, group_t value) {
group_[idx] = value;
}

/*!
Expand All @@ -220,7 +220,7 @@ class Metadata {

/*! \brief Get unique groups in data */
inline std::vector<group_t> group_values() const {
std::vector<group_t> values = group_; // copy
std::vector<group_t> values(group_);
std::sort(values.begin(), values.end());

auto last = std::unique(values.begin(), values.end());
Expand Down
100 changes: 59 additions & 41 deletions include/LightGBM/objective_function.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,6 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
label_ = metadata.label();
weights_ = metadata.weights();

// ----------------------------------------------------- START FAIRBGM
// Store Information about the group
group_ = metadata.group();
group_values_ = metadata.group_values();
Expand All @@ -154,7 +153,6 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
total_label_positives_ = 0;
total_label_negatives_ = 0;
ComputeLabelCounts();
// -----------------------------------------------------END FAIRGM Update

CHECK_NOTNULL(label_);
Common::CheckElementsIntervalClosed<label_t>(label_, 0.0f, 1.0f, num_data_, GetName());
Expand Down Expand Up @@ -197,7 +195,7 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
virtual std::vector<double> GetLagrangianGradientsWRTMultipliers(const double *score) const
{
if (weights_ != nullptr)
throw std::runtime_error("not implemented yet");
throw std::logic_error("not implemented yet"); // TODO: https://github.com/feedzai/fairgbm/issues/5

std::vector<double> functions;
std::unordered_map<group_t, double> group_fpr, group_fnr;
Expand All @@ -211,7 +209,8 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction

// 1st Lagrange multiplier corresponds to predictive loss function
double sum_loss = 0.0;
// #pragma omp parallel for schedule(static) reduction(+:sum_loss)

// #pragma omp parallel for schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i)
{
sum_loss += this->ComputePredictiveLoss(label_[i], score[i]);
Expand All @@ -223,16 +222,18 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
if (IsFPRConstrained())
{
ComputeFPR(score, score_threshold_, group_fpr);
double max_fpr = findMaxValuePair<group_t, double>(group_fpr).second;
double max_fpr = Constrained::findMaxValuePair<group_t, double>(group_fpr).second;

// Assuming group_values_ is in ascending order
for (const auto &group : group_values_)
{
double fpr_constraint_value = max_fpr - group_fpr[group] - fpr_threshold_;
functions.push_back(fpr_constraint_value);

#ifdef FAIRGBM_DEBUG
std::cout << "DEBUG; true FPR constraint value: c=" << max_fpr << "-" << group_fpr[group] << "=" << fpr_constraint_value << std::endl;
#ifdef DEBUG
Log::Debug(
"DEBUG; true FPR constraint value: c = %.3f - %.3f = %.3f\n",
max_fpr, group_fpr[group], fpr_constraint_value);
#endif
}
}
Expand All @@ -241,16 +242,18 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
if (IsFNRConstrained())
{
ComputeFNR(score, score_threshold_, group_fnr);
double max_fnr = findMaxValuePair<group_t, double>(group_fnr).second;
double max_fnr = Constrained::findMaxValuePair<group_t, double>(group_fnr).second;

// Assuming group_values_ is in ascending order
for (const auto &group : group_values_)
{
double fnr_constraint_value = max_fnr - group_fnr[group] - fnr_threshold_;
functions.push_back(fnr_constraint_value);

#ifdef FAIRGBM_DEBUG
std::cout << "DEBUG; true FNR constraint value: c=" << max_fnr << "-" << group_fnr[group] << "=" << fnr_constraint_value << std::endl;
#ifdef DEBUG
Log::Debug(
"DEBUG; true FNR constraint value: c = %.3f - %.3f = %.3f\n",
max_fnr, group_fnr[group], fnr_constraint_value);
#endif
}
}
Expand All @@ -263,8 +266,10 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction

functions.push_back(global_fpr_constraint_value);

#ifdef FAIRGBM_DEBUG
std::cout << "DEBUG; true global FPR constraint value: c=" << global_fpr << "-" << global_target_fpr_ << "=" << global_fpr_constraint_value << std::endl;
#ifdef DEBUG
Log::Debug(
"DEBUG; true global FPR constraint value: c = %.3f - %.3f = %.3f\n",
global_fpr, global_target_fpr_, global_fpr_constraint_value);
#endif
}

Expand All @@ -276,8 +281,10 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction

functions.push_back(global_fnr_constraint_value);

#ifdef FAIRGBM_DEBUG
std::cout << "DEBUG; true global FNR constraint value: c=" << global_fnr << "-" << global_target_fnr_ << "=" << global_fnr_constraint_value << std::endl;
#ifdef DEBUG
Log::Debug(
"DEBUG; true global FNR constraint value: c = %.3f - %.3f = %.3f\n",
global_fnr, global_target_fnr_, global_fnr_constraint_value);
#endif
}

Expand Down Expand Up @@ -329,7 +336,7 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
else
throw std::invalid_argument("constraint_stepwise_proxy=" + constraint_stepwise_proxy + " not implemented!");

max_proxy_fpr = findMaxValuePair<group_t, double>(group_fpr);
max_proxy_fpr = Constrained::findMaxValuePair<group_t, double>(group_fpr);
}
if (IsFNRConstrained())
{
Expand All @@ -342,19 +349,19 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
else
throw std::invalid_argument("constraint_stepwise_proxy=" + constraint_stepwise_proxy + " not implemented!");

max_proxy_fnr = findMaxValuePair<group_t, double>(group_fnr);
max_proxy_fnr = Constrained::findMaxValuePair<group_t, double>(group_fnr);
}

/** ---------------------------------------------------------------- *
* GRADIENTS (per instance) *
* ---------------------------------------------------------------- */
if (weights_ != nullptr)
{
throw std::runtime_error("not implemented");
throw std::logic_error("not implemented yet"); // TODO: https://github.com/feedzai/fairgbm/issues/5
}

// compute pointwise gradients and hessians with implied unit weights
// #pragma omp parallel for schedule(static) // FIXME: there seems to be weird behavior with this directive
// #pragma omp parallel for schedule(static) // TODO: https://github.com/feedzai/fairgbm/issues/6
for (data_size_t i = 0; i < num_data_; ++i)
{
const auto group = group_[i];
Expand Down Expand Up @@ -404,13 +411,15 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
fpr_constraints_gradient_wrt_pred = score[i] <= -proxy_margin_ ? 0. : 1. / group_ln;

// Derivative for BCE-based proxy FPR
else if (constraint_stepwise_proxy == "cross_entropy")
fpr_constraints_gradient_wrt_pred = (sigmoid(score[i] + xent_horizontal_shift)) / group_ln;
// fpr_constraints_gradient_wrt_pred = (sigmoid(score[i]) - label_[i]) / group_ln; // without margin
else if (constraint_stepwise_proxy == "cross_entropy") {
fpr_constraints_gradient_wrt_pred = (Constrained::sigmoid(score[i] + xent_horizontal_shift)) / group_ln;
// fpr_constraints_gradient_wrt_pred = (Constrained::sigmoid(score[i]) - label_[i]) / group_ln; // without margin
}

// Loss-function implicitly defined as having a hinge-based derivative (quadratic loss)
else if (constraint_stepwise_proxy == "quadratic")
else if (constraint_stepwise_proxy == "quadratic") {
fpr_constraints_gradient_wrt_pred = std::max(0., score[i] + proxy_margin_) / group_ln;
}

else
throw std::invalid_argument("constraint_stepwise_proxy=" + constraint_stepwise_proxy + " not implemented!");
Expand Down Expand Up @@ -464,13 +473,15 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
fnr_constraints_gradient_wrt_pred = score[i] >= proxy_margin_ ? 0. : -1. / group_lp;

// Derivative for BCE-based proxy FNR
else if (constraint_stepwise_proxy == "cross_entropy")
fnr_constraints_gradient_wrt_pred = (sigmoid(score[i] - xent_horizontal_shift) - 1) / group_lp;
// fnr_constraints_gradient_wrt_pred = (sigmoid(score[i]) - label_[i]) / group_lp; // without margin
else if (constraint_stepwise_proxy == "cross_entropy") {
fnr_constraints_gradient_wrt_pred = (Constrained::sigmoid(score[i] - xent_horizontal_shift) - 1) / group_lp;
// fnr_constraints_gradient_wrt_pred = (Constrained::sigmoid(score[i]) - label_[i]) / group_lp; // without margin
}

// Loss-function implicitly defined as having a hinge-based derivative (quadratic loss)
else if (constraint_stepwise_proxy == "quadratic")
else if (constraint_stepwise_proxy == "quadratic") {
fnr_constraints_gradient_wrt_pred = std::min(0., score[i] - proxy_margin_) / group_lp;
}

else
throw std::invalid_argument("constraint_stepwise_proxy=" + constraint_stepwise_proxy + " not implemented!");
Expand Down Expand Up @@ -517,17 +528,20 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
{ // Condition for non-zero gradient
double global_fpr_constraint_gradient_wrt_pred;
// Gradient for hinge proxy FPR
if (constraint_stepwise_proxy == "hinge")
if (constraint_stepwise_proxy == "hinge") {
global_fpr_constraint_gradient_wrt_pred = score[i] >= -proxy_margin_ ? 1. / total_label_negatives_ : 0.;
}

// Gradient for BCE proxy FPR
else if (constraint_stepwise_proxy == "cross_entropy")
global_fpr_constraint_gradient_wrt_pred = (sigmoid(score[i] + xent_horizontal_shift)) / total_label_negatives_;
// global_fpr_constraint_gradient_wrt_pred = (sigmoid(score[i]) - label_[i]) / total_label_negatives_; // without margin
else if (constraint_stepwise_proxy == "cross_entropy") {
global_fpr_constraint_gradient_wrt_pred = (Constrained::sigmoid(score[i] + xent_horizontal_shift)) / total_label_negatives_;
// global_fpr_constraint_gradient_wrt_pred = (Constrained::sigmoid(score[i]) - label_[i]) / total_label_negatives_; // without margin
}

// Hinge-based gradient (for quadratic proxy FPR)
else if (constraint_stepwise_proxy == "quadratic")
else if (constraint_stepwise_proxy == "quadratic") {
global_fpr_constraint_gradient_wrt_pred = std::max(0., score[i] + proxy_margin_) / total_label_negatives_;
}

else
throw std::invalid_argument("constraint_stepwise_proxy=" + constraint_stepwise_proxy + " not implemented!");
Expand All @@ -548,20 +562,24 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
double global_fnr_constraint_gradient_wrt_pred;

// Gradient for hinge proxy FNR
if (constraint_stepwise_proxy == "hinge")
if (constraint_stepwise_proxy == "hinge") {
global_fnr_constraint_gradient_wrt_pred = score[i] >= proxy_margin_ ? 0. : -1. / total_label_positives_;
}

// Gradient for BCE proxy FNR
else if (constraint_stepwise_proxy == "cross_entropy")
global_fnr_constraint_gradient_wrt_pred = (sigmoid(score[i] - xent_horizontal_shift) - 1) / total_label_positives_;
// global_fnr_constraint_gradient_wrt_pred = (sigmoid(score[i]) - label_[i]) / total_label_positives_; // without margin
else if (constraint_stepwise_proxy == "cross_entropy") {
global_fnr_constraint_gradient_wrt_pred = (Constrained::sigmoid(score[i] - xent_horizontal_shift) - 1) / total_label_positives_;
// global_fnr_constraint_gradient_wrt_pred = (Constrained::sigmoid(score[i]) - label_[i]) / total_label_positives_; // without margin
}

// Hinge-based gradient (for quadratic proxy FNR)
else if (constraint_stepwise_proxy == "quadratic")
else if (constraint_stepwise_proxy == "quadratic") {
global_fnr_constraint_gradient_wrt_pred = std::min(0., score[i] - proxy_margin_) / total_label_positives_;
}

else
else {
throw std::invalid_argument("constraint_stepwise_proxy=" + constraint_stepwise_proxy + " not implemented!");
}

// Update instance gradient and hessian
gradients[i] += (score_t)(lagrangian_multipliers[multipliers_base_index] *
Expand Down Expand Up @@ -695,7 +713,7 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
std::unordered_map<group_t, double> false_positives; // map of group index to the respective hinge-proxy FPs
std::unordered_map<group_t, int> label_negatives; // map of group index to the respective number of LNs

// #pragma omp parallel for schedule(static) // FIXME
// #pragma omp parallel for schedule(static) // TODO: https://github.com/feedzai/fairgbm/issues/6
for (data_size_t i = 0; i < num_data_; ++i)
{
group_t group = group_[i];
Expand Down Expand Up @@ -736,7 +754,7 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
std::unordered_map<group_t, double> false_positives; // map of group index to the respective proxy FPs
std::unordered_map<group_t, int> label_negatives; // map of group index to the respective number of LNs

// #pragma omp parallel for schedule(static) // FIXME
// #pragma omp parallel for schedule(static) // TODO: https://github.com/feedzai/fairgbm/issues/6
for (data_size_t i = 0; i < num_data_; ++i)
{
group_t group = group_[i];
Expand Down Expand Up @@ -779,7 +797,7 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
std::unordered_map<group_t, int> label_negatives; // map of group index to the respective number of LNs
double xent_horizontal_shift = log(exp(proxy_margin_) - 1);

// #pragma omp parallel for schedule(static)
// #pragma omp parallel for schedule(static) // TODO: https://github.com/feedzai/fairgbm/issues/6
for (data_size_t i = 0; i < num_data_; ++i)
{
group_t group = group_[i];
Expand Down Expand Up @@ -1093,4 +1111,4 @@ class ConstrainedObjectiveFunction : public ObjectiveFunction
};
} // namespace LightGBM

#endif // LightGBM_OBJECTIVE_FUNCTION_H_
#endif // LightGBM_OBJECTIVE_FUNCTION_H_
Loading

0 comments on commit 356664b

Please sign in to comment.