From 0fe2abb258b16dca506fd528ed9c9cadbb13c635 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Thu, 6 Aug 2026 03:28:49 -0700 Subject: [PATCH 01/61] Add a binary/int8 fast path to the CPU feasibility jump Instances whose variables are all binary and whose rows carry integer coefficients within int8 or int16 range run an integer engine instead of the general path. Feasibility is an exact compare against one row bound after a one-sided split, so there is no tolerance arithmetic and no compensated summation anywhere; a live per-variable score is patched through stored per-nnz contributions, and move selection is a global argmax over that score while the objective weight is zero. The hot kernels are vectorized with Google Highway, compiled once per SIMD target and dispatched at runtime. They live in a host-compiled .cpp because nvcc's frontend rejects Highway's x86 headers, so the seam passes only plain pointers and scalars. Choices settled by measurement rather than argument, on an EPYC 9554 (Zen 4, AVX-512) unless noted, with supportcase22 and bnatt400 at 16 climbers: - The row remainder is masked into the vector body on targets with mask registers and peeled into a scalar tail elsewhere. Masking cost 6.8% and 12.9% on AVX2, where the mask becomes a vector and scatter is emulated. - The score read-modify-write goes lane by lane on Zen 4 rather than through VPSCATTERDD, which is 89 uops there against ~19 on SPR-class parts: +5.8%. The assignment gather stays in hardware; doing that one by lane cost 8.2%, to store-to-load forwarding on the spilled index vector. - Rows dispatch to a 4-, 8- or native-width kernel by length, since a gather costs the same whether its lanes carry data or are masked off and rows are far shorter than a 512-bit vector. Worth +2.0% on supportcase22 and +3.9% on bnatt400. - The reverse-CSR row walk carries a software prefetch. Neutral here, kept because the walk is the one data-dependent access the hardware cannot follow. Also adds solve_CPUFJ, a benchmark-only harness that runs a portfolio of climbers on one instance, and an end-of-solve audit that recomputes the incumbent's row activities in int64 and its objective from the assignment alone, trusting nothing the incremental path maintained. Co-Authored-By: Claude Opus 5 --- .../linear_programming/cuopt/run_cpufj.cu | 207 +++ cpp/CMakeLists.txt | 69 + cpp/src/mip_heuristics/CMakeLists.txt | 2 + .../mip_heuristics/feasibility_jump/fj_cpu.cu | 11 + .../feasibility_jump/fj_cpu.cuh | 17 + .../feasibility_jump/fj_cpu_binary.cu | 1108 +++++++++++++++++ .../feasibility_jump/fj_cpu_binary.cuh | 169 +++ .../fj_cpu_binary_kernels.cpp | 449 +++++++ cpp/src/utilities/version_info.cpp | 21 + 9 files changed, 2053 insertions(+) create mode 100644 benchmarks/linear_programming/cuopt/run_cpufj.cu create mode 100644 cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu create mode 100644 cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh create mode 100644 cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu new file mode 100644 index 0000000000..6844d09347 --- /dev/null +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -0,0 +1,207 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +// Benchmark-only harness for the CPU feasibility-jump portfolio. Loads an instance, builds one +// climber per portfolio slot from a zero start clamped to variable bounds, and runs them in +// parallel on pinned cores for a fixed wall-clock budget. Reports per-climber crossing, objective +// and throughput. +// +// No presolve: pass an already-presolved instance. The climbers are built from problem_t, so the +// binary fast path is reachable when the instance qualifies. + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using i_t = int; +using f_t = double; +namespace mip = cuopt::mathematical_optimization::mip; + +using clk = std::chrono::high_resolution_clock; +double since(clk::time_point t0) +{ + return std::chrono::duration_cast>(clk::now() - t0).count(); +} + +struct climber_result_t { + bool crossed{false}; + double t_first{-1.0}; + f_t best_objective{std::numeric_limits::infinity()}; + i_t iterations{0}; + double seconds{0.0}; +}; + +void pin_to_core(int core) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(core, &set); + pthread_setaffinity_np(pthread_self(), sizeof(set), &set); +} + +// The CPUs this process is actually permitted to run on. A cgroup mask can be non-contiguous, so +// indexing hardware_concurrency() directly would collide several climbers onto one core. +std::vector allowed_cpus() +{ + std::vector allowed; + cpu_set_t set; + CPU_ZERO(&set); + if (sched_getaffinity(0, sizeof(set), &set) == 0) { + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &set)) allowed.push_back(cpu); + } + } + if (allowed.empty()) allowed.push_back(0); + return allowed; +} + +void run_climber(mip::fj_cpu_climber_t* climber, + f_t time_limit, + int core, + climber_result_t& result) +{ + pin_to_core(core); + const auto t0 = clk::now(); + + climber->improvement_callback = [&result, t0](f_t objective, const std::vector&, double) { + if (!result.crossed) { + result.crossed = true; + result.t_first = since(t0); + } + result.best_objective = objective; + }; + + mip::cpufj_solve(climber, time_limit); + + result.seconds = since(t0); + result.iterations = climber->iterations; +} + +} // namespace + +int main(int argc, char** argv) +{ + if (argc < 2) { + std::fprintf(stderr, "usage: %s [time_limit_s=60] [climbers=16] [seed=12345]\n", + argv[0]); + return 2; + } + const std::string path = argv[1]; + const f_t time_limit = argc > 2 ? std::atof(argv[2]) : 60.0; + const int n_climbers = argc > 3 ? std::atoi(argv[3]) : 16; + const unsigned base_seed = argc > 4 ? (unsigned)std::atoll(argv[4]) : 12345u; + + // Console sink so the engine's end-of-solve incumbent audit is visible, as solve_MIP does it. + cuopt::init_logger_t log_guard("", true); + + raft::handle_t handle; + + const auto mps_data_model = cuopt::mathematical_optimization::io::read_mps(path, false); + const auto op_problem = + cuopt::mathematical_optimization::mps_data_model_to_optimization_problem( + &handle, mps_data_model); + mip::problem_t problem(op_problem); + std::printf("instance: %s n_vars=%d n_cstrs=%d nnz=%d\n", + path.c_str(), + problem.n_variables, + problem.n_constraints, + problem.nnz); + + // Zero start, clamped into the variable bounds. Shared by every climber; diversity comes from + // the per-climber seed and sampling parameters below. + mip::solution_t solution(problem); + thrust::fill(handle.get_thrust_policy(), solution.assignment.begin(), solution.assignment.end(), f_t{0}); + mip::clamp_within_var_bounds(solution.assignment, &problem, &handle); + handle.sync_stream(); + + // Built serially: each climber host-copies the problem off the same stream. + std::vector> preemption_flags(n_climbers); + std::vector>> climbers(n_climbers); + for (int k = 0; k < n_climbers; ++k) { + preemption_flags[k].store(false); + mip::fj_settings_t settings; + settings.seed = (int)(base_seed + k); + climbers[k] = mip::init_fj_cpu_standalone(problem, solution, preemption_flags[k], settings); + + // Portfolio diversification, decorrelated from the value RNG. + std::mt19937 rng(base_seed + 7919u * k); + climbers[k]->mtm_viol_samples = std::uniform_int_distribution(15, 50)(rng); + climbers[k]->mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); + climbers[k]->nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); + climbers[k]->perturb_interval = std::uniform_int_distribution(50, 500)(rng); + climbers[k]->log_prefix = "[climber " + std::to_string(k) + "] "; + } + + const std::vector cpus = allowed_cpus(); + std::printf("running %d climbers x %.0fs, base seed %u, %zu allowed CPUs (%d..%d)\n", + n_climbers, (double)time_limit, base_seed, cpus.size(), cpus.front(), cpus.back()); + + std::vector results(n_climbers); + std::vector threads; + threads.reserve(n_climbers); + const auto wall0 = clk::now(); + for (int k = 0; k < n_climbers; ++k) { + threads.emplace_back( + run_climber, climbers[k].get(), time_limit, cpus[k % cpus.size()], std::ref(results[k])); + } + for (auto& t : threads) { + t.join(); + } + const double wall = since(wall0); + + int crossed = 0; + double sum_iters = 0; + f_t best_overall = std::numeric_limits::infinity(); + std::printf("\n climber | crossed | t_first(s) | obj | iters | iters/s\n"); + std::printf("---------+---------+------------+--------------+----------+---------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& r = results[k]; + sum_iters += r.iterations; + if (r.crossed) { + ++crossed; + best_overall = std::min(best_overall, r.best_objective); + } + std::printf(" %7d | %7s | %10s | %12.6g | %8d | %8.0f\n", + k, + r.crossed ? "YES" : "no", + r.crossed ? std::to_string(r.t_first).c_str() : "-", + r.crossed ? (double)r.best_objective : 0.0, + r.iterations, + r.seconds > 0 ? r.iterations / r.seconds : 0.0); + } + std::printf("\nSUMMARY: %d/%d crossed (%.0f%%) wall=%.1fs total_iters=%.0f agg_iters/s=%.0f\n", + crossed, + n_climbers, + 100.0 * crossed / n_climbers, + wall, + sum_iters, + wall > 0 ? sum_iters / wall : 0.0); + if (crossed > 0) { std::printf("BEST OBJECTIVE: %.10g\n", (double)best_overall); } + return 0; +} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b375cc4c56..d95f48f58b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -308,6 +308,31 @@ FetchContent_MakeAvailable(pslp) set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_SAVED}) +# Highway - portable SIMD with runtime dispatch, used by the binary CPU FJ fast-path kernels +# https://github.com/google/highway +# contrib carries vqsort/image/math, none of which we use, and each would be compiled once per +# SIMD target. +FetchContent_Declare( + highway + GIT_REPOSITORY "https://github.com/google/highway.git" + GIT_TAG "1.4.0" + GIT_PROGRESS TRUE + EXCLUDE_FROM_ALL + SYSTEM +) + +set(HWY_ENABLE_CONTRIB OFF CACHE BOOL "" FORCE) +set(HWY_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE) +set(HWY_ENABLE_TESTS OFF CACHE BOOL "" FORCE) +set(HWY_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) + +# Build Highway as static to embed in cuopt (mirrors PSLP above) +set(BUILD_SHARED_LIBS_SAVED ${BUILD_SHARED_LIBS}) +set(BUILD_SHARED_LIBS OFF) +FetchContent_MakeAvailable(highway) +set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_SAVED}) + + # dejavu - header-only graph automorphism library for MIP symmetry detection # https://github.com/markusa4/dejavu (header-only, skip its CMakeLists.txt) FetchContent_Declare( @@ -666,6 +691,7 @@ target_include_directories(cuopt_objs PRIVATE target_include_directories(cuopt_objs SYSTEM PRIVATE "${pslp_SOURCE_DIR}/include" "${dejavu_SOURCE_DIR}" + "${highway_SOURCE_DIR}" ) target_include_directories(cuopt_objs @@ -691,6 +717,11 @@ target_include_directories(cuopt_objs target_link_libraries(cuopt_objs PRIVATE $) add_dependencies(cuopt_objs PSLP) +# Highway, linked by file for the same reason. Static and fully embedded into libcuopt.so; it is +# never installed (HWY_ENABLE_INSTALL OFF) and consumers of cuopt::cuopt never reference it. +target_link_libraries(cuopt_objs PRIVATE $) +add_dependencies(cuopt_objs hwy) + # Link KaMinPar by file to avoid export dependency tracking (mirrors PSLP above). # KaMinPar is a from-source static library fully embedded into libcuopt.so; it is never # installed (INSTALL_KAMINPAR OFF) and consumers of cuopt::cuopt never use it, so it must @@ -813,6 +844,8 @@ if (BUILD_TESTS) ) target_link_libraries(cuopt_static PRIVATE $) add_dependencies(cuopt_static PSLP) + target_link_libraries(cuopt_static PRIVATE $) + add_dependencies(cuopt_static hwy) target_link_libraries(cuopt_static PRIVATE $) if (TARGET KaMinPar) add_dependencies(cuopt_static KaMinPar) @@ -857,6 +890,8 @@ target_link_libraries(cuopt ) target_link_libraries(cuopt PRIVATE $) add_dependencies(cuopt PSLP) +target_link_libraries(cuopt PRIVATE $) +add_dependencies(cuopt hwy) target_link_libraries(cuopt PRIVATE $) if (TARGET KaMinPar) add_dependencies(cuopt KaMinPar) @@ -1065,6 +1100,40 @@ if (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY) "${CMAKE_CURRENT_SOURCE_DIR}/src" ) + # CPU FJ portfolio benchmark. A .cu because it drives problem_t/solution_t directly and + # clamp_within_var_bounds instantiates a device lambda in this TU. + add_executable(solve_CPUFJ ../benchmarks/linear_programming/cuopt/run_cpufj.cu) + + set_target_properties(solve_CPUFJ PROPERTIES CXX_SCAN_FOR_MODULES OFF) + + # -fopenmp for the CUDA TU as well: the internal headers this pulls in (omp_helpers.hpp, + # omp_atomic_t) are OMP-dependent, and OpenMP::OpenMP_CXX only covers CXX. + target_compile_options(solve_CPUFJ + PRIVATE "$<$:${CUOPT_CXX_FLAGS}>" + "$<$:${CUOPT_CUDA_FLAGS}>" + "$<$:-fopenmp>" + ) + + target_link_libraries(solve_CPUFJ + PUBLIC + cuopt + OpenMP::OpenMP_CXX + OpenMP::OpenMP_CUDA + ) + + target_include_directories(solve_CPUFJ + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src" + "${papilo_SOURCE_DIR}/src" + "${papilo_BINARY_DIR}" + ) + + # Reached transitively: fj_cpu.cuh pulls branch_and_bound/symmetry.hpp, which needs dejavu. + target_include_directories(solve_CPUFJ SYSTEM PRIVATE + "${pslp_SOURCE_DIR}/include" + "${dejavu_SOURCE_DIR}" + ) + endif () option(BUILD_LP_BENCHMARKS "Build LP benchmarks" OFF) diff --git a/cpp/src/mip_heuristics/CMakeLists.txt b/cpp/src/mip_heuristics/CMakeLists.txt index f54619cfe3..a35cdd7e4f 100644 --- a/cpp/src/mip_heuristics/CMakeLists.txt +++ b/cpp/src/mip_heuristics/CMakeLists.txt @@ -45,6 +45,8 @@ set(MIP_NON_LP_FILES ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/feasibility_jump.cu ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/feasibility_jump_kernels.cu ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/fj_cpu.cu + ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/fj_cpu_binary.cu + ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/fj_cpu_binary_kernels.cpp ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/early_cpufj.cu ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/early_gpufj.cu) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index b789159953..e1b7082b0d 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -13,6 +13,7 @@ #include "feasibility_jump.cuh" #include "feasibility_jump_impl_common.cuh" #include "fj_cpu.cuh" +#include "fj_cpu_binary.cuh" #include "fj_cpu_worker.cuh" #include @@ -1700,6 +1701,11 @@ void finalize_fj_cpu_host_initialization( // Precompute static problem features for regression model precompute_problem_features(fj_cpu); + + // Binary fast path. Depends only on the host problem mirrors and the incoming weights, both + // populated above; engine state is initialized later, at solve entry. Climbers built from a + // host LP reach here too and are declined by the predicate at their first slack column. + try_build_binary_fastpath(fj_cpu); } template @@ -1893,6 +1899,11 @@ std::unique_ptr> fj_t::create_cpu_climber( template void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double work_unit_limit) { + if (fj_cpu->binary_fast) { + fj_cpu->binary_fast->solve(*fj_cpu, in_time_limit, work_unit_limit); + return; + } + i_t local_mins = 0; auto loop_start = std::chrono::high_resolution_clock::now(); auto time_limit = std::chrono::milliseconds(static_cast(std::floor(in_time_limit * 1000.0))); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 411b4083f7..ce1d010151 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,17 @@ namespace cuopt::mathematical_optimization::mip { +// Binary fast-path state. Defined in fj_cpu_binary.cuh, which only fj_cpu.cu includes, so this +// header stays independent of the fast path. The deleter is defined out of line for the same +// reason, mirroring fj_cpu_worker_t::fj_cpu_deleter_t. +template +struct fj_binary_state_t; + +template +struct fj_binary_state_deleter_t { + void operator()(fj_binary_state_t* ptr) const; +}; + template class probing_cache_t; @@ -215,6 +227,11 @@ struct fj_cpu_climber_t { instrumentation_aggregator_t memory_aggregator; // TODO atomic ref? c++20 std::atomic& preemption_flag; + + // Populated by try_build_binary_fastpath when the instance is all-binary with integer + // coefficients in int8/int16 range. Empty on every other instance, in which case cpufj_solve + // runs the general loop below. + std::unique_ptr, fj_binary_state_deleter_t> binary_fast; }; template diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu new file mode 100644 index 0000000000..5c5a28d9f5 --- /dev/null +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -0,0 +1,1108 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#include "fj_cpu_binary.cuh" + +#include "feasibility_jump.cuh" +#include "fj_cpu.cuh" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace cuopt::mathematical_optimization::mip { + +const char* fj_binary_reject_name(fj_binary_reject_t reason) +{ + switch (reason) { + case fj_binary_reject_t::none: return "none"; + case fj_binary_reject_t::empty_problem: return "empty problem"; + case fj_binary_reject_t::non_binary_var: return "non-binary variable"; + case fj_binary_reject_t::fractional_coefficient: return "fractional coefficient"; + case fj_binary_reject_t::coefficient_out_of_range: return "coefficient wider than int16"; + case fj_binary_reject_t::fractional_row_bound: return "fractional row bound"; + case fj_binary_reject_t::row_bound_out_of_range: return "row bound outside int32"; + case fj_binary_reject_t::lhs_headroom: return "row sum|coef| exceeds int32 headroom"; + case fj_binary_reject_t::narrow_check_failed: return "narrowing check failed"; + } + return "unknown"; +} + + +// Work-unit proxy: bytes attributed per nnz touched. Stands in for the byte counters the general +// path reads off its instrumented vectors. Calibrated by the owner. +constexpr double fj_bin_bytes_per_nnz = 16.0; + +// Tabu for binary variables. A binary variable's only move is a flip, so the direction is a +// function of the current assignment and the general four-array scheme collapses to two. +// flip_until is biased by a rolling base so it fits uint16. +struct fj_bin_tabu_t { + static constexpr int32_t window = 65535 - 64; + + std::vector flip_until; + std::vector last_flip; + int32_t base{0}; + + + void resize(int32_t n) + { + flip_until.assign(n, 0); + last_flip.assign(n, 0); + base = 0; + } + + void clear(int32_t iter) + { + std::fill(flip_until.begin(), flip_until.end(), (uint16_t)0); + std::fill(last_flip.begin(), last_flip.end(), 0); + base = iter; + } + + void on_flip(int32_t v, int32_t iter, int32_t tenure) + { + flip_until[v] = (uint16_t)(iter + tenure - base); + last_flip[v] = iter; + } + + + bool blocked(int32_t v, int32_t iter, bool localmin) const + { + return localmin ? (iter == last_flip[v] + 1) : ((uint16_t)(iter - base) < flip_until[v]); + } + + // Rebase before iter - base can overflow the uint16 window. Expired entries saturate to 0, + // which reads as not-tabu. + void maybe_advance(int32_t iter) + { + if ((int64_t)iter - base <= window) return; + const uint16_t shift = (uint16_t)(iter - base); + for (uint16_t& fu : flip_until) fu = (fu > shift) ? (uint16_t)(fu - shift) : (uint16_t)0; + base = iter; + } +}; + +// One row of the narrowed problem. bound/sign encode the single finite bound the split leaves: +// sign +1 for lhs <= bound, -1 for lhs >= bound. cmax is max|coef| over the row, which bounds how +// far a single flip can move the slack. +template +struct fj_bin_row_t { + int32_t lhs; + int32_t weight; + int32_t bound; + coef_t cmax; + int8_t sign; +}; + +// Narrowed problem: one-sided rows, integer coefficients, CSR plus its transpose. +template +struct fj_bin_problem_t { + int32_t n_variables{0}; + int32_t n_constraints{0}; + int32_t nnz{0}; + + std::vector offsets; + std::vector variables; + std::vector coefficients; + + std::vector reverse_offsets; + std::vector reverse_constraints; + std::vector reverse_coefficients; + std::vector reverse_to_csr; + + std::vector bound; + std::vector sign; + std::vector cmax; + std::vector initial_weight; + + std::vector objective; + std::vector objective_vars; +}; + +// Result of the width-independent eligibility scan. +struct fj_bin_scan_t { + fj_binary_reject_t reject{fj_binary_reject_t::none}; + int coefficient_bits{0}; + int32_t n_split_constraints{0}; + int32_t bad_row{-1}; + int32_t bad_var{-1}; +}; + +// DDFW and restart have no general-path equivalent, so their defaults live here until there is a +// reason to promote them alongside the other FJ knobs. +constexpr int32_t fj_bin_ddfw_init = 10; // initial weight, also the donation floor +constexpr int32_t fj_bin_ddfw_transfer = 1; +constexpr int32_t fj_bin_ddfw_donor_samples = 1; +constexpr int32_t fj_bin_restart_period = 5000000; + +// `[study]` Prefetch distance for the reverse-CSR row walk, in rows. Each move visits the rows of +// one variable through reverse_constraints, whose entries are effectively random indices into a row +// array far larger than L2, and the sequence is data-dependent so no hardware prefetcher can follow +// it. reverse_constraints is padded by this much so the lookahead needs no bounds test. +constexpr int32_t fj_bin_pf_dist = 8; + +// Limits of the packed score. Breaching either corrupts the ordering, so compute_saturation +// reports the observed peaks against them at end of solve. +constexpr int32_t fj_bin_base_limit = 1 << 16; +constexpr int32_t fj_bin_bonus_limit = 1 << 14; + +static inline bool fj_bin_is_integral(double v, double tol) { return std::fabs(v - std::round(v)) <= tol; } + +static inline bool fj_bin_in_int32(double v) +{ + return v >= (double)INT32_MIN && v <= (double)INT32_MAX; +} + +// Width-independent eligibility scan over the climber's host mirrors. Mutates nothing. +template +static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) +{ + fj_bin_scan_t out; + const int32_t n = c.view.pb.n_variables; + const int32_t m = c.view.pb.n_constraints; + if (n <= 0 || m <= 0) { + out.reject = fj_binary_reject_t::empty_problem; + return out; + } + + const double tol = c.view.pb.tolerances.integrality_tolerance; + const auto& var_bounds = c.h_var_bounds; + const auto& var_types = c.h_var_types; + + for (int32_t v = 0; v < n; ++v) { + auto bounds = var_bounds[v]; + if (var_types[v] != var_t::INTEGER || std::fabs(get_lower(bounds)) > tol || + std::fabs(get_upper(bounds) - 1.0) > tol) { + out.reject = fj_binary_reject_t::non_binary_var; + out.bad_var = v; + return out; + } + } + + const auto& offsets = c.h_offsets; + const auto& coeffs = c.h_coefficients; + const auto& cstr_lb = c.h_cstr_lb; + const auto& cstr_ub = c.h_cstr_ub; + + double max_abs_coefficient = 0; + for (int32_t r = 0; r < m; ++r) { + double row_abs_sum = 0; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + const double a = coeffs[k]; + if (!fj_bin_is_integral(a, tol)) { + out.reject = fj_binary_reject_t::fractional_coefficient; + out.bad_row = r; + return out; + } + const double abs_a = std::fabs(std::round(a)); + row_abs_sum += abs_a; + if (abs_a > max_abs_coefficient) max_abs_coefficient = abs_a; + } + + // A binary assignment can drive lhs to sum|coef|; keep that inside the int32 accumulator with + // room to spare. The int8-only reference engine never needed this bound. + if (row_abs_sum > (double)(INT32_MAX / 2)) { + out.reject = fj_binary_reject_t::lhs_headroom; + out.bad_row = r; + return out; + } + + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + const bool lb_fin = std::isfinite(lb); + const bool ub_fin = std::isfinite(ub); + const double sides[2] = {lb, ub}; + const bool finite[2] = {lb_fin, ub_fin}; + for (int s = 0; s < 2; ++s) { + if (!finite[s]) continue; + if (!fj_bin_is_integral(sides[s], tol)) { + out.reject = fj_binary_reject_t::fractional_row_bound; + out.bad_row = r; + return out; + } + if (!fj_bin_in_int32(std::round(sides[s]))) { + out.reject = fj_binary_reject_t::row_bound_out_of_range; + out.bad_row = r; + return out; + } + } + // Free rows are dropped: trivially satisfied, contributing nothing to the search. + out.n_split_constraints += (int32_t)lb_fin + (int32_t)ub_fin; + } + + if (out.n_split_constraints <= 0) { + out.reject = fj_binary_reject_t::empty_problem; + return out; + } + + if (max_abs_coefficient <= 127.0) { + out.coefficient_bits = 8; + } else if (max_abs_coefficient <= 32767.0) { + out.coefficient_bits = 16; + } else { + out.reject = fj_binary_reject_t::coefficient_out_of_range; + } + return out; +} + +// Build the narrowed, one-sided problem. Called only after fj_bin_scan cleared the instance, so a +// failing check here is a self-consistency bug and refuses the fast path rather than truncating. +template +static bool fj_bin_narrow(const fj_cpu_climber_t& c, + int32_t n_split, + fj_bin_problem_t& pb) +{ + const int32_t n = c.view.pb.n_variables; + const int32_t m = c.view.pb.n_constraints; + const double tol = c.view.pb.tolerances.integrality_tolerance; + + const auto& offsets = c.h_offsets; + const auto& variables = c.h_variables; + const auto& coeffs = c.h_coefficients; + const auto& cstr_lb = c.h_cstr_lb; + const auto& cstr_ub = c.h_cstr_ub; + const auto& left_w = c.h_cstr_left_weights; + const auto& right_w = c.h_cstr_right_weights; + const auto& obj = c.h_obj_coeffs; + + pb.n_variables = n; + pb.n_constraints = n_split; + pb.offsets.assign(1, 0); + pb.offsets.reserve(n_split + 1); + pb.bound.reserve(n_split); + pb.sign.reserve(n_split); + pb.cmax.reserve(n_split); + pb.initial_weight.reserve(n_split); + + std::vector incoming_weight; + incoming_weight.reserve(n_split); + + // Each split row inherits the weight of the side it came from: left is the lower-bound side, + // right the upper. + auto emit = [&](int32_t r, double side_bound, int8_t sign, double weight) -> bool { + coef_t row_cmax = 1; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + const double a = coeffs[k]; + const long ai = std::lround(a); + if (!fj_bin_is_integral(a, tol) || ai < std::numeric_limits::min() || + ai > std::numeric_limits::max()) { + return false; + } + pb.variables.push_back(variables[k]); + pb.coefficients.push_back((coef_t)ai); + const coef_t abs_a = (coef_t)std::labs(ai); + if (abs_a > row_cmax) row_cmax = abs_a; + } + const long b = std::lround(side_bound); + if (!fj_bin_in_int32((double)b)) return false; + pb.offsets.push_back((int32_t)pb.variables.size()); + pb.bound.push_back((int32_t)b); + pb.sign.push_back(sign); + pb.cmax.push_back(row_cmax); + incoming_weight.push_back(weight); + return true; + }; + + for (int32_t r = 0; r < m; ++r) { + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + if (std::isfinite(lb) && !emit(r, lb, (int8_t)-1, left_w[r])) return false; + if (std::isfinite(ub) && !emit(r, ub, (int8_t)1, right_w[r])) return false; + } + if ((int32_t)pb.bound.size() != n_split) return false; + pb.nnz = (int32_t)pb.variables.size(); + + // One vector of padding past nnz, so the row kernel can load and store whole vectors at the last + // row without running off the end and can therefore mask its remainder rather than peeling it + // into a scalar tail. The padding is never read as data: every lane past a row's end is excluded + // from the gather, the scatter and the store by the row-length mask. + const int32_t pad = fj_bin_simd_padding(); + pb.variables.resize(pb.nnz + pad, 0); + pb.coefficients.resize(pb.nnz + pad, (coef_t)0); + + // Scale the incoming weights into the DDFW band by one global factor, so relative structure + // survives while every row clears the donation floor. Capped so the largest scaled weight stays + // clear of packed-score saturation; where the cap binds, the smallest rows sit below the floor. + // TODO: bound the scaled weights by derivation instead of leaving them open. The packed score + // holds while a variable's aggregate base stays under 2^16, and that aggregate is bounded by the + // sum of weights over the rows the variable appears in, so 2^16 / max_var_degree gives a per-row + // bound computable here from the transpose. Left uncapped for now, matching the reference + // engine, which shipped with its weight cap disabled and relied on the end-of-solve saturation + // report to say whether a bound was needed. + double w_min = std::numeric_limits::infinity(); + for (double w : incoming_weight) { + if (w > 0 && w < w_min) w_min = w; + } + double scale = 1.0; + if (std::isfinite(w_min) && w_min > 0) { + scale = (double)fj_bin_ddfw_init / w_min; + if (scale < 1.0) scale = 1.0; + } + for (double w : incoming_weight) { + int32_t scaled = w > 0 ? (int32_t)std::lround(w * scale) : fj_bin_ddfw_init; + if (scaled < 1) scaled = 1; + pb.initial_weight.push_back(scaled); + } + + // Transpose, plus the reverse-nnz to CSR-nnz map the apply path uses to store the flipped + // variable's own score delta. + pb.reverse_offsets.assign(n + 1, 0); + for (int32_t k = 0; k < pb.nnz; ++k) pb.reverse_offsets[pb.variables[k] + 1]++; + for (int32_t v = 0; v < n; ++v) pb.reverse_offsets[v + 1] += pb.reverse_offsets[v]; + pb.reverse_constraints.resize(pb.nnz); + pb.reverse_coefficients.resize(pb.nnz); + pb.reverse_to_csr.resize(pb.nnz); + { + std::vector cursor(pb.reverse_offsets.begin(), pb.reverse_offsets.begin() + n); + for (int32_t r = 0; r < n_split; ++r) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t slot = cursor[pb.variables[k]]++; + pb.reverse_constraints[slot] = r; + pb.reverse_coefficients[slot] = pb.coefficients[k]; + pb.reverse_to_csr[slot] = k; + } + } + } + // Lookahead room for the row-walk prefetch. Reads land on row 0, which is prefetched harmlessly. + pb.reverse_constraints.resize(pb.nnz + fj_bin_pf_dist, 0); + + pb.objective.resize(n); + for (int32_t v = 0; v < n; ++v) { + pb.objective[v] = obj[v]; + if (pb.objective[v] != 0.0) pb.objective_vars.push_back(v); + } + return true; +} + + +// The integer engine. Feasibility is an exact compare against one bound per row, so there is no +// tolerance arithmetic and no compensated summation anywhere below. +template +struct fj_bin_engine_t : fj_binary_state_t { + fj_bin_problem_t pb; + std::vector> rows; + + std::vector assign; + std::vector best_assign; + std::vector seed_assign; // restart target + std::vector assign_i32; // gather mirror for the SIMD patch (Batch B) + + std::vector var_score; // live feasibility score of flipping each variable + std::vector nnz_score_delta; // per CSR nnz: last score delta of variables[k] in its row + + fj_bin_tabu_t tabu; + + std::vector is_violated; + std::vector violated_list; + std::vector vpos; + std::vector var_bitmap; + + // One generator advanced across the whole search, rather than one re-seeded per call site per + // iteration. Re-seeding from `seed + iters` gave every call site in an iteration the identical + // stream, and a 624-word Mersenne state was being built and discarded on every move selection. + raft::random::PCGenerator rng{0, 0, 0}; + std::vector sample_buf; // move-selection row sample, reused to keep the loop allocation-free + + int32_t objective_weight{0}; + double incumbent_objective{0}; + double best_objective{std::numeric_limits::infinity()}; + int32_t max_weight{1}; + bool feasible_found{false}; + + int32_t iters{0}; + int32_t last_feasible_entrance_iter{0}; + int32_t last_restart_iter{0}; + int64_t nnz_touched{0}; + + // Denominator for the ops-per-nnz roofline: nonzeros the row kernel actually processes, and the + // rows walked to find them. Unlike nnz_touched these are not mixed with the full-matrix rebuilds. + int64_t nnz_patched{0}; + int64_t rows_walked{0}; + + // Tile width for the argmax sweep. Governs how often the running maximum is raised, which is + // what bounds the index re-scan, so it is about the shape of the sweep and not cache capacity. + int32_t argmax_tile{256}; + + // Rows at or below these lengths go to the 4- and 8-lane patch kernels; 0 disables that width. + int32_t narrow4_max{0}; + int32_t narrow8_max{0}; + + // Settings read at solve entry, where the climber carries populated values. + int32_t seed{0}; + int32_t tabu_tenure_min{3}; + int32_t tabu_tenure_max{13}; + int32_t perturb_interval{100}; + int32_t mtm_viol_samples{25}; + int32_t mtm_sat_samples{15}; + double breakthrough_margin{1e-4}; + + int32_t max_aggregate_base{0}; + int32_t max_aggregate_bonus{0}; + + int coefficient_bits() const override { return 8 * (int)sizeof(coef_t); } + int n_split_constraints() const override { return pb.n_constraints; } + i_t iterations() const override { return (i_t)iters; } + + void saturation(int& base_peak, int& bonus_peak) const override + { + base_peak = max_aggregate_base; + bonus_peak = max_aggregate_bonus; + } + + // Largest per-variable aggregate base and bonus under the final weights and assignment, in raw + // int32. The packed representation is only order-preserving while these stay inside their + // limits, and weights grow without a cap, so this is the reading that says whether the packing + // survived the run. + // Independent audit of the incumbent at end of solve. Recomputes every row's lhs and the objective + // from best_assign alone, trusting nothing the incremental path maintained: not the live lhs, not + // violated_list, not the running incumbent_objective. Accumulates in int64 so an int32 lhs + // overflow the eligibility scan was supposed to preclude would show up here rather than wrap + // silently. Runs once per solve, so its cost is not on any hot path. + void verify_incumbent(fj_cpu_climber_t& climber) const + { + if (!feasible_found) return; + + int32_t n_violated = 0; + int64_t worst = 0; + bool lhs_overflow = false; + for (int32_t r = 0; r < pb.n_constraints; ++r) { + int64_t lhs = 0; + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + lhs += (int64_t)pb.coefficients[k] * (int64_t)best_assign[pb.variables[k]]; + } + if (lhs < INT32_MIN || lhs > INT32_MAX) lhs_overflow = true; + const int64_t slack = (int64_t)pb.sign[r] * ((int64_t)pb.bound[r] - lhs); + if (slack < 0) { + ++n_violated; + if (-slack > worst) worst = -slack; + } + } + + double objective = 0; + for (int32_t v = 0; v < pb.n_variables; ++v) objective += pb.objective[v] * (double)best_assign[v]; + const double drift = std::fabs(objective - best_objective); + + if (n_violated != 0 || lhs_overflow || drift > 1e-6) { + CUOPT_LOG_ERROR( + "%sCPUFJ[bin%d] incumbent audit FAILED: %d violated rows (worst %lld), lhs overflow %d, " + "objective recomputed %.17g vs tracked %.17g (drift %g)", + climber.log_prefix.c_str(), + coefficient_bits(), + n_violated, + (long long)worst, + (int)lhs_overflow, + objective, + best_objective, + drift); + } else { + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] incumbent audit ok: feasible, objective %.17g (drift %g)", + climber.log_prefix.c_str(), + coefficient_bits(), + objective, + drift); + } + } + + void compute_saturation() + { + int32_t peak_base = 0, peak_bonus = 0; + for (int32_t v = 0; v < pb.n_variables; ++v) { + const int8_t flip = (int8_t)(1 - 2 * assign[v]); + int32_t agg_base = 0, agg_bonus = 0; + for (int32_t i = pb.reverse_offsets[v]; i < pb.reverse_offsets[v + 1]; ++i) { + const fj_bin_row_t& h = rows[pb.reverse_constraints[i]]; + const int32_t s = h.sign; + const int32_t os = s * (h.bound - h.lhs); + const int32_t ns = os - s * ((int32_t)pb.reverse_coefficients[i] * flip); + int32_t base = 0, bonus = 0; + fj_bin_score_delta_parts(os, ns, h.weight, base, bonus); + agg_base += base; + agg_bonus += bonus; + } + const int32_t abs_base = agg_base < 0 ? -agg_base : agg_base; + const int32_t abs_bonus = agg_bonus < 0 ? -agg_bonus : agg_bonus; + if (abs_base > peak_base) peak_base = abs_base; + if (abs_bonus > peak_bonus) peak_bonus = abs_bonus; + } + max_aggregate_base = peak_base; + max_aggregate_bonus = peak_bonus; + } + + void set_violated(int32_t r) + { + if (!is_violated[r]) { + is_violated[r] = 1; + vpos[r] = (int32_t)violated_list.size(); + violated_list.push_back(r); + } + } + + void set_satisfied(int32_t r) + { + if (is_violated[r]) { + is_violated[r] = 0; + const int32_t p = vpos[r]; + const int32_t last = violated_list.back(); + violated_list[p] = last; + vpos[last] = p; + violated_list.pop_back(); + vpos[r] = -1; + } + } + + // Branchless score delta of flipping a variable, as seen by one row. base is the weighted change + // in satisfaction; bonus is the weighted change in strict slack. When both states are violated + // the improving direction earns half weight, matching excess_improvement_weight of 1/2. + int32_t score_delta(const fj_bin_row_t& h, int32_t lhs, int8_t delta, coef_t k) const + { + const int32_t s = h.sign; + const int32_t os = s * (h.bound - lhs); + const int32_t ns = os - s * ((int32_t)k * delta); + return fj_bin_packed_score_delta(os, ns, h.weight); + } + + void rebuild_scores() + { + std::fill(var_score.begin(), var_score.end(), 0); + for (int32_t r = 0; r < pb.n_constraints; ++r) { + const fj_bin_row_t& h = rows[r]; + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t v = pb.variables[k]; + const int32_t p = score_delta(h, h.lhs, (int8_t)(1 - 2 * assign[v]), pb.coefficients[k]); + nnz_score_delta[k] = p; + var_score[v] += p; + } + } + nnz_touched += pb.nnz; + } + + void recompute_lhs() + { + violated_list.clear(); + std::fill(is_violated.begin(), is_violated.end(), (uint8_t)0); + for (int32_t r = 0; r < pb.n_constraints; ++r) { + int32_t lhs = 0; + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) + lhs += (int32_t)pb.coefficients[k] * assign[pb.variables[k]]; + rows[r].lhs = lhs; + if (rows[r].sign * (rows[r].bound - lhs) < 0) set_violated(r); + } + incumbent_objective = 0; + for (int32_t v = 0; v < pb.n_variables; ++v) incumbent_objective += pb.objective[v] * assign[v]; + nnz_touched += pb.nnz; + rebuild_scores(); + } + + int32_t objective_terms(int32_t v, int8_t delta) const + { + const double obj_diff = pb.objective[v] * delta; + const int32_t base = obj_diff < 0 ? objective_weight : (obj_diff > 0 ? -objective_weight : 0); + int32_t bonus = 0; + const bool old_better = incumbent_objective < best_objective; + const bool new_better = incumbent_objective + obj_diff < best_objective; + if (!old_better && new_better) { + bonus += objective_weight; + } else if (old_better && !new_better) { + bonus -= objective_weight; + } + return base * fj_bin_score_k + bonus; + } + + int32_t full_score(int32_t v, int8_t delta) const + { + if (objective_weight == 0) return var_score[v]; + return var_score[v] + objective_terms(v, delta); + } + + bool tabu_blocked(int32_t v, bool localmin) const { return tabu.blocked(v, iters, localmin); } + + void apply_move(int32_t var, int8_t delta, fj_cpu_climber_t& climber) + { + const int8_t new_val = (int8_t)(assign[var] + delta); + const int8_t new_flip = (int8_t)(1 - 2 * new_val); + const int32_t ob = pb.reverse_offsets[var], oe = pb.reverse_offsets[var + 1]; + const int32_t prev_violated = (int32_t)violated_list.size(); + int32_t own_score = 0; + + for (int32_t ii = ob; ii < oe; ++ii) { + // Write hint: the row's lhs is updated at the end of every iteration, so the line is wanted + // exclusive. The padding on reverse_constraints makes the lookahead unconditional. + __builtin_prefetch(&rows[pb.reverse_constraints[ii + fj_bin_pf_dist]], 1, 3); + + const int32_t r = pb.reverse_constraints[ii]; + fj_bin_row_t& h = rows[r]; + const coef_t kv = pb.reverse_coefficients[ii]; + const int32_t old_lhs = h.lhs; + const int32_t new_lhs = old_lhs + (int32_t)kv * delta; + const int32_t s = h.sign; + const int32_t old_slack = s * (h.bound - old_lhs); + const int32_t new_slack = s * (h.bound - new_lhs); + + if (new_slack < 0 && old_slack >= 0) { + set_violated(r); + } else if (new_slack >= 0 && old_slack < 0) { + set_satisfied(r); + } + + // A row that stays clear of its boundary by more than max|coef| on both sides cannot change + // any variable's satisfaction flags, so its patch is skipped entirely. + const int32_t margin = h.cmax; + const bool deep_sat = old_slack > margin && new_slack > margin; + const bool deep_viol = old_slack < -margin && new_slack < -margin; + if (!(deep_sat || deep_viol)) { + const int32_t kb = pb.offsets[r], ke = pb.offsets[r + 1]; + // The offsets are already loaded for the call, so the width choice is a compare rather + // than a stored per-row flag. + // TODO: check that this may not cause AVX512 powerdown overheads if the AVX2 row/AVX512 row ratio is unbalanced + fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), + pb.variables.data(), + pb.coefficients.data(), + kb, + ke, + var_score.data(), + nnz_score_delta.data(), + assign_i32.data(), + s, + h.weight, + new_slack, + var); + nnz_touched += ke - kb; + nnz_patched += ke - kb; + } + + // The flipped variable's own score delta is provably zero when the row is deeply satisfied + // both ways, and already stored as zero there. + if (!deep_sat) { + const int32_t pv = score_delta(h, new_lhs, new_flip, kv); + own_score += pv; + nnz_score_delta[pb.reverse_to_csr[ii]] = pv; + } + h.lhs = new_lhs; + } + nnz_touched += oe - ob; + rows_walked += oe - ob; + + if (prev_violated > 0 && violated_list.empty()) last_feasible_entrance_iter = iters; + + assign[var] = new_val; + assign_i32[var] = new_val; + var_score[var] = own_score; + incumbent_objective += pb.objective[var] * delta; + + if (violated_list.empty() && incumbent_objective < best_objective) { + best_objective = incumbent_objective; + best_assign = assign; + feasible_found = true; + report_incumbent(climber); + } + + const int32_t tenure = + tabu_tenure_min + (int32_t)(rng.next_u32() % (uint32_t)(tabu_tenure_max - tabu_tenure_min)); + tabu.on_flip(var, iters, tenure); + std::fill(var_bitmap.begin(), var_bitmap.end(), (char)0); + } + + // Publish a new best into the climber, which owns the reporting contract. + void report_incumbent(fj_cpu_climber_t& climber) + { + auto& h_assign = climber.h_assignment; + auto& h_best = climber.h_best_assignment; + for (int32_t v = 0; v < pb.n_variables; ++v) { + h_assign[v] = (f_t)assign[v]; + h_best[v] = (f_t)assign[v]; + } + climber.h_incumbent_objective = (f_t)incumbent_objective; + climber.h_best_objective = (f_t)best_objective; + climber.feasible_found = true; + if (climber.improvement_callback) { + const double work_units = climber.work_units_elapsed.load(std::memory_order_acquire); + climber.improvement_callback((f_t)best_objective, h_best, work_units); + } + } + + void reweight_constraint(int32_t r, int32_t new_weight) + { + fj_bin_row_t& h = rows[r]; + if (new_weight == h.weight) return; + h.weight = new_weight; + if (new_weight > max_weight) max_weight = new_weight; + // lhs is unchanged here, and no variable is excluded, so skip_var matches no index. + const int32_t kb = pb.offsets[r], ke = pb.offsets[r + 1]; + fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), + pb.variables.data(), + pb.coefficients.data(), + kb, + ke, + var_score.data(), + nnz_score_delta.data(), + assign_i32.data(), + h.sign, + h.weight, + h.sign * (h.bound - h.lhs), + -1); + nnz_touched += ke - kb; + nnz_patched += ke - kb; + } + + // DDFW: every violated row gains weight taken from a satisfied neighbour above the donation + // floor, so total weight is roughly conserved and differentiation stays local to the hard region. + void update_weights() + { + for (int32_t cf : violated_list) { + reweight_constraint(cf, rows[cf].weight + fj_bin_ddfw_transfer); + const int32_t vo = pb.offsets[cf], ve = pb.offsets[cf + 1]; + if (ve <= vo) continue; + int32_t best_donor = -1, best_w = fj_bin_ddfw_init; + for (int32_t s = 0; s < fj_bin_ddfw_donor_samples; ++s) { + const int32_t v = pb.variables[vo + (int32_t)(rng.next_u32() % (uint32_t)(ve - vo))]; + const int32_t no = pb.reverse_offsets[v], ne = pb.reverse_offsets[v + 1]; + if (ne <= no) continue; + const int32_t d = + pb.reverse_constraints[no + (int32_t)(rng.next_u32() % (uint32_t)(ne - no))]; + if (d != cf && !is_violated[d] && rows[d].weight > best_w) { + best_w = rows[d].weight; + best_donor = d; + } + } + if (best_donor >= 0) reweight_constraint(best_donor, rows[best_donor].weight - fj_bin_ddfw_transfer); + } + if (violated_list.empty()) objective_weight += 1; + } + + // Global argmax over every variable, affordable because var_score is maintained live. While the + // objective weight is zero the full score is exactly var_score, which is the vectorized sweep's + // precondition; the objective and local-minimum paths fall to the scalar loop. + std::pair find_move_global(bool localmin) const + { + if (!localmin && objective_weight == 0) { + int32_t v = -1, s = fj_bin_score_invalid; + fj_bin_argmax(var_score.data(), + tabu.flip_until.data(), + pb.n_variables, + (uint16_t)(iters - tabu.base), + argmax_tile, + v, + s); + return {v, s}; + } + + int32_t best_v = -1, best_s = fj_bin_score_invalid; + for (int32_t v = 0; v < pb.n_variables; ++v) { + if (tabu_blocked(v, localmin)) continue; + const int32_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); + if (s > best_s) { + best_s = s; + best_v = v; + } + } + return {best_v, best_s}; + } + + std::pair find_move_in_rows(const std::vector& target_rows, + bool localmin) + { + int32_t best_v = -1, best_s = fj_bin_score_invalid; + for (int32_t r : target_rows) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t v = pb.variables[k]; + if (var_bitmap[v]) continue; + var_bitmap[v] = 1; + if (tabu_blocked(v, localmin)) continue; + const int32_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); + if (s > best_s) { + best_s = s; + best_v = v; + } + } + } + return {best_v, best_s}; + } + + std::pair find_move_violated(int32_t sample_size, bool localmin) + { + // Draw the rows directly instead of reservoir-sampling the violated list: `std::sample` is + // linear in the population, so it walked every violated row to keep a handful. Sampling with + // replacement is what `find_move_satisfied` already does, and `find_move_in_rows` deduplicates + // variables through `var_bitmap`, so a repeated row costs a bitmap sweep and no scoring. + const int32_t n = (int32_t)violated_list.size(); + const std::vector* sampled = &violated_list; + if (n > sample_size) { + sample_buf.clear(); + for (int32_t i = 0; i < sample_size; ++i) { + sample_buf.push_back(violated_list[rng.next_u32() % (uint32_t)n]); + } + sampled = &sample_buf; + } + auto move = find_move_in_rows(*sampled, localmin); + + // Breakthrough moves: once a feasible solution exists, allow objective-driven jumps. + if (feasible_found && incumbent_objective >= best_objective + breakthrough_margin) { + for (int32_t v : pb.objective_vars) { + const double step = (best_objective - incumbent_objective) / pb.objective[v]; + double target = pb.objective[v] > 0 ? std::floor(assign[v] + step) + : std::ceil(assign[v] + step); + if (target < 0) target = 0; + if (target > 1) target = 1; + if ((int8_t)target == assign[v]) continue; + if (tabu_blocked(v, false)) continue; + const int32_t s = full_score(v, (int8_t)((int8_t)target - assign[v])); + if (s > move.second) move = {v, s}; + } + } + return move; + } + + std::pair find_move_satisfied(int32_t sample_size) + { + sample_buf.clear(); + for (int32_t tries = 0; (int32_t)sample_buf.size() < sample_size && tries < sample_size * 8; + ++tries) { + const int32_t r = (int32_t)(rng.next_u32() % (uint32_t)pb.n_constraints); + if (!is_violated[r]) sample_buf.push_back(r); + } + return find_move_in_rows(sample_buf, false); + } + + std::pair find_lift_move() const + { + int32_t best_v = -1, best_s = 0; + for (int32_t v : pb.objective_vars) { + const int8_t delta = (int8_t)(1 - 2 * assign[v]); + if ((double)delta * pb.objective[v] >= 0) continue; + if (tabu_blocked(v, false)) continue; + const int32_t s = (int32_t)(-std::llround(pb.objective[v] * delta)) * fj_bin_score_k; + if (s > best_s) { + best_s = s; + best_v = v; + } + } + return {best_v, best_s}; + } + + void perturb() + { + if (pb.objective_vars.empty()) return; + const uint32_t n = (uint32_t)pb.objective_vars.size(); + for (int i = 0; i < 2; ++i) { + const int32_t v = pb.objective_vars[rng.next_u32() % n]; + assign[v] = (int8_t)(rng.next_u32() & 1u); + assign_i32[v] = assign[v]; + } + recompute_lhs(); + } + + // Restart returns the assignment to the seed the climber was constructed with, leaving the + // recorded best and the global iteration counter intact. + void do_restart() + { + assign = seed_assign; + for (int32_t v = 0; v < pb.n_variables; ++v) assign_i32[v] = assign[v]; + for (int32_t r = 0; r < pb.n_constraints; ++r) rows[r].weight = pb.initial_weight[r]; + max_weight = fj_bin_ddfw_init; + objective_weight = 0; + tabu.clear(iters); + recompute_lhs(); + last_restart_iter = iters; + last_feasible_entrance_iter = iters; + } + + void init(fj_cpu_climber_t& climber) + { + const auto& params = climber.settings.parameters; + seed = climber.settings.seed; + narrow4_max = fj_bin_simd_narrow4_max(); + narrow8_max = fj_bin_simd_narrow8_max(); + rng = raft::random::PCGenerator((uint64_t)seed, 0, 0); + tabu_tenure_min = params.tabu_tenure_min; + tabu_tenure_max = params.tabu_tenure_max; + breakthrough_margin = params.breakthrough_move_epsilon; + perturb_interval = climber.perturb_interval; + mtm_viol_samples = climber.mtm_viol_samples; + mtm_sat_samples = climber.mtm_sat_samples; + if (tabu_tenure_max <= tabu_tenure_min) tabu_tenure_max = tabu_tenure_min + 1; + + const int32_t n = pb.n_variables, m = pb.n_constraints; + const auto& h_assign = climber.h_assignment; + assign.resize(n); + for (int32_t v = 0; v < n; ++v) { + const double val = (double)h_assign[v]; + assign[v] = (int8_t)(val >= 0.5 ? 1 : 0); + } + seed_assign = assign; + best_assign = assign; + assign_i32.assign(n, 0); + for (int32_t v = 0; v < n; ++v) assign_i32[v] = assign[v]; + + rows.resize(m); + for (int32_t r = 0; r < m; ++r) + rows[r] = fj_bin_row_t{0, pb.initial_weight[r], pb.bound[r], pb.cmax[r], pb.sign[r]}; + + var_score.assign(n, 0); + nnz_score_delta.assign(pb.nnz + fj_bin_simd_padding(), 0); + tabu.resize(n); + is_violated.assign(m, 0); + vpos.assign(m, -1); + violated_list.clear(); + var_bitmap.assign(n, 0); + + objective_weight = 0; + max_weight = fj_bin_ddfw_init; + incumbent_objective = 0; + best_objective = std::numeric_limits::infinity(); + feasible_found = false; + iters = 0; + last_restart_iter = 0; + recompute_lhs(); + } + + void solve(fj_cpu_climber_t& climber, f_t time_limit, double work_unit_limit) override + { + init(climber); + + const auto loop_start = std::chrono::high_resolution_clock::now(); + const auto limit = + std::chrono::milliseconds((int64_t)std::floor((double)time_limit * 1000.0)); + const bool bounded_time = std::isfinite((double)time_limit); + + while (!climber.halted && !climber.preemption_flag.load()) { + if (bounded_time && std::chrono::high_resolution_clock::now() - loop_start > limit) break; + if (iters >= climber.settings.iteration_limit) break; + if (iters - last_restart_iter >= fj_bin_restart_period) do_restart(); + tabu.maybe_advance(iters); + + int32_t move_var = -1, score = fj_bin_score_invalid; + if (violated_list.empty()) std::tie(move_var, score) = find_lift_move(); + if (score <= 0) std::tie(move_var, score) = find_move_global(false); + if (feasible_found && score <= 0) std::tie(move_var, score) = find_move_satisfied(mtm_sat_samples); + + bool perturb_now = false; + if (violated_list.empty() && iters - last_feasible_entrance_iter > perturb_interval) { + perturb_now = true; + last_feasible_entrance_iter = iters; + } + + if (score > 0 && move_var >= 0 && !perturb_now) { + apply_move(move_var, (int8_t)(1 - 2 * assign[move_var]), climber); + } else { + update_weights(); + if (perturb_now) perturb(); + std::tie(move_var, score) = find_move_violated(1, true); + const int32_t v = move_var >= 0 ? move_var : 0; + apply_move(v, (int8_t)(1 - 2 * assign[v]), climber); + } + + if (iters % climber.log_interval == 0) { + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] iteration: %d, viol: %zu, best: %g, maxw: %d", + climber.log_prefix.c_str(), + coefficient_bits(), + iters, + violated_list.size(), + best_objective, + max_weight); + } + if (iters % climber.diversity_callback_interval == 0 && climber.diversity_callback) { + auto& h_assign = climber.h_assignment; + for (int32_t v = 0; v < pb.n_variables; ++v) h_assign[v] = (f_t)assign[v]; + climber.diversity_callback((f_t)incumbent_objective, h_assign); + } + + // Work-unit proxy. nnz_touched is cumulative, reproducing the accumulation shape the general + // path gets from its cumulative byte counters. + if (iters % 100 == 0 && iters > 0) { + const double work = (double)nnz_touched * fj_bin_bytes_per_nnz * climber.work_unit_bias / 1e10; + climber.work_units_elapsed.store(work, std::memory_order_release); + if (climber.producer_sync != nullptr) climber.producer_sync->notify_progress(); + if (work >= work_unit_limit) break; + } + + ++iters; + } + + compute_saturation(); + verify_incumbent(climber); + climber.iterations = (i_t)iters; + CUOPT_LOG_DEBUG( + "%sCPUFJ[bin%d] done: %d iterations, best %g, max weight %d, aggregate base %d/%d, bonus %d/%d", + climber.log_prefix.c_str(), + coefficient_bits(), + iters, + best_objective, + max_weight, + max_aggregate_base, + fj_bin_base_limit, + max_aggregate_bonus, + fj_bin_bonus_limit); + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] work: nnz_patched %lld, rows_walked %lld", + climber.log_prefix.c_str(), + coefficient_bits(), + (long long)nnz_patched, + (long long)rows_walked); + } +}; + +template +void fj_binary_state_deleter_t::operator()(fj_binary_state_t* ptr) const +{ + delete ptr; +} + +template +void try_build_binary_fastpath(fj_cpu_climber_t& climber) +{ + const fj_bin_scan_t scan = fj_bin_scan(climber); + if (scan.reject != fj_binary_reject_t::none) { + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s (row %d, var %d)", + climber.log_prefix.c_str(), + fj_binary_reject_name(scan.reject), + scan.bad_row, + scan.bad_var); + return; + } + + bool built = false; + if (scan.coefficient_bits == 8) { + auto engine = std::make_unique>(); + built = fj_bin_narrow(climber, scan.n_split_constraints, engine->pb); + if (built) climber.binary_fast.reset(engine.release()); + } else { + auto engine = std::make_unique>(); + built = fj_bin_narrow(climber, scan.n_split_constraints, engine->pb); + if (built) climber.binary_fast.reset(engine.release()); + } + + if (!built) { + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s", + climber.log_prefix.c_str(), + fj_binary_reject_name(fj_binary_reject_t::narrow_check_failed)); + return; + } + + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path enabled: int%d coefficients, %d rows after one-sided split", + climber.log_prefix.c_str(), + scan.coefficient_bits, + scan.n_split_constraints); +} + +#if MIP_INSTANTIATE_FLOAT +template struct fj_binary_state_deleter_t; +template void try_build_binary_fastpath(fj_cpu_climber_t& climber); +#endif + +#if MIP_INSTANTIATE_DOUBLE +template struct fj_binary_state_deleter_t; +template void try_build_binary_fastpath(fj_cpu_climber_t& climber); +#endif + +} // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh new file mode 100644 index 0000000000..fc3415e679 --- /dev/null +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -0,0 +1,169 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include +#include + +// Seam between the general CPU FJ path and the binary fast path. Only fj_cpu.cu includes this; +// fj_cpu.cuh forward-declares fj_binary_state_t so the climber can hold one without the general +// header depending on the fast path. +// +// The fast path applies to instances whose variables are all binary and whose rows carry integer +// coefficients within int8 or int16 range. On those it runs an integer engine: exact feasibility +// against a single row bound, a live per-variable score patched through stored per-nnz +// contributions, and a global argmax move selection. + +namespace cuopt::mathematical_optimization::mip { + +template +struct fj_cpu_climber_t; + +// Why an instance was refused, logged at DEBUG by the build entry. +enum class fj_binary_reject_t : uint8_t { + none, + empty_problem, + non_binary_var, + fractional_coefficient, + coefficient_out_of_range, + fractional_row_bound, + row_bound_out_of_range, + lhs_headroom, + narrow_check_failed, +}; + +const char* fj_binary_reject_name(fj_binary_reject_t reason); + +// Built state for one eligible instance. The concrete engine is templated on coefficient width +// (int8_t or int16_t) and lives in fj_cpu_binary.cu; the width choice is made once at build time, +// so dispatching through this base costs one virtual call per solve. +template +struct fj_binary_state_t { + virtual ~fj_binary_state_t() = default; + + virtual void solve(fj_cpu_climber_t& climber, + f_t time_limit, + double work_unit_limit) = 0; + + virtual int coefficient_bits() const = 0; // 8 or 16 + virtual int n_split_constraints() const = 0; + virtual i_t iterations() const = 0; + + // Largest per-variable aggregate base and bonus observed at end of solve. The packed score is + // order-preserving only while these stay under 2^16 and 2^14 respectively. + virtual void saturation(int& max_aggregate_base, int& max_aggregate_bonus) const = 0; +}; + +// Runs predicate -> one-sided split -> narrow. A pure function of the climber's host problem +// mirrors and its incoming constraint weights, so it is well defined wherever those are populated. +// Populates climber.binary_fast on success; leaves it empty and logs the reason otherwise. +template +void try_build_binary_fastpath(fj_cpu_climber_t& climber); + +// --------------------------------------------------------------------------------------------- +// Hot kernels and the score-delta formula they share with the engine. +// +// The kernels live in fj_cpu_binary_kernels.cpp, built with Google Highway, which compiles the +// bodies once per SIMD target and dispatches at runtime. That file is host-compiled: nvcc's +// frontend rejects Highway's x86 headers, which reinterpret-cast intrinsic vectors to +// compiler-specific vector types (GCC vector extensions in the constant-folding path, __m128bh +// for bfloat16). Every argument below is a plain pointer or scalar, so the seam names no cuOpt or +// CUDA type. +// --------------------------------------------------------------------------------------------- + +// Packed staged score: one int32 holding base * K + bonus. K exceeds twice the largest |bonus| the +// engine produces, so integer ordering on the packed word reproduces the lexicographic (base, +// bonus) ordering the general path gets from fj_staged_score_t. +constexpr int32_t fj_bin_score_shift = 15; +constexpr int32_t fj_bin_score_k = 1 << fj_bin_score_shift; +constexpr int32_t fj_bin_score_invalid = INT32_MIN; + +// Change in one row's weighted score when one variable flips, from the row's signed slack before +// (os) and after (ns) that flip. base is the weighted change in satisfaction; bonus is the +// weighted change in strict slack. When both states are violated the improving direction earns +// half weight, matching excess_improvement_weight of 1/2. +// +// Single source of the formula: the engine scores moves with it, compute_saturation walks it, and +// the vector kernels reproduce it lane-wise. +static inline void fj_bin_score_delta_parts( + int32_t os, int32_t ns, int32_t weight, int32_t& base, int32_t& bonus) +{ + const int32_t osat = os >= 0, nsat = ns >= 0; + const int32_t ost = os > 0, nst = ns > 0; + const int32_t improving = (os < ns) - (ns < os); + base = weight * (nsat - osat) + (1 - osat) * (1 - nsat) * improving * (weight / 2); + bonus = weight * (nst - ost); +} + +static inline int32_t fj_bin_packed_score_delta(int32_t os, int32_t ns, int32_t weight) +{ + int32_t base = 0, bonus = 0; + fj_bin_score_delta_parts(os, ns, weight, base, bonus); + return base * fj_bin_score_k + bonus; +} + +// Elements of padding the per-nnz arrays must carry past nnz, in int32 units. A target that masks +// its row remainder loads and stores a whole vector at the last row of the matrix, and the padding +// is what keeps that off memory it does not own. A target that peels the remainder into a scalar +// tail stops at the row end and asks for nothing, so this returns 0 there. +int32_t fj_bin_simd_padding(); + +// Vector width the row patch runs at. A gather costs the same whether its lanes carry data or are +// masked off, so a row filling only part of a native vector is cheaper through a narrower one; past +// the crossover the extra vector and its extra full gather cost more than the idle lanes. +enum class fj_bin_patch_width_t : int32_t { narrow4 = 0, narrow8 = 1, native = 2 }; + +// Longest row for which each narrower width beats the native one, or 0 where that width is not +// worth offering on this target: a width is offered only when it is strictly narrower than the +// native vector, and scalable targets decline both. +int32_t fj_bin_simd_narrow4_max(); +int32_t fj_bin_simd_narrow8_max(); + +static inline fj_bin_patch_width_t fj_bin_patch_width_for(int32_t row_len, + int32_t narrow4_max, + int32_t narrow8_max) +{ + if (row_len <= narrow4_max) return fj_bin_patch_width_t::narrow4; + if (row_len <= narrow8_max) return fj_bin_patch_width_t::narrow8; + return fj_bin_patch_width_t::native; +} + +// Patch every variable of one row against the row's current signed slack, skipping skip_var. The +// move case passes the post-move slack and the flipped variable's index; the reweight case passes +// the unchanged slack and -1, which matches no variable index. +// +// Reads and writes up to fj_bin_simd_padding() elements from kb, so variables, coefficients and +// nnz_score_delta must carry that padding. +// +// Defined in the host-compiled kernels TU and explicitly instantiated there for int8_t and int16_t. +template +void fj_bin_patch_row(fj_bin_patch_width_t width, + const int32_t* variables, + const coef_t* coefficients, + int32_t kb, + int32_t ke, + int32_t* var_score, + int32_t* nnz_score_delta, + const int32_t* assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var); + +// Argmax over var_score with the tabu window folded in, scanning all n variables. Valid while the +// objective weight is zero, where the full score is exactly var_score. Yields best_var of -1 when +// every variable is tabu. +void fj_bin_argmax(const int32_t* var_score, + const uint16_t* flip_until, + int32_t n, + uint16_t iter_biased, + int32_t tile, + int32_t& best_var, + int32_t& best_score); + +} // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp new file mode 100644 index 0000000000..a91858beff --- /dev/null +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -0,0 +1,449 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +// Hot kernels of the binary CPU FJ fast path, vectorized with Google Highway. foreach_target.h +// re-includes this file once per SIMD target; HWY_EXPORT builds the dispatch table and +// HWY_DYNAMIC_DISPATCH picks at runtime. Host-compiled rather than nvcc-compiled: nvcc's frontend +// rejects Highway's x86 headers, which reinterpret-cast intrinsic vectors to compiler-specific +// vector types. + +#include + +#include +#include + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp" +#include "hwy/foreach_target.h" // must precede highway.h +#include "hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); +namespace cuopt::mathematical_optimization::mip { +namespace HWY_NAMESPACE { + +namespace hn = hwy::HWY_NAMESPACE; + +// Whether the row remainder is masked into the vector body or peeled into a scalar tail. AVX-512 +// k-registers, SVE predicates and RVV masks make every operation maskable at no cost, so a row of +// three nonzeros is one masked iteration; peeling it would send most of the work to the tail, since +// row lengths are short and have nothing to do with the lane count. AVX2 and NEON have no mask +// registers: the mask becomes a vector, gather and scatter are emulated, and the tail is cheaper. +// Measured on AVX2, masking the remainder cost 6.8% on supportcase22 and 12.9% on bnatt400. +constexpr bool k_mask_remainder = + (HWY_TARGET <= HWY_AVX3) || HWY_TARGET_IS_SVE || (HWY_TARGET == HWY_RVV); + +// Padding the caller must carry past nnz. Only a masking target reads past a row's end; a peeling +// target stops at ke, so it asks for nothing and its buffers keep their natural size. +int32_t PaddingImpl() +{ + return k_mask_remainder ? (int32_t)hn::Lanes(hn::ScalableTag()) : 0; +} + +// Row remainder when it is peeled rather than masked, and the whole row on scalar targets. +template +void PatchRowScalar(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int32_t* HWY_RESTRICT var_score, + int32_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + for (int32_t k = kb; k < ke; ++k) { + const int32_t v = variables[k]; + if (v == skip_var) continue; + const int32_t flip = 1 - 2 * assign_i32[v]; + const int32_t ns = os_new - sign * ((int32_t)coefficients[k] * flip); + const int32_t nc = fj_bin_packed_score_delta(os_new, ns, weight); + var_score[v] += nc - nnz_score_delta[k]; + nnz_score_delta[k] = nc; + } +} + +// Templated on the vector tag so one body serves both the native-width kernel and the narrow one. +// Rows here average well under a native 512-bit vector, and a gather costs the same whether its +// lanes are used or discarded, so short rows are cheaper through a narrower vector. +template +static HWY_INLINE void PatchRowBody(D d, + const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int32_t* HWY_RESTRICT var_score, + int32_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + const hn::Rebind dc; // same lane count, narrower lanes + using V = hn::Vec; + const size_t N = hn::Lanes(d); + + // When the remainder is peeled, a row below one vector never reaches the body, so it skips the + // ten broadcasts below as well. + if constexpr (!k_mask_remainder) { + if ((size_t)(ke - kb) < N) { + PatchRowScalar(variables, coefficients, kb, ke, var_score, nnz_score_delta, + assign_i32, sign, weight, os_new, skip_var); + return; + } + } + + const V vone = hn::Set(d, 1), vzero = hn::Zero(d); + const V vsign = hn::Set(d, sign), vskip = hn::Set(d, skip_var); + const V vos = hn::Set(d, os_new); + const V vw = hn::Set(d, weight), vw2 = hn::Set(d, weight / 2); + + // The row's own slack is uniform across lanes, so its flags are scalars. + const int32_t osat = os_new >= 0, ost = os_new > 0; + const V vosat = hn::Set(d, osat), vost = hn::Set(d, ost); + const V v_not_osat = hn::Set(d, 1 - osat); + + // The loads always run unmasked and read into the per-nnz padding; when the remainder is masked, + // FirstN keeps the overhang out of the gather, the scatter and the store. + const int32_t vec_end = k_mask_remainder ? ke : ke - (int32_t)N + 1; + int32_t k = kb; + for (; k < vec_end; k += (int32_t)N) { + const V v = hn::LoadU(d, variables + k); + auto active = hn::Ne(v, vskip); + if constexpr (k_mask_remainder) { + active = hn::And(active, hn::FirstN(d, (size_t)(ke - k))); + } + + // Gathered in hardware even on Zen 4, unlike the score update below. Doing this one by lane + // instead measured 8.2% slower: it must spill the index vector and reload it 4 bytes at a time, + // which cannot store-to-load forward, and that cost 959 interlocks per iteration against 72. + // The score update escapes this because it already needs the spill for its read-modify-write. + const V a01 = hn::MaskedGatherIndex(active, d, assign_i32, v); + const V flip = vone - hn::ShiftLeft<1>(a01); + const V coef = hn::PromoteTo(d, hn::LoadU(dc, coefficients + k)); + + const V ns = vos - vsign * coef * flip; + + const V nsat = hn::IfThenElseZero(hn::Ge(ns, vzero), vone); + const V nst = hn::IfThenElseZero(hn::Gt(ns, vzero), vone); + const V improving = + hn::IfThenElseZero(hn::Gt(ns, vos), vone) - hn::IfThenElseZero(hn::Lt(ns, vos), vone); + + const V both_violated = v_not_osat * (vone - nsat); + const V base = vw * (nsat - vosat) + both_violated * improving * vw2; + const V bonus = vw * (nst - vost); + const V packed_new = hn::ShiftLeft(base) + bonus; + + const V delta = packed_new - hn::LoadU(d, nnz_score_delta + k); + +#if HWY_TARGET == HWY_AVX3_ZEN4 + // zmm VSIB is microcode on Zen 4: VPGATHERDD ~76-80 uops / ~21 CPI and VPSCATTERDD 89 / 24, + // against ~5 / ~10 and ~19 / ~11 on SPR-class Intel (Agner Fog, uops.info). So read-modify-write + // by lane here; measured +5.8% over the arm below on an EPYC 9554 (supportcase22, 16 climbers). + HWY_ALIGN int32_t idx[hn::MaxLanes(d)], dl[hn::MaxLanes(d)]; + hn::Store(v, d, idx); + hn::Store(delta, d, dl); + // Bounded by the row, not the vector: the lanes past it hold padding, whose zero index would + // otherwise be applied to variable 0. + const size_t lanes = HWY_MIN(N, (size_t)(ke - k)); + for (size_t i = 0; i < lanes; ++i) { + if (idx[i] != skip_var) var_score[idx[i]] += dl[i]; + } +#else + const V current = hn::MaskedGatherIndex(active, d, var_score, v); + hn::MaskedScatterIndex(current + delta, active, d, var_score, v); +#endif + + hn::BlendedStore(packed_new, active, d, nnz_score_delta + k); + } + + if constexpr (!k_mask_remainder) { + PatchRowScalar(variables, coefficients, k, ke, var_score, nnz_score_delta, assign_i32, + sign, weight, os_new, skip_var); + } +} + +// Native width, and the 8-lane variant for rows that would leave most of a native vector idle. +template +void PatchRowImpl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int32_t* HWY_RESTRICT var_score, + int32_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + PatchRowBody(hn::ScalableTag(), variables, coefficients, kb, ke, var_score, + nnz_score_delta, assign_i32, sign, weight, os_new, skip_var); +} + +template +void PatchRowNarrow8Impl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int32_t* HWY_RESTRICT var_score, + int32_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + PatchRowBody(hn::CappedTagIfFixed(), variables, coefficients, kb, ke, + var_score, nnz_score_delta, assign_i32, sign, weight, os_new, skip_var); +} + +template +void PatchRowNarrow4Impl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int32_t* HWY_RESTRICT var_score, + int32_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + PatchRowBody(hn::CappedTagIfFixed(), variables, coefficients, kb, ke, + var_score, nnz_score_delta, assign_i32, sign, weight, os_new, skip_var); +} + +// Longest row worth sending to each narrower kernel, or 0 where that width is not worth having. +// A gather costs the same whether its lanes carry data or are masked off, so a row that fills only +// part of a native vector is cheaper through a narrower one; past the crossover the extra vector +// and its extra full gather cost more than the wasted lanes. From the Zen 4 microcode ratio +// (VPGATHERDD ~78 uops at 512 bits, 48 at 256, 24 at 128) the crossovers land at 4 and 8. +// +// A width is offered only when it is strictly narrower than the native vector, so no target ever +// dispatches to a kernel identical to its own. Scalable targets opt out entirely: Highway notes +// that clamping Lanes() on RVV/SVE can cost more than the capping saves, which is why +// CappedTagIfFixed leaves them at native width above. +int32_t Narrow4MaxImpl() +{ + if (HWY_HAVE_SCALABLE) return 0; + return hn::Lanes(hn::ScalableTag()) > 4 ? 4 : 0; +} + +int32_t Narrow8MaxImpl() +{ + if (HWY_HAVE_SCALABLE) return 0; + return hn::Lanes(hn::ScalableTag()) > 8 ? 8 : 0; +} + +// Tiled sweep carrying a running maximum. The index re-scan fires only on a tile that raises it, +// and that tile is still cache-hot. The tabu window is uint16 against int32 scores, so the mask +// crosses a 2:1 width boundary through PromoteMaskTo. +void ArgmaxImpl(const int32_t* HWY_RESTRICT var_score, + const uint16_t* HWY_RESTRICT flip_until, + int32_t n, + uint16_t iter_biased, + int32_t tile, + int32_t* best_var, + int32_t* best_score) +{ + const hn::ScalableTag d; + const hn::Rebind d16; + using V = hn::Vec; + + const int32_t step = (int32_t)hn::Lanes(d); + const V vmin = hn::Set(d, fj_bin_score_invalid); + const auto viter = hn::Set(d16, iter_biased); + + // Whole vectors only; the remainder is scanned scalar below. + const int32_t nblk = n - (n % step); + int32_t tile_step = tile - (tile % step); + if (tile_step < step) tile_step = step; + + int32_t bv = -1, bs = fj_bin_score_invalid; + + for (int32_t t0 = 0; t0 < nblk; t0 += tile_step) { + const int32_t t1 = (t0 + tile_step < nblk) ? t0 + tile_step : nblk; + + V tile_max = vmin; + for (int32_t v = t0; v < t1; v += step) { + const auto tabu = + hn::PromoteMaskTo(d, d16, hn::Lt(viter, hn::LoadU(d16, flip_until + v))); + tile_max = hn::Max(tile_max, hn::IfThenElse(tabu, vmin, hn::LoadU(d, var_score + v))); + } + + const int32_t peak = hn::ReduceMax(d, tile_max); + if (peak > bs) { + const V vpeak = hn::Set(d, peak); + for (int32_t v = t0; v < t1; v += step) { + const auto tabu = + hn::PromoteMaskTo(d, d16, hn::Lt(viter, hn::LoadU(d16, flip_until + v))); + const V s = hn::IfThenElse(tabu, vmin, hn::LoadU(d, var_score + v)); + const intptr_t lane = hn::FindFirstTrue(d, hn::Eq(s, vpeak)); + if (lane >= 0) { + bv = v + (int32_t)lane; + break; + } + } + bs = peak; + } + } + + for (int32_t v = nblk; v < n; ++v) { + if (iter_biased < flip_until[v]) continue; + if (var_score[v] > bs) { + bs = var_score[v]; + bv = v; + } + } + + *best_var = bv; + *best_score = bs; +} + +} // namespace HWY_NAMESPACE +} // namespace cuopt::mathematical_optimization::mip +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE +namespace cuopt::mathematical_optimization::mip { + +// One dispatch table per (coefficient width, vector width). HWY_EXPORT_T names the table +// separately from the function, which lets the function be a template-id: only the table name goes +// through token pasting, so no hand-written non-template wrapper is needed. The template argument +// must stay comma-free, which is why the three tag-binding wrappers above take only coef_t. +HWY_EXPORT_T(PatchRowNatI8, PatchRowImpl); +HWY_EXPORT_T(PatchRowN8I8, PatchRowNarrow8Impl); +HWY_EXPORT_T(PatchRowN4I8, PatchRowNarrow4Impl); +HWY_EXPORT_T(PatchRowNatI16, PatchRowImpl); +HWY_EXPORT_T(PatchRowN8I16, PatchRowNarrow8Impl); +HWY_EXPORT_T(PatchRowN4I16, PatchRowNarrow4Impl); +HWY_EXPORT(ArgmaxImpl); +HWY_EXPORT(PaddingImpl); +HWY_EXPORT(Narrow4MaxImpl); +HWY_EXPORT(Narrow8MaxImpl); + +// HWY_DYNAMIC_DISPATCH resolves the target on every call, and the hwy::GetChosenTarget() call it +// expands to is a real out-of-line call: it clobbers the argument registers, so the compiler spills +// all eleven parameters to the stack and reloads them around it. These run once per row per move, +// so the pointers are resolved once instead. +// +// Entry 0 of a dispatch table is a trampoline that chooses the target and re-dispatches, and an +// unchosen target makes GetIndex() return 0. Caching then would pin that extra indirection for the +// process lifetime, so the target is chosen first. File scope rather than function scope keeps the +// guard variable of a magic static out of the call: its cold path can call __cxa_guard_acquire, so +// the compiler must preserve the arguments across it and cannot leave a bare tail jump. Nothing in +// cuOpt reaches feasibility jump during static initialization. +static void fj_bin_choose_target() +{ + if (!hwy::GetChosenTarget().IsInitialized()) { + hwy::GetChosenTarget().Update(hwy::SupportedTargets()); + } +} + +template +using fj_bin_patch_fn_t = void (*)(const int32_t*, + const coef_t*, + int32_t, + int32_t, + int32_t*, + int32_t*, + const int32_t*, + int32_t, + int32_t, + int32_t, + int32_t); + +// Indexed by fj_bin_patch_width_t. +static const fj_bin_patch_fn_t fj_bin_patch_i8[3] = { + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN4I8)), + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN8I8)), + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowNatI8)), +}; + +static const fj_bin_patch_fn_t fj_bin_patch_i16[3] = { + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN4I16)), + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN8I16)), + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowNatI16)), +}; + +// Overloaded rather than specialized so the tables stay plain arrays. +static const fj_bin_patch_fn_t* fj_bin_patch_table(int8_t) { return fj_bin_patch_i8; } +static const fj_bin_patch_fn_t* fj_bin_patch_table(int16_t) { return fj_bin_patch_i16; } + +static const auto fj_bin_argmax_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(ArgmaxImpl)); +static const auto fj_bin_padding_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(PaddingImpl)); +static const auto fj_bin_narrow4_max_fn = + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow4MaxImpl)); +static const auto fj_bin_narrow8_max_fn = + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow8MaxImpl)); + +int32_t fj_bin_simd_padding() { return fj_bin_padding_fn(); } +int32_t fj_bin_simd_narrow4_max() { return fj_bin_narrow4_max_fn(); } +int32_t fj_bin_simd_narrow8_max() { return fj_bin_narrow8_max_fn(); } + +template +void fj_bin_patch_row(fj_bin_patch_width_t width, + const int32_t* variables, + const coef_t* coefficients, + int32_t kb, + int32_t ke, + int32_t* var_score, + int32_t* nnz_score_delta, + const int32_t* assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + fj_bin_patch_table(coef_t{})[(int)width](variables, coefficients, kb, ke, var_score, + nnz_score_delta, assign_i32, sign, weight, os_new, + skip_var); +} + +template void fj_bin_patch_row(fj_bin_patch_width_t, + const int32_t*, + const int8_t*, + int32_t, + int32_t, + int32_t*, + int32_t*, + const int32_t*, + int32_t, + int32_t, + int32_t, + int32_t); + +template void fj_bin_patch_row(fj_bin_patch_width_t, + const int32_t*, + const int16_t*, + int32_t, + int32_t, + int32_t*, + int32_t*, + const int32_t*, + int32_t, + int32_t, + int32_t, + int32_t); + +void fj_bin_argmax(const int32_t* var_score, + const uint16_t* flip_until, + int32_t n, + uint16_t iter_biased, + int32_t tile, + int32_t& best_var, + int32_t& best_score) +{ + fj_bin_argmax_fn(var_score, flip_until, n, iter_biased, tile, &best_var, &best_score); +} + +} // namespace cuopt::mathematical_optimization::mip +#endif // HWY_ONCE diff --git a/cpp/src/utilities/version_info.cpp b/cpp/src/utilities/version_info.cpp index 71dfc20c22..2b1d3919d9 100644 --- a/cpp/src/utilities/version_info.cpp +++ b/cpp/src/utilities/version_info.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include + #include #include #include @@ -143,6 +146,23 @@ static std::string get_cpu_model() return "Unknown"; } +// The SIMD instruction set the vectorized CPU kernels actually dispatched to, which reflects the +// compiled target set as well as what this CPU supports. Highway names the AVX-512 family AVX3, +// with AVX3_DL / AVX3_ZEN4 / AVX3_SPR / AVX10_2 marking successively newer feature sets; those are +// reported under the name users know them by. +static const char* get_simd_target() +{ + const int64_t target = hwy::DispatchedTarget(); + switch (target) { + case HWY_AVX3: + case HWY_AVX3_DL: + case HWY_AVX3_ZEN4: + case HWY_AVX3_SPR: + case HWY_AVX10_2: return "AVX-512"; + default: return hwy::TargetName(target); + } +} + struct host_memory_info_t { double total_gb{}; double available_gb{}; @@ -201,6 +221,7 @@ void print_version_info(int num_devices) std::thread::hardware_concurrency(), memory.available_gb, memory.total_gb); + CUOPT_LOG_INFO("CPU SIMD target: %s", get_simd_target()); for (int device_id = 0; device_id < num_devices; ++device_id) { cudaDeviceProp device_prop{}; From ac9d47e5b960c7ca92854f4b4ad6b265c66470d9 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Thu, 6 Aug 2026 05:44:15 -0700 Subject: [PATCH 02/61] Take the tabu test out of the binary fast path's global argmax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep read two bytes of tenure per variable to answer a question that is true for at most a dozen of them: one variable flips per iteration and tenure is bounded, so only that many are ever tabu. It now reads var_score alone and the caller holds the tabu few at the invalid sentinel across the call, which is the same value the masked select wrote before. That drops the scan from six bytes per variable to four. On crypt16 it takes the argmax working set from 31 KB, which overflows a 32 KB L1, to 21 KB, which does not, and total L2 requests fall 59% (685 to 256 per iteration) — more than the byte count alone predicts, because below the threshold the var_score reads stop missing too. ArgmaxImpl goes from 31.2% of cycles to 23.0%. The tabu set is a ring of the last 16 flips indexed by iteration rather than by expiry. Indexing by expiry is cheaper — position encodes the deadline, so aging is a shift and no comparison is needed — but two flips can share a deadline and the later would silently displace the earlier; simulation over the [3,12] tenure range puts that at 35% of insertions. Indexing by iteration cannot collide, since exactly one flip happens per iteration. on_flip drops any earlier entry for the variable before inserting. Without that the ring holds a superseded deadline: flip_until keeps one deadline per variable and a reflip overwrites it, possibly with an earlier one, while the ring would keep both and block on the later. A variable can be reflipped while still tabu because the local-minimum path tests the weaker iter == last_flip + 1. The first version of this patch missed it and blocked too much, which showed up as fewer crossings at higher throughput; with the fix the crypt16 trajectory is identical to before, the same ten climbers crossing 10-12% earlier. Restores run in reverse: a variable flipped twice inside the ring appears twice, and its second save holds the sentinel written by the first. Measured at 16 climbers: crypt16 +8.4%, supportcase22 +1.1%, bnatt400 -2.9%. The gain tracks n_variables, since it is a residency effect; bnatt400's 401 variables were already resident and only pay the ring's bookkeeping. Co-Authored-By: Claude Opus 5 --- .../feasibility_jump/fj_cpu_binary.cu | 93 +++++++++++++++++-- .../feasibility_jump/fj_cpu_binary.cuh | 11 ++- .../fj_cpu_binary_kernels.cpp | 18 +--- 3 files changed, 94 insertions(+), 28 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 5c5a28d9f5..d4eeab6ac3 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -56,11 +56,25 @@ struct fj_bin_tabu_t { std::vector last_flip; int32_t base{0}; + // Ring of the last ring_size flips, indexed by iteration rather than by expiry. One variable + // flips per iteration, so each slot takes exactly one entry and nothing is ever displaced while + // still tabu; indexing by expiry instead would collide whenever two flips share a deadline, which + // a simulation over the [3,12] tenure range puts at 35% of insertions. + // + // This is what lets the global argmax stop reading flip_until. Tenure is bounded by the ring, so + // at most ring_size variables are tabu at once out of n: reading two bytes per variable to answer + // a question that is almost always no costs a third of that scan's traffic. On crypt16 it is the + // difference between a 31 KB working set that overflows a 32 KB L1 and a 21 KB one that does not. + static constexpr int32_t ring_size = 16; + int32_t ring_var[ring_size]; + int32_t ring_expiry[ring_size]; + void resize(int32_t n) { flip_until.assign(n, 0); last_flip.assign(n, 0); + clear_ring(); base = 0; } @@ -68,13 +82,67 @@ struct fj_bin_tabu_t { { std::fill(flip_until.begin(), flip_until.end(), (uint16_t)0); std::fill(last_flip.begin(), last_flip.end(), 0); + clear_ring(); base = iter; } + void clear_ring() + { + for (int32_t i = 0; i < ring_size; ++i) { + ring_var[i] = -1; + ring_expiry[i] = 0; + } + } + void on_flip(int32_t v, int32_t iter, int32_t tenure) { flip_until[v] = (uint16_t)(iter + tenure - base); last_flip[v] = iter; + + // Drop any earlier entry for v before inserting the new one. flip_until keeps one deadline per + // variable and a reflip overwrites it, so without this the ring would hold a superseded, later + // deadline and block v past the point the per-variable test would have released it. A variable + // can be reflipped while still tabu: the local-minimum path tests the weaker + // iter == last_flip + 1. With this the ring holds at most one live entry per variable carrying + // its current deadline, which is exactly the invariant flip_until maintains. + for (int32_t i = 0; i < ring_size; ++i) { + if (ring_var[i] == v) ring_var[i] = -1; + } + + const int32_t slot = iter & (ring_size - 1); + ring_var[slot] = v; + ring_expiry[slot] = iter + tenure; + } + + // Blocks every currently-tabu variable by writing the invalid sentinel over its score, and reports + // how many were touched. Paired with unblock around one argmax and nothing else: the score is + // maintained incrementally, so it may only be disturbed across a window in which no patch runs. + int32_t block_tabu(int32_t iter, + int32_t* var_score, + int32_t (&saved_var)[ring_size], + int32_t (&saved_score)[ring_size]) const + { + int32_t k = 0; + for (int32_t i = 0; i < ring_size; ++i) { + const int32_t v = ring_var[i]; + if (v >= 0 && ring_expiry[i] > iter) { + saved_var[k] = v; + saved_score[k] = var_score[v]; + var_score[v] = fj_bin_score_invalid; + ++k; + } + } + return k; + } + + // Reverse order on purpose: a variable flipped twice inside the ring appears twice, and its second + // save holds the sentinel written by the first. Unwinding backwards restores the true score last. + static void unblock_tabu(int32_t k, + int32_t* var_score, + const int32_t (&saved_var)[ring_size], + const int32_t (&saved_score)[ring_size]) + { + for (int32_t i = k - 1; i >= 0; --i) var_score[saved_var[i]] = saved_score[i]; } @@ -784,17 +852,18 @@ struct fj_bin_engine_t : fj_binary_state_t { // Global argmax over every variable, affordable because var_score is maintained live. While the // objective weight is zero the full score is exactly var_score, which is the vectorized sweep's // precondition; the objective and local-minimum paths fall to the scalar loop. - std::pair find_move_global(bool localmin) const + std::pair find_move_global(bool localmin) { if (!localmin && objective_weight == 0) { + // The sweep reads var_score alone; the handful of tabu variables are held at the invalid + // sentinel across it rather than tested per variable. + int32_t saved_var[fj_bin_tabu_t::ring_size], saved_score[fj_bin_tabu_t::ring_size]; + const int32_t blocked = tabu.block_tabu(iters, var_score.data(), saved_var, saved_score); + int32_t v = -1, s = fj_bin_score_invalid; - fj_bin_argmax(var_score.data(), - tabu.flip_until.data(), - pb.n_variables, - (uint16_t)(iters - tabu.base), - argmax_tile, - v, - s); + fj_bin_argmax(var_score.data(), pb.n_variables, argmax_tile, v, s); + + fj_bin_tabu_t::unblock_tabu(blocked, var_score.data(), saved_var, saved_score); return {v, s}; } @@ -933,6 +1002,14 @@ struct fj_bin_engine_t : fj_binary_state_t { mtm_sat_samples = climber.mtm_sat_samples; if (tabu_tenure_max <= tabu_tenure_min) tabu_tenure_max = tabu_tenure_min + 1; + // The tabu ring is indexed by iteration modulo its size, so a slot is reused after ring_size + // iterations. A tenure that long would be overwritten while the variable is still tabu, and the + // argmax would stop excluding it. Clamped as well as asserted: release builds compile the assert + // out, and silently dropping tabu entries is worse than a shorter tenure. + cuopt_assert(tabu_tenure_max <= fj_bin_tabu_t::ring_size, + "tabu tenure exceeds the tabu ring, live entries would be evicted"); + if (tabu_tenure_max > fj_bin_tabu_t::ring_size) tabu_tenure_max = fj_bin_tabu_t::ring_size; + const int32_t n = pb.n_variables, m = pb.n_constraints; const auto& h_assign = climber.h_assignment; assign.resize(n); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh index fc3415e679..6fcb53eea5 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -155,13 +155,14 @@ void fj_bin_patch_row(fj_bin_patch_width_t width, int32_t os_new, int32_t skip_var); -// Argmax over var_score with the tabu window folded in, scanning all n variables. Valid while the -// objective weight is zero, where the full score is exactly var_score. Yields best_var of -1 when -// every variable is tabu. +// Argmax over var_score, scanning all n variables. Valid while the objective weight is zero, where +// the full score is exactly var_score. Yields best_var of -1 only if n is 0. +// +// Carries no tabu argument: at most a ring's worth of variables are tabu at once, so the caller +// holds those few at fj_bin_score_invalid across the call instead of making this read a per-variable +// tenure array. That keeps the scan to four bytes per variable. void fj_bin_argmax(const int32_t* var_score, - const uint16_t* flip_until, int32_t n, - uint16_t iter_biased, int32_t tile, int32_t& best_var, int32_t& best_score); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index a91858beff..64541a8777 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -247,20 +247,16 @@ int32_t Narrow8MaxImpl() // and that tile is still cache-hot. The tabu window is uint16 against int32 scores, so the mask // crosses a 2:1 width boundary through PromoteMaskTo. void ArgmaxImpl(const int32_t* HWY_RESTRICT var_score, - const uint16_t* HWY_RESTRICT flip_until, int32_t n, - uint16_t iter_biased, int32_t tile, int32_t* best_var, int32_t* best_score) { const hn::ScalableTag d; - const hn::Rebind d16; using V = hn::Vec; const int32_t step = (int32_t)hn::Lanes(d); const V vmin = hn::Set(d, fj_bin_score_invalid); - const auto viter = hn::Set(d16, iter_biased); // Whole vectors only; the remainder is scanned scalar below. const int32_t nblk = n - (n % step); @@ -274,19 +270,14 @@ void ArgmaxImpl(const int32_t* HWY_RESTRICT var_score, V tile_max = vmin; for (int32_t v = t0; v < t1; v += step) { - const auto tabu = - hn::PromoteMaskTo(d, d16, hn::Lt(viter, hn::LoadU(d16, flip_until + v))); - tile_max = hn::Max(tile_max, hn::IfThenElse(tabu, vmin, hn::LoadU(d, var_score + v))); + tile_max = hn::Max(tile_max, hn::LoadU(d, var_score + v)); } const int32_t peak = hn::ReduceMax(d, tile_max); if (peak > bs) { const V vpeak = hn::Set(d, peak); for (int32_t v = t0; v < t1; v += step) { - const auto tabu = - hn::PromoteMaskTo(d, d16, hn::Lt(viter, hn::LoadU(d16, flip_until + v))); - const V s = hn::IfThenElse(tabu, vmin, hn::LoadU(d, var_score + v)); - const intptr_t lane = hn::FindFirstTrue(d, hn::Eq(s, vpeak)); + const intptr_t lane = hn::FindFirstTrue(d, hn::Eq(hn::LoadU(d, var_score + v), vpeak)); if (lane >= 0) { bv = v + (int32_t)lane; break; @@ -297,7 +288,6 @@ void ArgmaxImpl(const int32_t* HWY_RESTRICT var_score, } for (int32_t v = nblk; v < n; ++v) { - if (iter_biased < flip_until[v]) continue; if (var_score[v] > bs) { bs = var_score[v]; bv = v; @@ -435,14 +425,12 @@ template void fj_bin_patch_row(fj_bin_patch_width_t, int32_t); void fj_bin_argmax(const int32_t* var_score, - const uint16_t* flip_until, int32_t n, - uint16_t iter_biased, int32_t tile, int32_t& best_var, int32_t& best_score) { - fj_bin_argmax_fn(var_score, flip_until, n, iter_biased, tile, &best_var, &best_score); + fj_bin_argmax_fn(var_score, n, tile, &best_var, &best_score); } } // namespace cuopt::mathematical_optimization::mip From 4f4bcb1d461c5c92042fc81ce73df9ac772045d5 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Thu, 6 Aug 2026 09:47:06 -0700 Subject: [PATCH 03/61] perf(cpufj-bin): drop the per-iteration bitmap clear and hoist apply_move's base pointers Two findings from a Zen 4 profile of supportcase22 (16 climbers, 20 s): 3.537M -> 3.619M aggregate iterations/s, +2.3%. apply_move ended by clearing var_bitmap in full, which GCC emitted as a tail jump to memset: 1.50% of the run spent zeroing 6,489 bytes per iteration for a guard that only find_move_in_rows reads, on the sampling paths. That function now clears the entries it set, which costs 0.20%. This is not bit-identical. find_move_satisfied and find_move_violated can both run in one iteration, and the second used to inherit the first's dedup bits and skip variables it had already seen. The carryover was accidental -- the comment there only justifies dedup within a single call -- but it was real behaviour. Only a climber that has crossed diverges; on supportcase22 the crossings hold at 1/16 and the objective moves 117 -> 116. The loop in apply_move also reloaded eleven .data() pointers out of `this` per row visit, the top three at ~1.4% each. That is aliasing, not register pressure: the body writes h.lhs through fj_bin_row_t* and nnz_score_delta[..] through int32_t*, either of which may alias a vector's internal pointer. Hoisting them into const locals removes every this-relative load from the loop and lets GCC walk reverse_constraints and reverse_coefficients by pointer induction. +1.0%, trajectory bit-identical. Co-Authored-By: Claude Opus 5 --- .../feasibility_jump/fj_cpu_binary.cu | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index d4eeab6ac3..5b2ffae2eb 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -475,6 +475,8 @@ struct fj_bin_engine_t : fj_binary_state_t { std::vector is_violated; std::vector violated_list; std::vector vpos; + // Duplicate guard for find_move_in_rows, its only reader. Zero everywhere outside that function, + // which clears what it set before returning. std::vector var_bitmap; // One generator advanced across the whole search, rather than one re-seeded per call site per @@ -704,14 +706,29 @@ struct fj_bin_engine_t : fj_binary_state_t { const int32_t prev_violated = (int32_t)violated_list.size(); int32_t own_score = 0; + // The loop writes a row's lhs through fj_bin_row_t* and a score delta through int32_t*, either + // of which may alias a vector's internal pointer as far as the compiler can prove. Without + // these locals it reloads every base pointer below out of `this` on each row visit, and cannot + // turn the index walks into pointer inductions. + fj_bin_row_t* const rows_p = rows.data(); + const int32_t* const rcon_p = pb.reverse_constraints.data(); + const coef_t* const rcoef_p = pb.reverse_coefficients.data(); + const int32_t* const rcsr_p = pb.reverse_to_csr.data(); + const int32_t* const offsets_p = pb.offsets.data(); + const int32_t* const vars_p = pb.variables.data(); + const coef_t* const coefs_p = pb.coefficients.data(); + int32_t* const var_score_p = var_score.data(); + int32_t* const nnz_delta_p = nnz_score_delta.data(); + const int32_t* const assign_p = assign_i32.data(); + for (int32_t ii = ob; ii < oe; ++ii) { // Write hint: the row's lhs is updated at the end of every iteration, so the line is wanted // exclusive. The padding on reverse_constraints makes the lookahead unconditional. - __builtin_prefetch(&rows[pb.reverse_constraints[ii + fj_bin_pf_dist]], 1, 3); + __builtin_prefetch(&rows_p[rcon_p[ii + fj_bin_pf_dist]], 1, 3); - const int32_t r = pb.reverse_constraints[ii]; - fj_bin_row_t& h = rows[r]; - const coef_t kv = pb.reverse_coefficients[ii]; + const int32_t r = rcon_p[ii]; + fj_bin_row_t& h = rows_p[r]; + const coef_t kv = rcoef_p[ii]; const int32_t old_lhs = h.lhs; const int32_t new_lhs = old_lhs + (int32_t)kv * delta; const int32_t s = h.sign; @@ -730,18 +747,18 @@ struct fj_bin_engine_t : fj_binary_state_t { const bool deep_sat = old_slack > margin && new_slack > margin; const bool deep_viol = old_slack < -margin && new_slack < -margin; if (!(deep_sat || deep_viol)) { - const int32_t kb = pb.offsets[r], ke = pb.offsets[r + 1]; + const int32_t kb = offsets_p[r], ke = offsets_p[r + 1]; // The offsets are already loaded for the call, so the width choice is a compare rather // than a stored per-row flag. // TODO: check that this may not cause AVX512 powerdown overheads if the AVX2 row/AVX512 row ratio is unbalanced fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), - pb.variables.data(), - pb.coefficients.data(), + vars_p, + coefs_p, kb, ke, - var_score.data(), - nnz_score_delta.data(), - assign_i32.data(), + var_score_p, + nnz_delta_p, + assign_p, s, h.weight, new_slack, @@ -755,7 +772,7 @@ struct fj_bin_engine_t : fj_binary_state_t { if (!deep_sat) { const int32_t pv = score_delta(h, new_lhs, new_flip, kv); own_score += pv; - nnz_score_delta[pb.reverse_to_csr[ii]] = pv; + nnz_delta_p[rcsr_p[ii]] = pv; } h.lhs = new_lhs; } @@ -779,7 +796,6 @@ struct fj_bin_engine_t : fj_binary_state_t { const int32_t tenure = tabu_tenure_min + (int32_t)(rng.next_u32() % (uint32_t)(tabu_tenure_max - tabu_tenure_min)); tabu.on_flip(var, iters, tenure); - std::fill(var_bitmap.begin(), var_bitmap.end(), (char)0); } // Publish a new best into the climber, which owns the reporting contract. @@ -896,6 +912,11 @@ struct fj_bin_engine_t : fj_binary_state_t { } } } + // Restore the all-zero invariant by revisiting only what was set: the sampled rows hold a few + // dozen variables against n in the thousands, so this is far cheaper than clearing the array. + for (int32_t r : target_rows) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) var_bitmap[pb.variables[k]] = 0; + } return {best_v, best_s}; } From f334465014e1c33046492c010da9eb36458e7c4e Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Thu, 6 Aug 2026 12:10:22 -0700 Subject: [PATCH 04/61] perf(cpufj-bin): vectorize apply_move's row walk supportcase22 goes from 3.598M to 4.348M aggregate iterations/s, +20.6%, with the search trajectory bit-identical. 85% of row visits leave their row deeply satisfied on both sides of the flip and do nothing but advance its slack. That part is uniform and now runs in Highway: fj_bin_walk_rows advances a tile of a variable's incidences and returns the ones that need scalar attention, which the caller finishes. The layout is what makes it cheap. Rows now store sign * (bound - lhs) rather than lhs, in a standalone row_slack, so the update is new_slack = old_slack - signed_coefficient[i] * delta and neither bound nor lhs appears. signed_coefficient and incident_row_cmax are replicated per incidence, so the kernel reads them at unit stride and the only irregular access left is row_slack itself: one gather and one scatter per vector, against four gathers and a scatter for a literal SoA split of the row record. That also empties fj_bin_row_t down to weight and sign. Only deep_sat is tested in the kernel. deep_viol stays with the caller because it fires on 0.02% of visits but guards the widest rows in the matrix -- deleting it outright cost 1.0% on supportcase22, since those rows average ~2,100 nonzeros and it was suppressing 16% of all patch work. The caller tiles rather than interleaving its tail into the vector loop: the lane loop sits behind a dispatch pointer that cannot be inlined, while the tail needs engine state the kernels TU cannot reach, so tiling is what keeps the patch calls outside it without a per-group callback. Trajectory verified by diffing the per-climber (iteration, viol, best, maxw) traces: identical multisets across all 16 climbers to 3,000,000 iterations each, 48,016 log points. A scalar arm for short incidence ranges was tried and dropped. Sweeping the degree below which apply_move walked the rows itself, bnatt400 degraded monotonically 14.43M -> 14.19M as the threshold went 0 -> 64, and crypt16 and supportcase22 were flat. bnatt400 (-1.1%) and crypt16 (-3.1%) do regress, but not for that reason, and the cause is still open. Co-Authored-By: Claude Opus 5 --- .../feasibility_jump/fj_cpu_binary.cu | 191 ++++++++++-------- .../feasibility_jump/fj_cpu_binary.cuh | 52 +++++ .../fj_cpu_binary_kernels.cpp | 142 +++++++++++++ 3 files changed, 305 insertions(+), 80 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 5b2ffae2eb..48aac555e8 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -162,15 +162,17 @@ struct fj_bin_tabu_t { } }; -// One row of the narrowed problem. bound/sign encode the single finite bound the split leaves: -// sign +1 for lhs <= bound, -1 for lhs >= bound. cmax is max|coef| over the row, which bounds how -// far a single flip can move the slack. +// What the apply path still needs per row once the row walk is vectorized: everything else it used +// to read from here now reaches it at unit stride. +// +// The mutable state moved out to the engine's row_slack, which holds sign * (bound - lhs) rather +// than lhs, because that is what turns the walk's update into a subtraction and lets it gather one +// array instead of four. bound went with it -- only the rebuild paths need it, and they read +// pb.bound. cmax went to pb.incident_row_cmax, replicated per incidence. sign stays because +// fj_bin_patch_row still takes it, and weight because it is genuinely mutable. template struct fj_bin_row_t { - int32_t lhs; int32_t weight; - int32_t bound; - coef_t cmax; int8_t sign; }; @@ -190,6 +192,12 @@ struct fj_bin_problem_t { std::vector reverse_coefficients; std::vector reverse_to_csr; + // Per incidence, for the vectorized row walk: sign * coefficient folded once, and the row's cmax + // replicated. Both are structural. Indexed like the transpose above, so the walk reads them at + // unit stride instead of gathering sign, coefficient and cmax per row. + std::vector signed_coefficient; + std::vector incident_row_cmax; + std::vector bound; std::vector sign; std::vector cmax; @@ -432,6 +440,8 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, pb.reverse_constraints.resize(pb.nnz); pb.reverse_coefficients.resize(pb.nnz); pb.reverse_to_csr.resize(pb.nnz); + pb.signed_coefficient.resize(pb.nnz); + pb.incident_row_cmax.resize(pb.nnz); { std::vector cursor(pb.reverse_offsets.begin(), pb.reverse_offsets.begin() + n); for (int32_t r = 0; r < n_split; ++r) { @@ -440,11 +450,20 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, pb.reverse_constraints[slot] = r; pb.reverse_coefficients[slot] = pb.coefficients[k]; pb.reverse_to_csr[slot] = k; + // The scan admits int8 only up to |coef| 127 and int16 only up to 32767, so negating a + // coefficient cannot overflow its own width. + pb.signed_coefficient[slot] = (coef_t)(pb.sign[r] * pb.coefficients[k]); + pb.incident_row_cmax[slot] = pb.cmax[r]; } } } - // Lookahead room for the row-walk prefetch. Reads land on row 0, which is prefetched harmlessly. - pb.reverse_constraints.resize(pb.nnz + fj_bin_pf_dist, 0); + // Lookahead room for the row walk: a vector of overhang for the kernel's unit-stride loads, and + // the prefetch distance the scalar path uses. Reads land on row 0, harmlessly, and every lane past + // a variable's range is masked out of the gather, the scatter and the compress. + const int32_t rpad = fj_bin_pf_dist > pad ? fj_bin_pf_dist : pad; + pb.reverse_constraints.resize(pb.nnz + rpad, 0); + pb.signed_coefficient.resize(pb.nnz + rpad, (coef_t)0); + pb.incident_row_cmax.resize(pb.nnz + rpad, (coef_t)1); pb.objective.resize(n); for (int32_t v = 0; v < n; ++v) { @@ -462,6 +481,10 @@ struct fj_bin_engine_t : fj_binary_state_t { fj_bin_problem_t pb; std::vector> rows; + // Per row, sign * (bound - lhs): negative exactly when the row is violated, and moved by a flip + // by exactly -signed_coefficient. The only mutable state the vectorized walk gathers. + std::vector row_slack; + std::vector assign; std::vector best_assign; std::vector seed_assign; // restart target @@ -509,6 +532,7 @@ struct fj_bin_engine_t : fj_binary_state_t { int32_t narrow4_max{0}; int32_t narrow8_max{0}; + // Settings read at solve entry, where the climber carries populated values. int32_t seed{0}; int32_t tabu_tenure_min{3}; @@ -593,9 +617,8 @@ struct fj_bin_engine_t : fj_binary_state_t { int32_t agg_base = 0, agg_bonus = 0; for (int32_t i = pb.reverse_offsets[v]; i < pb.reverse_offsets[v + 1]; ++i) { const fj_bin_row_t& h = rows[pb.reverse_constraints[i]]; - const int32_t s = h.sign; - const int32_t os = s * (h.bound - h.lhs); - const int32_t ns = os - s * ((int32_t)pb.reverse_coefficients[i] * flip); + const int32_t os = row_slack[pb.reverse_constraints[i]]; + const int32_t ns = os - (int32_t)pb.signed_coefficient[i] * flip; int32_t base = 0, bonus = 0; fj_bin_score_delta_parts(os, ns, h.weight, base, bonus); agg_base += base; @@ -632,25 +655,18 @@ struct fj_bin_engine_t : fj_binary_state_t { } } - // Branchless score delta of flipping a variable, as seen by one row. base is the weighted change - // in satisfaction; bonus is the weighted change in strict slack. When both states are violated - // the improving direction earns half weight, matching excess_improvement_weight of 1/2. - int32_t score_delta(const fj_bin_row_t& h, int32_t lhs, int8_t delta, coef_t k) const - { - const int32_t s = h.sign; - const int32_t os = s * (h.bound - lhs); - const int32_t ns = os - s * ((int32_t)k * delta); - return fj_bin_packed_score_delta(os, ns, h.weight); - } - void rebuild_scores() { std::fill(var_score.begin(), var_score.end(), 0); for (int32_t r = 0; r < pb.n_constraints; ++r) { - const fj_bin_row_t& h = rows[r]; + const int32_t weight = rows[r].weight; + const int32_t sign = rows[r].sign; + const int32_t os = row_slack[r]; for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { - const int32_t v = pb.variables[k]; - const int32_t p = score_delta(h, h.lhs, (int8_t)(1 - 2 * assign[v]), pb.coefficients[k]); + const int32_t v = pb.variables[k]; + const int32_t flip = 1 - 2 * assign[v]; + const int32_t ns = os - sign * ((int32_t)pb.coefficients[k] * flip); + const int32_t p = fj_bin_packed_score_delta(os, ns, weight); nnz_score_delta[k] = p; var_score[v] += p; } @@ -658,7 +674,7 @@ struct fj_bin_engine_t : fj_binary_state_t { nnz_touched += pb.nnz; } - void recompute_lhs() + void recompute_slack() { violated_list.clear(); std::fill(is_violated.begin(), is_violated.end(), (uint8_t)0); @@ -666,8 +682,9 @@ struct fj_bin_engine_t : fj_binary_state_t { int32_t lhs = 0; for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) lhs += (int32_t)pb.coefficients[k] * assign[pb.variables[k]]; - rows[r].lhs = lhs; - if (rows[r].sign * (rows[r].bound - lhs) < 0) set_violated(r); + const int32_t slack = pb.sign[r] * (pb.bound[r] - lhs); + row_slack[r] = slack; + if (slack < 0) set_violated(r); } incumbent_objective = 0; for (int32_t v = 0; v < pb.n_variables; ++v) incumbent_objective += pb.objective[v] * assign[v]; @@ -706,13 +723,14 @@ struct fj_bin_engine_t : fj_binary_state_t { const int32_t prev_violated = (int32_t)violated_list.size(); int32_t own_score = 0; - // The loop writes a row's lhs through fj_bin_row_t* and a score delta through int32_t*, either - // of which may alias a vector's internal pointer as far as the compiler can prove. Without - // these locals it reloads every base pointer below out of `this` on each row visit, and cannot - // turn the index walks into pointer inductions. + // The tail writes a score delta through int32_t* and calls out to the patch, either of which may + // alias a vector's internal pointer as far as the compiler can prove. Without these locals it + // reloads every base pointer below out of `this` on each visit. fj_bin_row_t* const rows_p = rows.data(); + int32_t* const slack_p = row_slack.data(); const int32_t* const rcon_p = pb.reverse_constraints.data(); - const coef_t* const rcoef_p = pb.reverse_coefficients.data(); + const coef_t* const skv_p = pb.signed_coefficient.data(); + const coef_t* const rcmax_p = pb.incident_row_cmax.data(); const int32_t* const rcsr_p = pb.reverse_to_csr.data(); const int32_t* const offsets_p = pb.offsets.data(); const int32_t* const vars_p = pb.variables.data(); @@ -721,60 +739,72 @@ struct fj_bin_engine_t : fj_binary_state_t { int32_t* const nnz_delta_p = nnz_score_delta.data(); const int32_t* const assign_p = assign_i32.data(); - for (int32_t ii = ob; ii < oe; ++ii) { - // Write hint: the row's lhs is updated at the end of every iteration, so the line is wanted - // exclusive. The padding on reverse_constraints makes the lookahead unconditional. - __builtin_prefetch(&rows_p[rcon_p[ii + fj_bin_pf_dist]], 1, 3); - + // Everything a visit still needs once its slack has been advanced. Shared by the two arms below + // so the walk's shape is the only thing that differs between them. + auto finish = [&](int32_t ii) { const int32_t r = rcon_p[ii]; fj_bin_row_t& h = rows_p[r]; - const coef_t kv = rcoef_p[ii]; - const int32_t old_lhs = h.lhs; - const int32_t new_lhs = old_lhs + (int32_t)kv * delta; - const int32_t s = h.sign; - const int32_t old_slack = s * (h.bound - old_lhs); - const int32_t new_slack = s * (h.bound - new_lhs); + const int32_t skv = (int32_t)skv_p[ii]; + const int32_t new_slack = slack_p[r]; + const int32_t old_slack = new_slack + skv * delta; + // A row can only cross its boundary if the flip moves it by at least the distance to it, so + // every transition is inside this list and none was lost with the rows the walk absorbed. if (new_slack < 0 && old_slack >= 0) { set_violated(r); } else if (new_slack >= 0 && old_slack < 0) { set_satisfied(r); } - // A row that stays clear of its boundary by more than max|coef| on both sides cannot change - // any variable's satisfaction flags, so its patch is skipped entirely. - const int32_t margin = h.cmax; - const bool deep_sat = old_slack > margin && new_slack > margin; - const bool deep_viol = old_slack < -margin && new_slack < -margin; - if (!(deep_sat || deep_viol)) { - const int32_t kb = offsets_p[r], ke = offsets_p[r + 1]; - // The offsets are already loaded for the call, so the width choice is a compare rather - // than a stored per-row flag. - // TODO: check that this may not cause AVX512 powerdown overheads if the AVX2 row/AVX512 row ratio is unbalanced - fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), - vars_p, - coefs_p, - kb, - ke, - var_score_p, - nnz_delta_p, - assign_p, - s, - h.weight, - new_slack, - var); + // The mirror of the walk's deep_sat test. Kept here rather than there because it fires on + // 0.02% of visits and guards the widest rows in the matrix: measured, moving it into the + // vector loop costs more in the 85% case than it saves in the 0.02% one. + const int32_t margin = (int32_t)rcmax_p[ii]; + if (!(old_slack < -margin && new_slack < -margin)) { + const int32_t kb = offsets_p[r], ke = offsets_p[r + 1]; + // The offsets are already loaded for the call, so the width choice is a compare rather + // than a stored per-row flag. + // TODO: check that this may not cause AVX512 powerdown overheads if the AVX2 row/AVX512 row ratio is unbalanced + fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), + vars_p, + coefs_p, + kb, + ke, + var_score_p, + nnz_delta_p, + assign_p, + h.sign, + h.weight, + new_slack, + var); nnz_touched += ke - kb; nnz_patched += ke - kb; } - // The flipped variable's own score delta is provably zero when the row is deeply satisfied - // both ways, and already stored as zero there. - if (!deep_sat) { - const int32_t pv = score_delta(h, new_lhs, new_flip, kv); - own_score += pv; - nnz_delta_p[rcsr_p[ii]] = pv; - } - h.lhs = new_lhs; + // The flipped variable's own score delta. Zero on the rows the walk absorbed -- deeply + // satisfied both ways -- and already stored as zero there. + const int32_t pv = fj_bin_packed_score_delta(new_slack, new_slack - skv * new_flip, h.weight); + own_score += pv; + nnz_delta_p[rcsr_p[ii]] = pv; + }; + + // A tile at a time: the kernel advances every slack in the tile and reports back only the visits + // that left the row within reach of its boundary, which on supportcase22 is 15.1% of them. The + // buffer is a stack array rather than one sized to the widest reverse degree because the tail + // runs between tiles, which is also what keeps the patch calls out of the vector loop. + // + // Unconditional: a scalar arm for short ranges was tried and never won. Sweeping the degree + // below which apply_move walked the rows itself, bnatt400 degraded monotonically from 14.43M to + // 14.19M iterations/s as the threshold rose from 0 to 64, and crypt16 and supportcase22 were + // flat. At a median degree of 13 and 7 respectively, one gather still beats that many dependent + // scalar load-modify-stores, because it breaks the dependence chain through row_slack rather + // than following it. + int32_t tile_incidence[fj_bin_walk_tile]; + for (int32_t t0 = ob; t0 < oe; t0 += fj_bin_walk_tile) { + const int32_t t1 = (t0 + fj_bin_walk_tile < oe) ? t0 + fj_bin_walk_tile : oe; + const int32_t n_tail = + fj_bin_walk_rows(slack_p, rcon_p, skv_p, rcmax_p, t0, t1, delta, tile_incidence); + for (int32_t j = 0; j < n_tail; ++j) finish(tile_incidence[j]); } nnz_touched += oe - ob; rows_walked += oe - ob; @@ -822,7 +852,7 @@ struct fj_bin_engine_t : fj_binary_state_t { if (new_weight == h.weight) return; h.weight = new_weight; if (new_weight > max_weight) max_weight = new_weight; - // lhs is unchanged here, and no variable is excluded, so skip_var matches no index. + // The slack is unchanged here, and no variable is excluded, so skip_var matches no index. const int32_t kb = pb.offsets[r], ke = pb.offsets[r + 1]; fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), pb.variables.data(), @@ -834,7 +864,7 @@ struct fj_bin_engine_t : fj_binary_state_t { assign_i32.data(), h.sign, h.weight, - h.sign * (h.bound - h.lhs), + row_slack[r], -1); nnz_touched += ke - kb; nnz_patched += ke - kb; @@ -990,7 +1020,7 @@ struct fj_bin_engine_t : fj_binary_state_t { assign[v] = (int8_t)(rng.next_u32() & 1u); assign_i32[v] = assign[v]; } - recompute_lhs(); + recompute_slack(); } // Restart returns the assignment to the seed the climber was constructed with, leaving the @@ -1003,7 +1033,7 @@ struct fj_bin_engine_t : fj_binary_state_t { max_weight = fj_bin_ddfw_init; objective_weight = 0; tabu.clear(iters); - recompute_lhs(); + recompute_slack(); last_restart_iter = iters; last_feasible_entrance_iter = iters; } @@ -1045,7 +1075,8 @@ struct fj_bin_engine_t : fj_binary_state_t { rows.resize(m); for (int32_t r = 0; r < m; ++r) - rows[r] = fj_bin_row_t{0, pb.initial_weight[r], pb.bound[r], pb.cmax[r], pb.sign[r]}; + rows[r] = fj_bin_row_t{pb.initial_weight[r], pb.sign[r]}; + row_slack.assign(m, 0); var_score.assign(n, 0); nnz_score_delta.assign(pb.nnz + fj_bin_simd_padding(), 0); @@ -1062,7 +1093,7 @@ struct fj_bin_engine_t : fj_binary_state_t { feasible_found = false; iters = 0; last_restart_iter = 0; - recompute_lhs(); + recompute_slack(); } void solve(fj_cpu_climber_t& climber, f_t time_limit, double work_unit_limit) override diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh index 6fcb53eea5..d6daf9c160 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -155,6 +155,58 @@ void fj_bin_patch_row(fj_bin_patch_width_t width, int32_t os_new, int32_t skip_var); +// Incidences handed to fj_bin_walk_rows per call, and so the size of the caller's output buffer. +// Large enough that the dynamic dispatch and the call's setup amortize away -- a variable of +// supportcase22 averages 344 incidences, so one or two calls per move -- and small enough that the +// buffer is a stack array and its touched part stays in L1. A multiple of every supported lane +// count, so only a variable's last tile has a masked remainder. +constexpr int32_t fj_bin_walk_tile = 256; + +// Advance every row incident to one flipped variable, and report which of those visits the caller +// must finish by hand. +// +// An "incidence" is one (variable, row) pair of the matrix, so the reverse CSR indexed by a +// variable's [incidence_begin, incidence_end) names the rows that variable appears in. Three arrays +// are read at that index, all at unit stride: +// +// incident_row[i] the row this incidence touches +// signed_coefficient[i] sign * coefficient, folded once at build time +// incident_row_cmax[i] that row's max|coef|, replicated here so it need not be gathered +// +// The last two are structural and fixed for the life of the solve. row_slack[r] is indexed by row, +// and holds sign * (bound - lhs) -- the signed slack the engine stores in place of lhs, which is +// what reduces the update to a subtraction. delta is the flip direction, +1 or -1, uniform over the +// call. +// +// For every incidence i in the range this applies +// row_slack[incident_row[i]] -= signed_coefficient[i] * delta +// then writes to out_incidence, in increasing order, the subset of i whose row is not deeply +// satisfied on both sides of the flip -- 15.1% of visits on supportcase22 -- and returns how many. +// Everything those visits still need is indirect (the row's weight, the own-score delta, the +// violated-set transitions, the patch) and stays with the caller. +// +// The caller drives this a tile at a time, running each tile's tail before asking for the next, so +// out_incidence is a fj_bin_walk_tile-sized buffer the caller can keep on its stack rather than one +// sized to the widest reverse degree in the matrix. Tiling this way rather than interleaving the +// tail into the vector loop keeps every patch call outside it: the loop over lanes lives here, +// behind a dynamic dispatch pointer that cannot be inlined, while the tail needs engine-side state +// this translation unit cannot reach. A tile still runs the vector body many times before yielding, +// which is what lets consecutive gathers overlap. +// +// out_incidence must hold incidence_end - incidence_begin entries. Reads up to +// fj_bin_simd_padding() elements past incidence_end from the three per-incidence arrays. Interior +// tiles read into the next tile, which is in bounds; the final one reads past the variable's range, +// so those arrays carry that padding. +template +int32_t fj_bin_walk_rows(int32_t* row_slack, + const int32_t* incident_row, + const coef_t* signed_coefficient, + const coef_t* incident_row_cmax, + int32_t incidence_begin, + int32_t incidence_end, + int32_t delta, + int32_t* out_incidence); + // Argmax over var_score, scanning all n variables. Valid while the objective weight is zero, where // the full score is exactly var_score. Yields best_var of -1 only if n is 0. // diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index 64541a8777..90affda3fb 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -43,6 +43,108 @@ int32_t PaddingImpl() return k_mask_remainder ? (int32_t)hn::Lanes(hn::ScalableTag()) : 0; } +// Whether the row walk below is worth vectorizing on this target. It needs a real gather to read the +// slacks and a real compress to emit the tail list; where either is emulated the emulation costs +// more than the scalar loop it replaces, since 85% of visits do nothing but subtract and compare. +// The scalar arm still returns the same list, so the caller needs no second code path -- it pays +// only one store per reported visit. +constexpr bool k_vector_walk = + (HWY_TARGET <= HWY_AVX3) || HWY_TARGET_IS_SVE || (HWY_TARGET == HWY_RVV); + +// One tile of a flipped variable's incidence range, vectorized. The caller tiles the range and runs +// each tile's tail before asking for the next; see fj_bin_walk_tile. +// +// Measured on supportcase22: 84.87% of row visits leave the row deeply satisfied on both sides of +// the flip, and those visits do nothing but update the slack. The remaining 15.13% need the row's +// weight, the flipped variable's own score delta, the violated-set transitions and usually a +// patch -- all indirect, all awkward in a vector. So this kernel does only the uniform part and +// hands back the indices of the visits that are not deep_sat, in increasing order, for the caller +// to finish scalar. +// +// The layout this assumes is what makes it worth doing. Storing the row's signed slack rather than +// its lhs collapses the update to +// +// new_slack = old_slack - sign * coef * delta = old_slack - skv[ii] * delta +// +// so bound and lhs never appear, and sign and coef fold into one per-incidence constant. skv and +// cmax are replicated per incidence, which makes them unit-stride loads. What remains irregular is +// the slack itself: one gather and one scatter per vector, against four gathers and a scatter for a +// literal SoA split of the row record. +// +// Trajectory is preserved exactly. The slack update is per row and order-independent; the caller's +// tail visits its indices in the same order the scalar loop did; and a deep_sat row is never read by +// the tail, so updating it early is not observable. +template +int32_t WalkRowsImpl(int32_t* HWY_RESTRICT row_slack, + const int32_t* HWY_RESTRICT incident_row, + const coef_t* HWY_RESTRICT signed_coefficient, + const coef_t* HWY_RESTRICT incident_row_cmax, + int32_t incidence_begin, + int32_t incidence_end, + int32_t delta, + int32_t* HWY_RESTRICT out_incidence) +{ + int32_t n_out = 0; + int32_t ii = incidence_begin; + + if constexpr (k_vector_walk) { + const hn::ScalableTag d; + const hn::Rebind dc; // same lane count, narrower lanes + using V = hn::Vec; + const size_t N = hn::Lanes(d); + + const V vdelta = hn::Set(d, delta); + + // The unit-stride loads always run whole and read into the per-incidence padding; FirstN keeps + // the overhang out of the gather, the scatter and the compress. + for (; ii < incidence_end; ii += (int32_t)N) { + const auto active = hn::FirstN(d, (size_t)(incidence_end - ii)); + + const V rows = hn::LoadU(d, incident_row + ii); + const V skv = hn::PromoteTo(d, hn::LoadU(dc, signed_coefficient + ii)); + const V cmax = hn::PromoteTo(d, hn::LoadU(dc, incident_row_cmax + ii)); + + const V os = hn::MaskedGatherIndex(active, d, row_slack, rows); + const V ns = hn::Sub(os, hn::Mul(skv, vdelta)); + + // Only the satisfied side. deep_viol is the caller's business: it fires on 0.02% of visits but + // guards the widest rows in the matrix, so it belongs where the row length is already known. + const auto deep_sat = hn::And(hn::Gt(os, cmax), hn::Gt(ns, cmax)); + const auto to_tail = hn::AndNot(deep_sat, active); + +#if HWY_TARGET == HWY_AVX3_ZEN4 + // Same Zen 4 microcode argument as the score scatter in PatchRowBody: VPSCATTERDD is 89 uops + // at ~24 CPI, against two vector stores and N scalar stores here. Unlike that one this is a + // pure store with no read-modify-write, so it needs its own A/B before the arm is settled. + HWY_ALIGN int32_t row_lane[hn::MaxLanes(d)], slack_lane[hn::MaxLanes(d)]; + hn::Store(rows, d, row_lane); + hn::Store(ns, d, slack_lane); + const size_t lanes = HWY_MIN(N, (size_t)(incidence_end - ii)); + for (size_t i = 0; i < lanes; ++i) row_slack[row_lane[i]] = slack_lane[i]; +#else + hn::MaskedScatterIndex(ns, active, d, row_slack, rows); +#endif + + // A variable meets each row at most once, so no two lanes carry the same row and neither the + // scatter above nor the store loop needs conflict detection. + n_out += (int32_t)hn::CompressStore(hn::Iota(d, ii), to_tail, d, out_incidence + n_out); + } + return n_out; + } + + // Targets without a native gather or compress. Also the remainder is not reached here: the loop + // above runs to oe under FirstN, and this arm replaces it wholesale rather than tailing it. + for (; ii < incidence_end; ++ii) { + const int32_t row = incident_row[ii]; + const int32_t os = row_slack[row]; + const int32_t ns = os - (int32_t)signed_coefficient[ii] * delta; + row_slack[row] = ns; + const int32_t cmax = (int32_t)incident_row_cmax[ii]; + if (!(os > cmax && ns > cmax)) out_incidence[n_out++] = ii; + } + return n_out; +} + // Row remainder when it is peeled rather than masked, and the whole row on scalar targets. template void PatchRowScalar(const int32_t* HWY_RESTRICT variables, @@ -312,6 +414,8 @@ namespace cuopt::mathematical_optimization::mip { HWY_EXPORT_T(PatchRowNatI8, PatchRowImpl); HWY_EXPORT_T(PatchRowN8I8, PatchRowNarrow8Impl); HWY_EXPORT_T(PatchRowN4I8, PatchRowNarrow4Impl); +HWY_EXPORT_T(WalkRowsI8, WalkRowsImpl); +HWY_EXPORT_T(WalkRowsI16, WalkRowsImpl); HWY_EXPORT_T(PatchRowNatI16, PatchRowImpl); HWY_EXPORT_T(PatchRowN8I16, PatchRowNarrow8Impl); HWY_EXPORT_T(PatchRowN4I16, PatchRowNarrow4Impl); @@ -368,6 +472,18 @@ static const fj_bin_patch_fn_t fj_bin_patch_i16[3] = { static const fj_bin_patch_fn_t* fj_bin_patch_table(int8_t) { return fj_bin_patch_i8; } static const fj_bin_patch_fn_t* fj_bin_patch_table(int16_t) { return fj_bin_patch_i16; } +template +using fj_bin_walk_fn_t = int32_t (*)( + int32_t*, const int32_t*, const coef_t*, const coef_t*, int32_t, int32_t, int32_t, int32_t*); + +static const auto fj_bin_walk_i8 = + (fj_bin_choose_target(), (fj_bin_walk_fn_t)HWY_DYNAMIC_POINTER_T(WalkRowsI8)); +static const auto fj_bin_walk_i16 = + (fj_bin_choose_target(), (fj_bin_walk_fn_t)HWY_DYNAMIC_POINTER_T(WalkRowsI16)); + +static fj_bin_walk_fn_t fj_bin_walk_fn(int8_t) { return fj_bin_walk_i8; } +static fj_bin_walk_fn_t fj_bin_walk_fn(int16_t) { return fj_bin_walk_i16; } + static const auto fj_bin_argmax_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(ArgmaxImpl)); static const auto fj_bin_padding_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(PaddingImpl)); static const auto fj_bin_narrow4_max_fn = @@ -376,6 +492,32 @@ static const auto fj_bin_narrow8_max_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow8MaxImpl)); int32_t fj_bin_simd_padding() { return fj_bin_padding_fn(); } + +template +int32_t fj_bin_walk_rows(int32_t* row_slack, + const int32_t* incident_row, + const coef_t* signed_coefficient, + const coef_t* incident_row_cmax, + int32_t incidence_begin, + int32_t incidence_end, + int32_t delta, + int32_t* out_incidence) +{ + return fj_bin_walk_fn(coef_t{})(row_slack, + incident_row, + signed_coefficient, + incident_row_cmax, + incidence_begin, + incidence_end, + delta, + out_incidence); +} + +template int32_t fj_bin_walk_rows( + int32_t*, const int32_t*, const int8_t*, const int8_t*, int32_t, int32_t, int32_t, int32_t*); +template int32_t fj_bin_walk_rows( + int32_t*, const int32_t*, const int16_t*, const int16_t*, int32_t, int32_t, int32_t, int32_t*); + int32_t fj_bin_simd_narrow4_max() { return fj_bin_narrow4_max_fn(); } int32_t fj_bin_simd_narrow8_max() { return fj_bin_narrow8_max_fn(); } From dd1501d3e289440e72c2d1bcedebd0520c23e968 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 05:12:21 -0700 Subject: [PATCH 05/61] docs: record the Highway third-party licence The binary fast path's kernels are built with Google Highway, fetched at configure time like PSLP and PaPILO, but the dependency was never recorded in thirdparty/THIRD_PARTY_LICENSES. Highway is dual-licensed under Apache 2.0 or BSD 3-Clause; cuOpt elects Apache 2.0, so the entry names that licence and reproduces its text verbatim from the upstream LICENSE file. Highway ships no NOTICE file, so Apache 2.0 section 4(d) has nothing to propagate, and the sources are consumed unmodified, so 4(b) does not apply either. Signed-off-by: Alice Boucher --- thirdparty/THIRD_PARTY_LICENSES | 214 ++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/thirdparty/THIRD_PARTY_LICENSES b/thirdparty/THIRD_PARTY_LICENSES index 7424a65232..15d5a08e00 100644 --- a/thirdparty/THIRD_PARTY_LICENSES +++ b/thirdparty/THIRD_PARTY_LICENSES @@ -597,3 +597,217 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +----------------------------------------------------------------------------------------- + +== highway Apache-2.0 + +Files: cpp/build/_deps/highway-src + +Copyright (c) The Highway Project Authors. All rights reserved. + +Highway is dual-licensed under the Apache License 2.0 or the BSD 3-Clause License; +cuOpt elects the Apache License 2.0, reproduced below. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 57e2ef7712a24126bac13790a9b7a4fb52d9220e Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 05:18:47 -0700 Subject: [PATCH 06/61] bit of cleanup --- .../linear_programming/cuopt/run_cpufj.cu | 11 ----------- cpp/CMakeLists.txt | 19 +------------------ cpp/src/utilities/version_info.cpp | 4 ---- 3 files changed, 1 insertion(+), 33 deletions(-) diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu index 6844d09347..e62a8ef224 100644 --- a/benchmarks/linear_programming/cuopt/run_cpufj.cu +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -5,14 +5,6 @@ */ /* clang-format on */ -// Benchmark-only harness for the CPU feasibility-jump portfolio. Loads an instance, builds one -// climber per portfolio slot from a zero start clamped to variable bounds, and runs them in -// parallel on pinned cores for a fixed wall-clock budget. Reports per-climber crossing, objective -// and throughput. -// -// No presolve: pass an already-presolved instance. The climbers are built from problem_t, so the -// binary fast path is reachable when the instance qualifies. - #include #include #include @@ -27,14 +19,11 @@ #include #include -#include -#include #include #include #include #include #include -#include #include namespace { diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d95f48f58b..0fe2bf064b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -307,11 +307,6 @@ set(BUILD_SHARED_LIBS OFF) FetchContent_MakeAvailable(pslp) set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_SAVED}) - -# Highway - portable SIMD with runtime dispatch, used by the binary CPU FJ fast-path kernels -# https://github.com/google/highway -# contrib carries vqsort/image/math, none of which we use, and each would be compiled once per -# SIMD target. FetchContent_Declare( highway GIT_REPOSITORY "https://github.com/google/highway.git" @@ -326,7 +321,6 @@ set(HWY_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE) set(HWY_ENABLE_TESTS OFF CACHE BOOL "" FORCE) set(HWY_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) -# Build Highway as static to embed in cuopt (mirrors PSLP above) set(BUILD_SHARED_LIBS_SAVED ${BUILD_SHARED_LIBS}) set(BUILD_SHARED_LIBS OFF) FetchContent_MakeAvailable(highway) @@ -717,8 +711,6 @@ target_include_directories(cuopt_objs target_link_libraries(cuopt_objs PRIVATE $) add_dependencies(cuopt_objs PSLP) -# Highway, linked by file for the same reason. Static and fully embedded into libcuopt.so; it is -# never installed (HWY_ENABLE_INSTALL OFF) and consumers of cuopt::cuopt never reference it. target_link_libraries(cuopt_objs PRIVATE $) add_dependencies(cuopt_objs hwy) @@ -1100,35 +1092,26 @@ if (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY) "${CMAKE_CURRENT_SOURCE_DIR}/src" ) - # CPU FJ portfolio benchmark. A .cu because it drives problem_t/solution_t directly and - # clamp_within_var_bounds instantiates a device lambda in this TU. + # CPU FJ standalone portfolio benchmark add_executable(solve_CPUFJ ../benchmarks/linear_programming/cuopt/run_cpufj.cu) - set_target_properties(solve_CPUFJ PROPERTIES CXX_SCAN_FOR_MODULES OFF) - - # -fopenmp for the CUDA TU as well: the internal headers this pulls in (omp_helpers.hpp, - # omp_atomic_t) are OMP-dependent, and OpenMP::OpenMP_CXX only covers CXX. target_compile_options(solve_CPUFJ PRIVATE "$<$:${CUOPT_CXX_FLAGS}>" "$<$:${CUOPT_CUDA_FLAGS}>" "$<$:-fopenmp>" ) - target_link_libraries(solve_CPUFJ PUBLIC cuopt OpenMP::OpenMP_CXX OpenMP::OpenMP_CUDA ) - target_include_directories(solve_CPUFJ PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" "${papilo_SOURCE_DIR}/src" "${papilo_BINARY_DIR}" ) - - # Reached transitively: fj_cpu.cuh pulls branch_and_bound/symmetry.hpp, which needs dejavu. target_include_directories(solve_CPUFJ SYSTEM PRIVATE "${pslp_SOURCE_DIR}/include" "${dejavu_SOURCE_DIR}" diff --git a/cpp/src/utilities/version_info.cpp b/cpp/src/utilities/version_info.cpp index 2b1d3919d9..67ec9e6794 100644 --- a/cpp/src/utilities/version_info.cpp +++ b/cpp/src/utilities/version_info.cpp @@ -146,10 +146,6 @@ static std::string get_cpu_model() return "Unknown"; } -// The SIMD instruction set the vectorized CPU kernels actually dispatched to, which reflects the -// compiled target set as well as what this CPU supports. Highway names the AVX-512 family AVX3, -// with AVX3_DL / AVX3_ZEN4 / AVX3_SPR / AVX10_2 marking successively newer feature sets; those are -// reported under the name users know them by. static const char* get_simd_target() { const int64_t target = hwy::DispatchedTarget(); From febac7b83930d46c8d85ad6f1daba5c081309ce5 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 06:16:09 -0700 Subject: [PATCH 07/61] cleanup --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 11 +- .../feasibility_jump/fj_cpu.cuh | 16 -- .../feasibility_jump/fj_cpu_binary.cu | 202 +++++++----------- .../feasibility_jump/fj_cpu_binary.cuh | 139 ++---------- .../fj_cpu_binary_kernels.cpp | 55 +++-- 5 files changed, 125 insertions(+), 298 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index e1b7082b0d..da009c13a6 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1701,11 +1701,6 @@ void finalize_fj_cpu_host_initialization( // Precompute static problem features for regression model precompute_problem_features(fj_cpu); - - // Binary fast path. Depends only on the host problem mirrors and the incoming weights, both - // populated above; engine state is initialized later, at solve entry. Climbers built from a - // host LP reach here too and are declined by the predicate at their first slack column. - try_build_binary_fastpath(fj_cpu); } template @@ -1899,10 +1894,8 @@ std::unique_ptr> fj_t::create_cpu_climber( template void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double work_unit_limit) { - if (fj_cpu->binary_fast) { - fj_cpu->binary_fast->solve(*fj_cpu, in_time_limit, work_unit_limit); - return; - } + // problem fits the binary fastpath shape? run it (engine is solve-local) + if (try_cpufj_binary_solve(*fj_cpu, in_time_limit, work_unit_limit)) return; i_t local_mins = 0; auto loop_start = std::chrono::high_resolution_clock::now(); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index ce1d010151..bb528f54ea 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -22,17 +22,6 @@ namespace cuopt::mathematical_optimization::mip { -// Binary fast-path state. Defined in fj_cpu_binary.cuh, which only fj_cpu.cu includes, so this -// header stays independent of the fast path. The deleter is defined out of line for the same -// reason, mirroring fj_cpu_worker_t::fj_cpu_deleter_t. -template -struct fj_binary_state_t; - -template -struct fj_binary_state_deleter_t { - void operator()(fj_binary_state_t* ptr) const; -}; - template class probing_cache_t; @@ -227,11 +216,6 @@ struct fj_cpu_climber_t { instrumentation_aggregator_t memory_aggregator; // TODO atomic ref? c++20 std::atomic& preemption_flag; - - // Populated by try_build_binary_fastpath when the instance is all-binary with integer - // coefficients in int8/int16 range. Empty on every other instance, in which case cpufj_solve - // runs the general loop below. - std::unique_ptr, fj_binary_state_deleter_t> binary_fast; }; template diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 48aac555e8..a63d3ad802 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -15,12 +15,10 @@ #include -#include #include #include #include #include -#include #include namespace cuopt::mathematical_optimization::mip { @@ -41,31 +39,25 @@ const char* fj_binary_reject_name(fj_binary_reject_t reason) return "unknown"; } - -// Work-unit proxy: bytes attributed per nnz touched. Stands in for the byte counters the general -// path reads off its instrumented vectors. Calibrated by the owner. +// work unit proxy. will likely require a lot of tuning constexpr double fj_bin_bytes_per_nnz = 16.0; -// Tabu for binary variables. A binary variable's only move is a flip, so the direction is a -// function of the current assignment and the general four-array scheme collapses to two. -// flip_until is biased by a rolling base so it fits uint16. +// Tabu for binary variables, expressed as a ring buffer +// There can be at most max_tenure tabu'd variables at any given time. +// since max_tenure << n_vars, it's cheaper to maintain a ring buffer than a full array +// and it allows smaller instances to become L1 resident struct fj_bin_tabu_t { - static constexpr int32_t window = 65535 - 64; + static constexpr int32_t ring_size = 16; + static constexpr int32_t max_tenure = ring_size; + // Headroom so iter + tenure - iter_bias still fits uint16 when iter - iter_bias is at the rebase + // threshold. + static constexpr int32_t window = + (int32_t)std::numeric_limits::max() - max_tenure; std::vector flip_until; std::vector last_flip; - int32_t base{0}; - - // Ring of the last ring_size flips, indexed by iteration rather than by expiry. One variable - // flips per iteration, so each slot takes exactly one entry and nothing is ever displaced while - // still tabu; indexing by expiry instead would collide whenever two flips share a deadline, which - // a simulation over the [3,12] tenure range puts at 35% of insertions. - // - // This is what lets the global argmax stop reading flip_until. Tenure is bounded by the ring, so - // at most ring_size variables are tabu at once out of n: reading two bytes per variable to answer - // a question that is almost always no costs a third of that scan's traffic. On crypt16 it is the - // difference between a 31 KB working set that overflows a 32 KB L1 and a 21 KB one that does not. - static constexpr int32_t ring_size = 16; + int32_t iter_bias{0}; + int32_t ring_var[ring_size]; int32_t ring_expiry[ring_size]; @@ -75,7 +67,7 @@ struct fj_bin_tabu_t { flip_until.assign(n, 0); last_flip.assign(n, 0); clear_ring(); - base = 0; + iter_bias = 0; } void clear(int32_t iter) @@ -83,7 +75,7 @@ struct fj_bin_tabu_t { std::fill(flip_until.begin(), flip_until.end(), (uint16_t)0); std::fill(last_flip.begin(), last_flip.end(), 0); clear_ring(); - base = iter; + iter_bias = iter; } void clear_ring() @@ -96,15 +88,10 @@ struct fj_bin_tabu_t { void on_flip(int32_t v, int32_t iter, int32_t tenure) { - flip_until[v] = (uint16_t)(iter + tenure - base); + flip_until[v] = (uint16_t)(iter + tenure - iter_bias); last_flip[v] = iter; - // Drop any earlier entry for v before inserting the new one. flip_until keeps one deadline per - // variable and a reflip overwrites it, so without this the ring would hold a superseded, later - // deadline and block v past the point the per-variable test would have released it. A variable - // can be reflipped while still tabu: the local-minimum path tests the weaker - // iter == last_flip + 1. With this the ring holds at most one live entry per variable carrying - // its current deadline, which is exactly the invariant flip_until maintains. + // keep only one tabu entry per var for (int32_t i = 0; i < ring_size; ++i) { if (ring_var[i] == v) ring_var[i] = -1; } @@ -114,9 +101,7 @@ struct fj_bin_tabu_t { ring_expiry[slot] = iter + tenure; } - // Blocks every currently-tabu variable by writing the invalid sentinel over its score, and reports - // how many were touched. Paired with unblock around one argmax and nothing else: the score is - // maintained incrementally, so it may only be disturbed across a window in which no patch runs. + // replace the scores of tabu'd variable with sentinel values int32_t block_tabu(int32_t iter, int32_t* var_score, int32_t (&saved_var)[ring_size], @@ -135,8 +120,7 @@ struct fj_bin_tabu_t { return k; } - // Reverse order on purpose: a variable flipped twice inside the ring appears twice, and its second - // save holds the sentinel written by the first. Unwinding backwards restores the true score last. + // reverse the above operation. static void unblock_tabu(int32_t k, int32_t* var_score, const int32_t (&saved_var)[ring_size], @@ -148,17 +132,17 @@ struct fj_bin_tabu_t { bool blocked(int32_t v, int32_t iter, bool localmin) const { - return localmin ? (iter == last_flip[v] + 1) : ((uint16_t)(iter - base) < flip_until[v]); + return localmin ? (iter == last_flip[v] + 1) + : ((uint16_t)(iter - iter_bias) < flip_until[v]); } - // Rebase before iter - base can overflow the uint16 window. Expired entries saturate to 0, - // which reads as not-tabu. - void maybe_advance(int32_t iter) + // rebase the iteration bias value every 64k iter + void maybe_rebase(int32_t iter) { - if ((int64_t)iter - base <= window) return; - const uint16_t shift = (uint16_t)(iter - base); + if ((int64_t)iter - iter_bias <= window) return; + const uint16_t shift = (uint16_t)(iter - iter_bias); for (uint16_t& fu : flip_until) fu = (fu > shift) ? (uint16_t)(fu - shift) : (uint16_t)0; - base = iter; + iter_bias = iter; } }; @@ -223,19 +207,13 @@ constexpr int32_t fj_bin_ddfw_transfer = 1; constexpr int32_t fj_bin_ddfw_donor_samples = 1; constexpr int32_t fj_bin_restart_period = 5000000; -// `[study]` Prefetch distance for the reverse-CSR row walk, in rows. Each move visits the rows of -// one variable through reverse_constraints, whose entries are effectively random indices into a row -// array far larger than L2, and the sequence is data-dependent so no hardware prefetcher can follow -// it. reverse_constraints is padded by this much so the lookahead needs no bounds test. +// prefetch distance +// TODO: check if it actually matters at all for performance constexpr int32_t fj_bin_pf_dist = 8; -// Limits of the packed score. Breaching either corrupts the ordering, so compute_saturation -// reports the observed peaks against them at end of solve. constexpr int32_t fj_bin_base_limit = 1 << 16; constexpr int32_t fj_bin_bonus_limit = 1 << 14; -static inline bool fj_bin_is_integral(double v, double tol) { return std::fabs(v - std::round(v)) <= tol; } - static inline bool fj_bin_in_int32(double v) { return v >= (double)INT32_MIN && v <= (double)INT32_MAX; @@ -253,14 +231,13 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) return out; } - const double tol = c.view.pb.tolerances.integrality_tolerance; - const auto& var_bounds = c.h_var_bounds; - const auto& var_types = c.h_var_types; + const double tol = c.view.pb.tolerances.integrality_tolerance; + const auto& is_binary_variable = c.h_is_binary_variable; + cuopt_assert((int32_t)is_binary_variable.size() == n, "is_binary_variable size mismatch"); for (int32_t v = 0; v < n; ++v) { - auto bounds = var_bounds[v]; - if (var_types[v] != var_t::INTEGER || std::fabs(get_lower(bounds)) > tol || - std::fabs(get_upper(bounds) - 1.0) > tol) { + // Populated at climber init with integer_equal on [0,1] bounds. + if (!is_binary_variable[v]) { out.reject = fj_binary_reject_t::non_binary_var; out.bad_var = v; return out; @@ -277,7 +254,7 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) double row_abs_sum = 0; for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { const double a = coeffs[k]; - if (!fj_bin_is_integral(a, tol)) { + if (!is_integer(a, tol)) { out.reject = fj_binary_reject_t::fractional_coefficient; out.bad_row = r; return out; @@ -303,7 +280,7 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) const bool finite[2] = {lb_fin, ub_fin}; for (int s = 0; s < 2; ++s) { if (!finite[s]) continue; - if (!fj_bin_is_integral(sides[s], tol)) { + if (!is_integer(sides[s], tol)) { out.reject = fj_binary_reject_t::fractional_row_bound; out.bad_row = r; return out; @@ -372,7 +349,7 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { const double a = coeffs[k]; const long ai = std::lround(a); - if (!fj_bin_is_integral(a, tol) || ai < std::numeric_limits::min() || + if (!is_integer(a, tol) || ai < std::numeric_limits::min() || ai > std::numeric_limits::max()) { return false; } @@ -404,9 +381,8 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, // row without running off the end and can therefore mask its remainder rather than peeling it // into a scalar tail. The padding is never read as data: every lane past a row's end is excluded // from the gather, the scatter and the store by the row-length mask. - const int32_t pad = fj_bin_simd_padding(); - pb.variables.resize(pb.nnz + pad, 0); - pb.coefficients.resize(pb.nnz + pad, (coef_t)0); + pb.variables.resize(pb.nnz + fj_bin_simd_padding, 0); + pb.coefficients.resize(pb.nnz + fj_bin_simd_padding, (coef_t)0); // Scale the incoming weights into the DDFW band by one global factor, so relative structure // survives while every row clears the donation floor. Capped so the largest scaled weight stays @@ -460,7 +436,8 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, // Lookahead room for the row walk: a vector of overhang for the kernel's unit-stride loads, and // the prefetch distance the scalar path uses. Reads land on row 0, harmlessly, and every lane past // a variable's range is masked out of the gather, the scatter and the compress. - const int32_t rpad = fj_bin_pf_dist > pad ? fj_bin_pf_dist : pad; + const int32_t rpad = + fj_bin_pf_dist > fj_bin_simd_padding ? fj_bin_pf_dist : fj_bin_simd_padding; pb.reverse_constraints.resize(pb.nnz + rpad, 0); pb.signed_coefficient.resize(pb.nnz + rpad, (coef_t)0); pb.incident_row_cmax.resize(pb.nnz + rpad, (coef_t)1); @@ -477,7 +454,7 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, // The integer engine. Feasibility is an exact compare against one bound per row, so there is no // tolerance arithmetic and no compensated summation anywhere below. template -struct fj_bin_engine_t : fj_binary_state_t { +struct fj_bin_engine_t { fj_bin_problem_t pb; std::vector> rows; @@ -528,11 +505,6 @@ struct fj_bin_engine_t : fj_binary_state_t { // what bounds the index re-scan, so it is about the shape of the sweep and not cache capacity. int32_t argmax_tile{256}; - // Rows at or below these lengths go to the 4- and 8-lane patch kernels; 0 disables that width. - int32_t narrow4_max{0}; - int32_t narrow8_max{0}; - - // Settings read at solve entry, where the climber carries populated values. int32_t seed{0}; int32_t tabu_tenure_min{3}; @@ -545,15 +517,7 @@ struct fj_bin_engine_t : fj_binary_state_t { int32_t max_aggregate_base{0}; int32_t max_aggregate_bonus{0}; - int coefficient_bits() const override { return 8 * (int)sizeof(coef_t); } - int n_split_constraints() const override { return pb.n_constraints; } - i_t iterations() const override { return (i_t)iters; } - - void saturation(int& base_peak, int& bonus_peak) const override - { - base_peak = max_aggregate_base; - bonus_peak = max_aggregate_bonus; - } + int coefficient_bits() const { return 8 * (int)sizeof(coef_t); } // Largest per-variable aggregate base and bonus under the final weights and assignment, in raw // int32. The packed representation is only order-preserving while these stay inside their @@ -762,11 +726,8 @@ struct fj_bin_engine_t : fj_binary_state_t { const int32_t margin = (int32_t)rcmax_p[ii]; if (!(old_slack < -margin && new_slack < -margin)) { const int32_t kb = offsets_p[r], ke = offsets_p[r + 1]; - // The offsets are already loaded for the call, so the width choice is a compare rather - // than a stored per-row flag. // TODO: check that this may not cause AVX512 powerdown overheads if the AVX2 row/AVX512 row ratio is unbalanced - fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), - vars_p, + fj_bin_patch_row(vars_p, coefs_p, kb, ke, @@ -854,8 +815,7 @@ struct fj_bin_engine_t : fj_binary_state_t { if (new_weight > max_weight) max_weight = new_weight; // The slack is unchanged here, and no variable is excluded, so skip_var matches no index. const int32_t kb = pb.offsets[r], ke = pb.offsets[r + 1]; - fj_bin_patch_row(fj_bin_patch_width_for(ke - kb, narrow4_max, narrow8_max), - pb.variables.data(), + fj_bin_patch_row(pb.variables.data(), pb.coefficients.data(), kb, ke, @@ -1042,8 +1002,6 @@ struct fj_bin_engine_t : fj_binary_state_t { { const auto& params = climber.settings.parameters; seed = climber.settings.seed; - narrow4_max = fj_bin_simd_narrow4_max(); - narrow8_max = fj_bin_simd_narrow8_max(); rng = raft::random::PCGenerator((uint64_t)seed, 0, 0); tabu_tenure_min = params.tabu_tenure_min; tabu_tenure_max = params.tabu_tenure_max; @@ -1057,9 +1015,9 @@ struct fj_bin_engine_t : fj_binary_state_t { // iterations. A tenure that long would be overwritten while the variable is still tabu, and the // argmax would stop excluding it. Clamped as well as asserted: release builds compile the assert // out, and silently dropping tabu entries is worse than a shorter tenure. - cuopt_assert(tabu_tenure_max <= fj_bin_tabu_t::ring_size, + cuopt_assert(tabu_tenure_max <= fj_bin_tabu_t::max_tenure, "tabu tenure exceeds the tabu ring, live entries would be evicted"); - if (tabu_tenure_max > fj_bin_tabu_t::ring_size) tabu_tenure_max = fj_bin_tabu_t::ring_size; + if (tabu_tenure_max > fj_bin_tabu_t::max_tenure) tabu_tenure_max = fj_bin_tabu_t::max_tenure; const int32_t n = pb.n_variables, m = pb.n_constraints; const auto& h_assign = climber.h_assignment; @@ -1079,7 +1037,7 @@ struct fj_bin_engine_t : fj_binary_state_t { row_slack.assign(m, 0); var_score.assign(n, 0); - nnz_score_delta.assign(pb.nnz + fj_bin_simd_padding(), 0); + nnz_score_delta.assign(pb.nnz + fj_bin_simd_padding, 0); tabu.resize(n); is_violated.assign(m, 0); vpos.assign(m, -1); @@ -1096,7 +1054,7 @@ struct fj_bin_engine_t : fj_binary_state_t { recompute_slack(); } - void solve(fj_cpu_climber_t& climber, f_t time_limit, double work_unit_limit) override + void solve(fj_cpu_climber_t& climber, f_t time_limit, double work_unit_limit) { init(climber); @@ -1109,7 +1067,7 @@ struct fj_bin_engine_t : fj_binary_state_t { if (bounded_time && std::chrono::high_resolution_clock::now() - loop_start > limit) break; if (iters >= climber.settings.iteration_limit) break; if (iters - last_restart_iter >= fj_bin_restart_period) do_restart(); - tabu.maybe_advance(iters); + tabu.maybe_rebase(iters); int32_t move_var = -1, score = fj_bin_score_invalid; if (violated_list.empty()) std::tie(move_var, score) = find_lift_move(); @@ -1182,13 +1140,9 @@ struct fj_bin_engine_t : fj_binary_state_t { }; template -void fj_binary_state_deleter_t::operator()(fj_binary_state_t* ptr) const -{ - delete ptr; -} - -template -void try_build_binary_fastpath(fj_cpu_climber_t& climber) +bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + f_t time_limit, + double work_unit_limit) { const fj_bin_scan_t scan = fj_bin_scan(climber); if (scan.reject != fj_binary_reject_t::none) { @@ -1197,41 +1151,43 @@ void try_build_binary_fastpath(fj_cpu_climber_t& climber) fj_binary_reject_name(scan.reject), scan.bad_row, scan.bad_var); - return; + return false; } - bool built = false; - if (scan.coefficient_bits == 8) { - auto engine = std::make_unique>(); - built = fj_bin_narrow(climber, scan.n_split_constraints, engine->pb); - if (built) climber.binary_fast.reset(engine.release()); - } else { - auto engine = std::make_unique>(); - built = fj_bin_narrow(climber, scan.n_split_constraints, engine->pb); - if (built) climber.binary_fast.reset(engine.release()); - } + auto run = [&](auto& engine) -> bool { + if (!fj_bin_narrow(climber, scan.n_split_constraints, engine.pb)) { + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s", + climber.log_prefix.c_str(), + fj_binary_reject_name(fj_binary_reject_t::narrow_check_failed)); + return false; + } + CUOPT_LOG_DEBUG( + "%sCPUFJ binary fast path enabled: int%d coefficients, %d rows after one-sided split", + climber.log_prefix.c_str(), + scan.coefficient_bits, + scan.n_split_constraints); + engine.solve(climber, time_limit, work_unit_limit); + return true; + }; - if (!built) { - CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s", - climber.log_prefix.c_str(), - fj_binary_reject_name(fj_binary_reject_t::narrow_check_failed)); - return; + if (scan.coefficient_bits == 8) { + fj_bin_engine_t engine; + return run(engine); } - - CUOPT_LOG_DEBUG("%sCPUFJ binary fast path enabled: int%d coefficients, %d rows after one-sided split", - climber.log_prefix.c_str(), - scan.coefficient_bits, - scan.n_split_constraints); + fj_bin_engine_t engine; + return run(engine); } #if MIP_INSTANTIATE_FLOAT -template struct fj_binary_state_deleter_t; -template void try_build_binary_fastpath(fj_cpu_climber_t& climber); +template bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + float time_limit, + double work_unit_limit); #endif #if MIP_INSTANTIATE_DOUBLE -template struct fj_binary_state_deleter_t; -template void try_build_binary_fastpath(fj_cpu_climber_t& climber); +template bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + double time_limit, + double work_unit_limit); #endif } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh index d6daf9c160..e7cc1a2818 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -8,14 +8,9 @@ #pragma once #include -#include -// Seam between the general CPU FJ path and the binary fast path. Only fj_cpu.cu includes this; -// fj_cpu.cuh forward-declares fj_binary_state_t so the climber can hold one without the general -// header depending on the fast path. -// // The fast path applies to instances whose variables are all binary and whose rows carry integer -// coefficients within int8 or int16 range. On those it runs an integer engine: exact feasibility +// coefficients within int8 or int16 range. On those it runs a SIMD integer engine: exact feasibility // against a single row bound, a live per-variable score patched through stored per-nnz // contributions, and a global argmax move selection. @@ -24,7 +19,6 @@ namespace cuopt::mathematical_optimization::mip { template struct fj_cpu_climber_t; -// Why an instance was refused, logged at DEBUG by the build entry. enum class fj_binary_reject_t : uint8_t { none, empty_problem, @@ -36,49 +30,16 @@ enum class fj_binary_reject_t : uint8_t { lhs_headroom, narrow_check_failed, }; - const char* fj_binary_reject_name(fj_binary_reject_t reason); -// Built state for one eligible instance. The concrete engine is templated on coefficient width -// (int8_t or int16_t) and lives in fj_cpu_binary.cu; the width choice is made once at build time, -// so dispatching through this base costs one virtual call per solve. -template -struct fj_binary_state_t { - virtual ~fj_binary_state_t() = default; - - virtual void solve(fj_cpu_climber_t& climber, - f_t time_limit, - double work_unit_limit) = 0; - - virtual int coefficient_bits() const = 0; // 8 or 16 - virtual int n_split_constraints() const = 0; - virtual i_t iterations() const = 0; - - // Largest per-variable aggregate base and bonus observed at end of solve. The packed score is - // order-preserving only while these stay under 2^16 and 2^14 respectively. - virtual void saturation(int& max_aggregate_base, int& max_aggregate_bonus) const = 0; -}; - -// Runs predicate -> one-sided split -> narrow. A pure function of the climber's host problem -// mirrors and its incoming constraint weights, so it is well defined wherever those are populated. -// Populates climber.binary_fast on success; leaves it empty and logs the reason otherwise. +// Returns true if the fast path ran (eligible and narrowed); false if declined, in which case the caller should take the general path. +// TODO: worth revisiting if the same climber is solved repeatedly to cache the fastpath state template -void try_build_binary_fastpath(fj_cpu_climber_t& climber); +bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + f_t time_limit, + double work_unit_limit); -// --------------------------------------------------------------------------------------------- -// Hot kernels and the score-delta formula they share with the engine. -// -// The kernels live in fj_cpu_binary_kernels.cpp, built with Google Highway, which compiles the -// bodies once per SIMD target and dispatches at runtime. That file is host-compiled: nvcc's -// frontend rejects Highway's x86 headers, which reinterpret-cast intrinsic vectors to -// compiler-specific vector types (GCC vector extensions in the constant-folding path, __m128bh -// for bfloat16). Every argument below is a plain pointer or scalar, so the seam names no cuOpt or -// CUDA type. -// --------------------------------------------------------------------------------------------- - -// Packed staged score: one int32 holding base * K + bonus. K exceeds twice the largest |bonus| the -// engine produces, so integer ordering on the packed word reproduces the lexicographic (base, -// bonus) ordering the general path gets from fj_staged_score_t. +// Packed staged score: one int32 holding base * K + bonus constexpr int32_t fj_bin_score_shift = 15; constexpr int32_t fj_bin_score_k = 1 << fj_bin_score_shift; constexpr int32_t fj_bin_score_invalid = INT32_MIN; @@ -87,9 +48,7 @@ constexpr int32_t fj_bin_score_invalid = INT32_MIN; // (os) and after (ns) that flip. base is the weighted change in satisfaction; bonus is the // weighted change in strict slack. When both states are violated the improving direction earns // half weight, matching excess_improvement_weight of 1/2. -// -// Single source of the formula: the engine scores moves with it, compute_saturation walks it, and -// the vector kernels reproduce it lane-wise. +// purpose: implements the scoring delta logic from feasibility_jump.cuh in a form easier to port to SIMD static inline void fj_bin_score_delta_parts( int32_t os, int32_t ns, int32_t weight, int32_t& base, int32_t& bonus) { @@ -107,43 +66,14 @@ static inline int32_t fj_bin_packed_score_delta(int32_t os, int32_t ns, int32_t return base * fj_bin_score_k + bonus; } -// Elements of padding the per-nnz arrays must carry past nnz, in int32 units. A target that masks -// its row remainder loads and stores a whole vector at the last row of the matrix, and the padding -// is what keeps that off memory it does not own. A target that peels the remainder into a scalar -// tail stops at the row end and asks for nothing, so this returns 0 there. -int32_t fj_bin_simd_padding(); - -// Vector width the row patch runs at. A gather costs the same whether its lanes carry data or are -// masked off, so a row filling only part of a native vector is cheaper through a narrower one; past -// the crossover the extra vector and its extra full gather cost more than the idle lanes. -enum class fj_bin_patch_width_t : int32_t { narrow4 = 0, narrow8 = 1, native = 2 }; - -// Longest row for which each narrower width beats the native one, or 0 where that width is not -// worth offering on this target: a width is offered only when it is strictly narrower than the -// native vector, and scalable targets decline both. -int32_t fj_bin_simd_narrow4_max(); -int32_t fj_bin_simd_narrow8_max(); - -static inline fj_bin_patch_width_t fj_bin_patch_width_for(int32_t row_len, - int32_t narrow4_max, - int32_t narrow8_max) -{ - if (row_len <= narrow4_max) return fj_bin_patch_width_t::narrow4; - if (row_len <= narrow8_max) return fj_bin_patch_width_t::narrow8; - return fj_bin_patch_width_t::native; -} +// Padding margin to prevent faults on tail SIMD loads +constexpr int32_t fj_bin_simd_padding = 256; -// Patch every variable of one row against the row's current signed slack, skipping skip_var. The +// Patch every variable of one row against the row's current signed slack. The // move case passes the post-move slack and the flipped variable's index; the reweight case passes // the unchanged slack and -1, which matches no variable index. -// -// Reads and writes up to fj_bin_simd_padding() elements from kb, so variables, coefficients and -// nnz_score_delta must carry that padding. -// -// Defined in the host-compiled kernels TU and explicitly instantiated there for int8_t and int16_t. template -void fj_bin_patch_row(fj_bin_patch_width_t width, - const int32_t* variables, +void fj_bin_patch_row(const int32_t* variables, const coef_t* coefficients, int32_t kb, int32_t ke, @@ -155,48 +85,15 @@ void fj_bin_patch_row(fj_bin_patch_width_t width, int32_t os_new, int32_t skip_var); -// Incidences handed to fj_bin_walk_rows per call, and so the size of the caller's output buffer. -// Large enough that the dynamic dispatch and the call's setup amortize away -- a variable of -// supportcase22 averages 344 incidences, so one or two calls per move -- and small enough that the -// buffer is a stack array and its touched part stays in L1. A multiple of every supported lane -// count, so only a variable's last tile has a masked remainder. constexpr int32_t fj_bin_walk_tile = 256; -// Advance every row incident to one flipped variable, and report which of those visits the caller -// must finish by hand. -// -// An "incidence" is one (variable, row) pair of the matrix, so the reverse CSR indexed by a -// variable's [incidence_begin, incidence_end) names the rows that variable appears in. Three arrays -// are read at that index, all at unit stride: -// -// incident_row[i] the row this incidence touches -// signed_coefficient[i] sign * coefficient, folded once at build time -// incident_row_cmax[i] that row's max|coef|, replicated here so it need not be gathered -// -// The last two are structural and fixed for the life of the solve. row_slack[r] is indexed by row, -// and holds sign * (bound - lhs) -- the signed slack the engine stores in place of lhs, which is -// what reduces the update to a subtraction. delta is the flip direction, +1 or -1, uniform over the -// call. +// Advance every row incident to one flipped variable within apply_move, and report which of those visits the caller +// must finish by hand (e.g. if the row needs patching) // // For every incidence i in the range this applies // row_slack[incident_row[i]] -= signed_coefficient[i] * delta // then writes to out_incidence, in increasing order, the subset of i whose row is not deeply -// satisfied on both sides of the flip -- 15.1% of visits on supportcase22 -- and returns how many. -// Everything those visits still need is indirect (the row's weight, the own-score delta, the -// violated-set transitions, the patch) and stays with the caller. -// -// The caller drives this a tile at a time, running each tile's tail before asking for the next, so -// out_incidence is a fj_bin_walk_tile-sized buffer the caller can keep on its stack rather than one -// sized to the widest reverse degree in the matrix. Tiling this way rather than interleaving the -// tail into the vector loop keeps every patch call outside it: the loop over lanes lives here, -// behind a dynamic dispatch pointer that cannot be inlined, while the tail needs engine-side state -// this translation unit cannot reach. A tile still runs the vector body many times before yielding, -// which is what lets consecutive gathers overlap. -// -// out_incidence must hold incidence_end - incidence_begin entries. Reads up to -// fj_bin_simd_padding() elements past incidence_end from the three per-incidence arrays. Interior -// tiles read into the next tile, which is in bounds; the final one reads past the variable's range, -// so those arrays carry that padding. +// satisfied on both sides of the flip and returns how many. template int32_t fj_bin_walk_rows(int32_t* row_slack, const int32_t* incident_row, @@ -209,10 +106,8 @@ int32_t fj_bin_walk_rows(int32_t* row_slack, // Argmax over var_score, scanning all n variables. Valid while the objective weight is zero, where // the full score is exactly var_score. Yields best_var of -1 only if n is 0. -// -// Carries no tabu argument: at most a ring's worth of variables are tabu at once, so the caller -// holds those few at fj_bin_score_invalid across the call instead of making this read a per-variable -// tenure array. That keeps the scan to four bytes per variable. +// Tabu is handled by "blocking" the scores corresponding to the tabu vars, and restoring them after the argmax +// affordable since max_tenure is small void fj_bin_argmax(const int32_t* var_score, int32_t n, int32_t tile, diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index 90affda3fb..08125f44b3 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -36,13 +36,6 @@ namespace hn = hwy::HWY_NAMESPACE; constexpr bool k_mask_remainder = (HWY_TARGET <= HWY_AVX3) || HWY_TARGET_IS_SVE || (HWY_TARGET == HWY_RVV); -// Padding the caller must carry past nnz. Only a masking target reads past a row's end; a peeling -// target stops at ke, so it asks for nothing and its buffers keep their natural size. -int32_t PaddingImpl() -{ - return k_mask_remainder ? (int32_t)hn::Lanes(hn::ScalableTag()) : 0; -} - // Whether the row walk below is worth vectorizing on this target. It needs a real gather to read the // slacks and a real compress to emit the tail list; where either is emulated the emulation costs // more than the scalar loop it replaces, since 85% of visits do nothing but subtract and compare. @@ -420,7 +413,6 @@ HWY_EXPORT_T(PatchRowNatI16, PatchRowImpl); HWY_EXPORT_T(PatchRowN8I16, PatchRowNarrow8Impl); HWY_EXPORT_T(PatchRowN4I16, PatchRowNarrow4Impl); HWY_EXPORT(ArgmaxImpl); -HWY_EXPORT(PaddingImpl); HWY_EXPORT(Narrow4MaxImpl); HWY_EXPORT(Narrow8MaxImpl); @@ -455,7 +447,7 @@ using fj_bin_patch_fn_t = void (*)(const int32_t*, int32_t, int32_t); -// Indexed by fj_bin_patch_width_t. +// Indexed by width: 0 = 4-lane, 1 = 8-lane, 2 = native. static const fj_bin_patch_fn_t fj_bin_patch_i8[3] = { (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN4I8)), (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN8I8)), @@ -472,6 +464,24 @@ static const fj_bin_patch_fn_t fj_bin_patch_i16[3] = { static const fj_bin_patch_fn_t* fj_bin_patch_table(int8_t) { return fj_bin_patch_i8; } static const fj_bin_patch_fn_t* fj_bin_patch_table(int16_t) { return fj_bin_patch_i16; } +static const auto fj_bin_narrow4_max_fn = + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow4MaxImpl)); +static const auto fj_bin_narrow8_max_fn = + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow8MaxImpl)); + +// Resolved once at load: a gather costs the same whether its lanes carry data or are masked off, so +// a row filling only part of a native vector is cheaper through a narrower one. Scalable targets +// decline both (see Narrow*MaxImpl). +static const int32_t fj_bin_n4_max = fj_bin_narrow4_max_fn(); +static const int32_t fj_bin_n8_max = fj_bin_narrow8_max_fn(); + +static int32_t fj_bin_patch_width_index(int32_t row_len) +{ + if (row_len <= fj_bin_n4_max) return 0; + if (row_len <= fj_bin_n8_max) return 1; + return 2; +} + template using fj_bin_walk_fn_t = int32_t (*)( int32_t*, const int32_t*, const coef_t*, const coef_t*, int32_t, int32_t, int32_t, int32_t*); @@ -485,13 +495,6 @@ static fj_bin_walk_fn_t fj_bin_walk_fn(int8_t) { return fj_bin_walk_i8; static fj_bin_walk_fn_t fj_bin_walk_fn(int16_t) { return fj_bin_walk_i16; } static const auto fj_bin_argmax_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(ArgmaxImpl)); -static const auto fj_bin_padding_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(PaddingImpl)); -static const auto fj_bin_narrow4_max_fn = - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow4MaxImpl)); -static const auto fj_bin_narrow8_max_fn = - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow8MaxImpl)); - -int32_t fj_bin_simd_padding() { return fj_bin_padding_fn(); } template int32_t fj_bin_walk_rows(int32_t* row_slack, @@ -518,12 +521,8 @@ template int32_t fj_bin_walk_rows( template int32_t fj_bin_walk_rows( int32_t*, const int32_t*, const int16_t*, const int16_t*, int32_t, int32_t, int32_t, int32_t*); -int32_t fj_bin_simd_narrow4_max() { return fj_bin_narrow4_max_fn(); } -int32_t fj_bin_simd_narrow8_max() { return fj_bin_narrow8_max_fn(); } - template -void fj_bin_patch_row(fj_bin_patch_width_t width, - const int32_t* variables, +void fj_bin_patch_row(const int32_t* variables, const coef_t* coefficients, int32_t kb, int32_t ke, @@ -535,13 +534,14 @@ void fj_bin_patch_row(fj_bin_patch_width_t width, int32_t os_new, int32_t skip_var) { - fj_bin_patch_table(coef_t{})[(int)width](variables, coefficients, kb, ke, var_score, - nnz_score_delta, assign_i32, sign, weight, os_new, - skip_var); + // The offsets are already in hand, so the width choice is a compare rather than a stored per-row + // flag. + fj_bin_patch_table(coef_t{})[fj_bin_patch_width_index(ke - kb)]( + variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, sign, weight, os_new, + skip_var); } -template void fj_bin_patch_row(fj_bin_patch_width_t, - const int32_t*, +template void fj_bin_patch_row(const int32_t*, const int8_t*, int32_t, int32_t, @@ -553,8 +553,7 @@ template void fj_bin_patch_row(fj_bin_patch_width_t, int32_t, int32_t); -template void fj_bin_patch_row(fj_bin_patch_width_t, - const int32_t*, +template void fj_bin_patch_row(const int32_t*, const int16_t*, int32_t, int32_t, From 2dc1f79594a4e7f1d6b97e13ea94305df184976c Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 07:45:41 -0700 Subject: [PATCH 08/61] perf: fold the patch width choice into the Highway target The width crossovers are per-target constants, but reaching them from the call seam turned them into loads of runtime globals: the stub compared the row length against fj_bin_n4_max and fj_bin_n8_max, selected a table entry with a cmov, loaded the pointer and jumped through it. That put an unpredictable branch on row length directly in front of an indirect jump, in a serialized cmp -> select -> load -> jmp chain, with two extra callee-saved registers spilled around it. Decide the width inside HWY_NAMESPACE instead, where the crossovers are constexpr and the three kernels are direct calls, and export one entry point per coefficient width. On a scalable target both bounds are 0, so the compares fold away and the narrow arms are stripped, which is what the runtime HWY_HAVE_SCALABLE check did before. The seam collapses from 28 instructions to one: jmp *fj_bin_patch_i8(%rip) supportcase22, 20s x 16 climbers, mean of 3: 4.243 -> 4.354 M iters/s (+2.6%), which recovers the 2.4% this seam had cost and lands slightly ahead of the shape that preceded it. Search behaviour is unchanged (1/16 crossed, objective 116). Signed-off-by: Alice Boucher --- .../fj_cpu_binary_kernels.cpp | 104 ++++++++---------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index 08125f44b3..5ceb5914aa 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -326,16 +326,41 @@ void PatchRowNarrow4Impl(const int32_t* HWY_RESTRICT variables, // dispatches to a kernel identical to its own. Scalable targets opt out entirely: Highway notes // that clamping Lanes() on RVV/SVE can cost more than the capping saves, which is why // CappedTagIfFixed leaves them at native width above. -int32_t Narrow4MaxImpl() -{ - if (HWY_HAVE_SCALABLE) return 0; - return hn::Lanes(hn::ScalableTag()) > 4 ? 4 : 0; -} - -int32_t Narrow8MaxImpl() +// +// These are per-target constants, so the width choice belongs here rather than at the call seam: a +// caller outside this file can only reach them through a dispatch pointer, which turns two +// immediates into two loads of runtime globals and puts an unpredictable branch directly in front +// of the indirect jump that follows it. Measured on supportcase22, that seam cost 2.4%. +constexpr size_t k_native_lanes = HWY_MAX_LANES_D(hn::ScalableTag); +constexpr int32_t k_narrow4_max = HWY_HAVE_SCALABLE ? 0 : (k_native_lanes > 4 ? 4 : 0); +constexpr int32_t k_narrow8_max = HWY_HAVE_SCALABLE ? 0 : (k_native_lanes > 8 ? 8 : 0); + +// Single entry point the seam dispatches to. On a scalable target both bounds are 0, so both +// compares fold away and the narrow arms are stripped. +template +void PatchRowDispatchImpl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int32_t* HWY_RESTRICT var_score, + int32_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t sign, + int32_t weight, + int32_t os_new, + int32_t skip_var) { - if (HWY_HAVE_SCALABLE) return 0; - return hn::Lanes(hn::ScalableTag()) > 8 ? 8 : 0; + const int32_t row_len = ke - kb; + if (row_len <= k_narrow4_max) { + PatchRowNarrow4Impl(variables, coefficients, kb, ke, var_score, nnz_score_delta, + assign_i32, sign, weight, os_new, skip_var); + } else if (row_len <= k_narrow8_max) { + PatchRowNarrow8Impl(variables, coefficients, kb, ke, var_score, nnz_score_delta, + assign_i32, sign, weight, os_new, skip_var); + } else { + PatchRowImpl(variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, + sign, weight, os_new, skip_var); + } } // Tiled sweep carrying a running maximum. The index re-scan fires only on a tile that raises it, @@ -404,17 +429,11 @@ namespace cuopt::mathematical_optimization::mip { // separately from the function, which lets the function be a template-id: only the table name goes // through token pasting, so no hand-written non-template wrapper is needed. The template argument // must stay comma-free, which is why the three tag-binding wrappers above take only coef_t. -HWY_EXPORT_T(PatchRowNatI8, PatchRowImpl); -HWY_EXPORT_T(PatchRowN8I8, PatchRowNarrow8Impl); -HWY_EXPORT_T(PatchRowN4I8, PatchRowNarrow4Impl); +HWY_EXPORT_T(PatchRowI8, PatchRowDispatchImpl); +HWY_EXPORT_T(PatchRowI16, PatchRowDispatchImpl); HWY_EXPORT_T(WalkRowsI8, WalkRowsImpl); HWY_EXPORT_T(WalkRowsI16, WalkRowsImpl); -HWY_EXPORT_T(PatchRowNatI16, PatchRowImpl); -HWY_EXPORT_T(PatchRowN8I16, PatchRowNarrow8Impl); -HWY_EXPORT_T(PatchRowN4I16, PatchRowNarrow4Impl); HWY_EXPORT(ArgmaxImpl); -HWY_EXPORT(Narrow4MaxImpl); -HWY_EXPORT(Narrow8MaxImpl); // HWY_DYNAMIC_DISPATCH resolves the target on every call, and the hwy::GetChosenTarget() call it // expands to is a real out-of-line call: it clobbers the argument registers, so the compiler spills @@ -447,40 +466,16 @@ using fj_bin_patch_fn_t = void (*)(const int32_t*, int32_t, int32_t); -// Indexed by width: 0 = 4-lane, 1 = 8-lane, 2 = native. -static const fj_bin_patch_fn_t fj_bin_patch_i8[3] = { - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN4I8)), - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN8I8)), - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowNatI8)), -}; - -static const fj_bin_patch_fn_t fj_bin_patch_i16[3] = { - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN4I16)), - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowN8I16)), - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER_T(PatchRowNatI16)), -}; - -// Overloaded rather than specialized so the tables stay plain arrays. -static const fj_bin_patch_fn_t* fj_bin_patch_table(int8_t) { return fj_bin_patch_i8; } -static const fj_bin_patch_fn_t* fj_bin_patch_table(int16_t) { return fj_bin_patch_i16; } - -static const auto fj_bin_narrow4_max_fn = - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow4MaxImpl)); -static const auto fj_bin_narrow8_max_fn = - (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(Narrow8MaxImpl)); - -// Resolved once at load: a gather costs the same whether its lanes carry data or are masked off, so -// a row filling only part of a native vector is cheaper through a narrower one. Scalable targets -// decline both (see Narrow*MaxImpl). -static const int32_t fj_bin_n4_max = fj_bin_narrow4_max_fn(); -static const int32_t fj_bin_n8_max = fj_bin_narrow8_max_fn(); - -static int32_t fj_bin_patch_width_index(int32_t row_len) -{ - if (row_len <= fj_bin_n4_max) return 0; - if (row_len <= fj_bin_n8_max) return 1; - return 2; -} +// The vector width is chosen inside the target (see PatchRowDispatchImpl), so the seam carries one +// pointer per coefficient width and nothing else. +static const auto fj_bin_patch_i8 = + (fj_bin_choose_target(), (fj_bin_patch_fn_t)HWY_DYNAMIC_POINTER_T(PatchRowI8)); +static const auto fj_bin_patch_i16 = + (fj_bin_choose_target(), (fj_bin_patch_fn_t)HWY_DYNAMIC_POINTER_T(PatchRowI16)); + +// Overloaded rather than specialized, matching fj_bin_walk_fn below. +static fj_bin_patch_fn_t fj_bin_patch_fn(int8_t) { return fj_bin_patch_i8; } +static fj_bin_patch_fn_t fj_bin_patch_fn(int16_t) { return fj_bin_patch_i16; } template using fj_bin_walk_fn_t = int32_t (*)( @@ -534,11 +529,8 @@ void fj_bin_patch_row(const int32_t* variables, int32_t os_new, int32_t skip_var) { - // The offsets are already in hand, so the width choice is a compare rather than a stored per-row - // flag. - fj_bin_patch_table(coef_t{})[fj_bin_patch_width_index(ke - kb)]( - variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, sign, weight, os_new, - skip_var); + fj_bin_patch_fn(coef_t{})(variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, + sign, weight, os_new, skip_var); } template void fj_bin_patch_row(const int32_t*, From eb4ce7561e8eef61ac03610224037354c8987921 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 08:02:16 -0700 Subject: [PATCH 09/61] perf: normalize every row to a'x <= b and drop the per-row sign The one-sided split emitted the lower-bound side as a'x >= lb and carried an int8 sign per row to reconstruct the slack as sign * (bound - lhs). Negating that side's coefficients and bound at build time makes every row a'x <= b, so the slack is bound - lhs everywhere and the sign disappears from the record, the kernels and every rebuild path. This costs nothing to store: emit() already pushes a separate coefficient copy per side. Negation is safe on both fields -- the eligibility scan admits |coef| up to 127 for int8 and 32767 for int16, and the bound is range-checked after negating rather than before. With sign == 1 everywhere, signed_coefficient became a plain transpose-order copy of the coefficients, i.e. identical to reverse_coefficients, which was written but never read. The two merge under the latter name, which is what the array now is. Removed: pb.sign (0.26 MB), one of the two per-nnz coefficient copies (2.2 MB), the sign field of fj_bin_row_t (leaving a bare weight), one argument from fj_bin_patch_row, and the vsign multiply from both the vector and scalar patch arms. supportcase22, 20s x 16 climbers, mean of 3: 4.354 -> 4.396 M iters/s (+0.96%). Trajectory is bit-identical: 84951 common (climber, iteration) samples across all 16 climbers match on viol, best and maxw, and the incumbent audit passes on every run. Signed-off-by: Alice Boucher --- .../feasibility_jump/fj_cpu_binary.cu | 70 +++++++++---------- .../feasibility_jump/fj_cpu_binary.cuh | 5 +- .../fj_cpu_binary_kernels.cpp | 48 +++++-------- 3 files changed, 53 insertions(+), 70 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index a63d3ad802..f43a37aec0 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -149,15 +149,14 @@ struct fj_bin_tabu_t { // What the apply path still needs per row once the row walk is vectorized: everything else it used // to read from here now reaches it at unit stride. // -// The mutable state moved out to the engine's row_slack, which holds sign * (bound - lhs) rather -// than lhs, because that is what turns the walk's update into a subtraction and lets it gather one -// array instead of four. bound went with it -- only the rebuild paths need it, and they read -// pb.bound. cmax went to pb.incident_row_cmax, replicated per incidence. sign stays because -// fj_bin_patch_row still takes it, and weight because it is genuinely mutable. +// The mutable state moved out to the engine's row_slack, which holds bound - lhs rather than lhs, +// because that is what turns the walk's update into a subtraction and lets it gather one array +// instead of four. bound went with it -- only the rebuild paths need it, and they read pb.bound. +// cmax went to pb.incident_row_cmax, replicated per incidence. Every row is stored as a'x <= b, so +// there is no sign to carry. Only the weight is left, and it is genuinely mutable. template struct fj_bin_row_t { int32_t weight; - int8_t sign; }; // Narrowed problem: one-sided rows, integer coefficients, CSR plus its transpose. @@ -173,17 +172,15 @@ struct fj_bin_problem_t { std::vector reverse_offsets; std::vector reverse_constraints; - std::vector reverse_coefficients; std::vector reverse_to_csr; - // Per incidence, for the vectorized row walk: sign * coefficient folded once, and the row's cmax - // replicated. Both are structural. Indexed like the transpose above, so the walk reads them at - // unit stride instead of gathering sign, coefficient and cmax per row. - std::vector signed_coefficient; + // Per incidence, for the vectorized row walk: the coefficient and the row's cmax, both replicated + // in transpose order so the walk reads them at unit stride instead of gathering per row. Both are + // structural. + std::vector reverse_coefficients; std::vector incident_row_cmax; std::vector bound; - std::vector sign; std::vector cmax; std::vector initial_weight; @@ -335,7 +332,6 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, pb.offsets.assign(1, 0); pb.offsets.reserve(n_split + 1); pb.bound.reserve(n_split); - pb.sign.reserve(n_split); pb.cmax.reserve(n_split); pb.initial_weight.reserve(n_split); @@ -344,11 +340,17 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, // Each split row inherits the weight of the side it came from: left is the lower-bound side, // right the upper. - auto emit = [&](int32_t r, double side_bound, int8_t sign, double weight) -> bool { + // + // Both sides are stored as a'x <= b. The lower-bound side is negated on the way in, which costs + // nothing because each side already gets its own copy of the row, and it leaves the slack as + // bound - lhs everywhere -- so no per-row sign reaches the engine at all. Negation is safe on both + // fields: the scan admits |coef| up to 127 for int8 and 32767 for int16, and the bound is checked + // for int32 range after negating. + auto emit = [&](int32_t r, double side_bound, long side, double weight) -> bool { coef_t row_cmax = 1; for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { const double a = coeffs[k]; - const long ai = std::lround(a); + const long ai = side * std::lround(a); if (!is_integer(a, tol) || ai < std::numeric_limits::min() || ai > std::numeric_limits::max()) { return false; @@ -358,11 +360,10 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, const coef_t abs_a = (coef_t)std::labs(ai); if (abs_a > row_cmax) row_cmax = abs_a; } - const long b = std::lround(side_bound); + const long b = side * std::lround(side_bound); if (!fj_bin_in_int32((double)b)) return false; pb.offsets.push_back((int32_t)pb.variables.size()); pb.bound.push_back((int32_t)b); - pb.sign.push_back(sign); pb.cmax.push_back(row_cmax); incoming_weight.push_back(weight); return true; @@ -371,8 +372,8 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, for (int32_t r = 0; r < m; ++r) { const double lb = cstr_lb[r]; const double ub = cstr_ub[r]; - if (std::isfinite(lb) && !emit(r, lb, (int8_t)-1, left_w[r])) return false; - if (std::isfinite(ub) && !emit(r, ub, (int8_t)1, right_w[r])) return false; + if (std::isfinite(lb) && !emit(r, lb, -1, left_w[r])) return false; + if (std::isfinite(ub) && !emit(r, ub, 1, right_w[r])) return false; } if ((int32_t)pb.bound.size() != n_split) return false; pb.nnz = (int32_t)pb.variables.size(); @@ -416,20 +417,16 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, pb.reverse_constraints.resize(pb.nnz); pb.reverse_coefficients.resize(pb.nnz); pb.reverse_to_csr.resize(pb.nnz); - pb.signed_coefficient.resize(pb.nnz); pb.incident_row_cmax.resize(pb.nnz); { std::vector cursor(pb.reverse_offsets.begin(), pb.reverse_offsets.begin() + n); for (int32_t r = 0; r < n_split; ++r) { for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { - const int32_t slot = cursor[pb.variables[k]]++; - pb.reverse_constraints[slot] = r; + const int32_t slot = cursor[pb.variables[k]]++; + pb.reverse_constraints[slot] = r; pb.reverse_coefficients[slot] = pb.coefficients[k]; pb.reverse_to_csr[slot] = k; - // The scan admits int8 only up to |coef| 127 and int16 only up to 32767, so negating a - // coefficient cannot overflow its own width. - pb.signed_coefficient[slot] = (coef_t)(pb.sign[r] * pb.coefficients[k]); - pb.incident_row_cmax[slot] = pb.cmax[r]; + pb.incident_row_cmax[slot] = pb.cmax[r]; } } } @@ -439,7 +436,7 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, const int32_t rpad = fj_bin_pf_dist > fj_bin_simd_padding ? fj_bin_pf_dist : fj_bin_simd_padding; pb.reverse_constraints.resize(pb.nnz + rpad, 0); - pb.signed_coefficient.resize(pb.nnz + rpad, (coef_t)0); + pb.reverse_coefficients.resize(pb.nnz + rpad, (coef_t)0); pb.incident_row_cmax.resize(pb.nnz + rpad, (coef_t)1); pb.objective.resize(n); @@ -458,8 +455,8 @@ struct fj_bin_engine_t { fj_bin_problem_t pb; std::vector> rows; - // Per row, sign * (bound - lhs): negative exactly when the row is violated, and moved by a flip - // by exactly -signed_coefficient. The only mutable state the vectorized walk gathers. + // Per row, bound - lhs: negative exactly when the row is violated, and moved by a flip by exactly + // -reverse_coefficients. The only mutable state the vectorized walk gathers. std::vector row_slack; std::vector assign; @@ -541,7 +538,7 @@ struct fj_bin_engine_t { lhs += (int64_t)pb.coefficients[k] * (int64_t)best_assign[pb.variables[k]]; } if (lhs < INT32_MIN || lhs > INT32_MAX) lhs_overflow = true; - const int64_t slack = (int64_t)pb.sign[r] * ((int64_t)pb.bound[r] - lhs); + const int64_t slack = (int64_t)pb.bound[r] - lhs; if (slack < 0) { ++n_violated; if (-slack > worst) worst = -slack; @@ -582,7 +579,7 @@ struct fj_bin_engine_t { for (int32_t i = pb.reverse_offsets[v]; i < pb.reverse_offsets[v + 1]; ++i) { const fj_bin_row_t& h = rows[pb.reverse_constraints[i]]; const int32_t os = row_slack[pb.reverse_constraints[i]]; - const int32_t ns = os - (int32_t)pb.signed_coefficient[i] * flip; + const int32_t ns = os - (int32_t)pb.reverse_coefficients[i] * flip; int32_t base = 0, bonus = 0; fj_bin_score_delta_parts(os, ns, h.weight, base, bonus); agg_base += base; @@ -624,12 +621,11 @@ struct fj_bin_engine_t { std::fill(var_score.begin(), var_score.end(), 0); for (int32_t r = 0; r < pb.n_constraints; ++r) { const int32_t weight = rows[r].weight; - const int32_t sign = rows[r].sign; const int32_t os = row_slack[r]; for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { const int32_t v = pb.variables[k]; const int32_t flip = 1 - 2 * assign[v]; - const int32_t ns = os - sign * ((int32_t)pb.coefficients[k] * flip); + const int32_t ns = os - (int32_t)pb.coefficients[k] * flip; const int32_t p = fj_bin_packed_score_delta(os, ns, weight); nnz_score_delta[k] = p; var_score[v] += p; @@ -646,7 +642,7 @@ struct fj_bin_engine_t { int32_t lhs = 0; for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) lhs += (int32_t)pb.coefficients[k] * assign[pb.variables[k]]; - const int32_t slack = pb.sign[r] * (pb.bound[r] - lhs); + const int32_t slack = pb.bound[r] - lhs; row_slack[r] = slack; if (slack < 0) set_violated(r); } @@ -693,7 +689,7 @@ struct fj_bin_engine_t { fj_bin_row_t* const rows_p = rows.data(); int32_t* const slack_p = row_slack.data(); const int32_t* const rcon_p = pb.reverse_constraints.data(); - const coef_t* const skv_p = pb.signed_coefficient.data(); + const coef_t* const skv_p = pb.reverse_coefficients.data(); const coef_t* const rcmax_p = pb.incident_row_cmax.data(); const int32_t* const rcsr_p = pb.reverse_to_csr.data(); const int32_t* const offsets_p = pb.offsets.data(); @@ -734,7 +730,6 @@ struct fj_bin_engine_t { var_score_p, nnz_delta_p, assign_p, - h.sign, h.weight, new_slack, var); @@ -822,7 +817,6 @@ struct fj_bin_engine_t { var_score.data(), nnz_score_delta.data(), assign_i32.data(), - h.sign, h.weight, row_slack[r], -1); @@ -1033,7 +1027,7 @@ struct fj_bin_engine_t { rows.resize(m); for (int32_t r = 0; r < m; ++r) - rows[r] = fj_bin_row_t{pb.initial_weight[r], pb.sign[r]}; + rows[r] = fj_bin_row_t{pb.initial_weight[r]}; row_slack.assign(m, 0); var_score.assign(n, 0); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh index e7cc1a2818..9b45af515d 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -80,7 +80,6 @@ void fj_bin_patch_row(const int32_t* variables, int32_t* var_score, int32_t* nnz_score_delta, const int32_t* assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var); @@ -91,13 +90,13 @@ constexpr int32_t fj_bin_walk_tile = 256; // must finish by hand (e.g. if the row needs patching) // // For every incidence i in the range this applies -// row_slack[incident_row[i]] -= signed_coefficient[i] * delta +// row_slack[incident_row[i]] -= reverse_coefficients[i] * delta // then writes to out_incidence, in increasing order, the subset of i whose row is not deeply // satisfied on both sides of the flip and returns how many. template int32_t fj_bin_walk_rows(int32_t* row_slack, const int32_t* incident_row, - const coef_t* signed_coefficient, + const coef_t* reverse_coefficients, const coef_t* incident_row_cmax, int32_t incidence_begin, int32_t incidence_end, diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index 5ceb5914aa..f2da24c580 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -57,9 +57,9 @@ constexpr bool k_vector_walk = // The layout this assumes is what makes it worth doing. Storing the row's signed slack rather than // its lhs collapses the update to // -// new_slack = old_slack - sign * coef * delta = old_slack - skv[ii] * delta +// new_slack = old_slack - coef * delta // -// so bound and lhs never appear, and sign and coef fold into one per-incidence constant. skv and +// so bound and lhs never appear, and the coefficient is the only per-incidence constant. It and // cmax are replicated per incidence, which makes them unit-stride loads. What remains irregular is // the slack itself: one gather and one scatter per vector, against four gathers and a scatter for a // literal SoA split of the row record. @@ -70,7 +70,7 @@ constexpr bool k_vector_walk = template int32_t WalkRowsImpl(int32_t* HWY_RESTRICT row_slack, const int32_t* HWY_RESTRICT incident_row, - const coef_t* HWY_RESTRICT signed_coefficient, + const coef_t* HWY_RESTRICT reverse_coefficients, const coef_t* HWY_RESTRICT incident_row_cmax, int32_t incidence_begin, int32_t incidence_end, @@ -94,7 +94,7 @@ int32_t WalkRowsImpl(int32_t* HWY_RESTRICT row_slack, const auto active = hn::FirstN(d, (size_t)(incidence_end - ii)); const V rows = hn::LoadU(d, incident_row + ii); - const V skv = hn::PromoteTo(d, hn::LoadU(dc, signed_coefficient + ii)); + const V skv = hn::PromoteTo(d, hn::LoadU(dc, reverse_coefficients + ii)); const V cmax = hn::PromoteTo(d, hn::LoadU(dc, incident_row_cmax + ii)); const V os = hn::MaskedGatherIndex(active, d, row_slack, rows); @@ -130,7 +130,7 @@ int32_t WalkRowsImpl(int32_t* HWY_RESTRICT row_slack, for (; ii < incidence_end; ++ii) { const int32_t row = incident_row[ii]; const int32_t os = row_slack[row]; - const int32_t ns = os - (int32_t)signed_coefficient[ii] * delta; + const int32_t ns = os - (int32_t)reverse_coefficients[ii] * delta; row_slack[row] = ns; const int32_t cmax = (int32_t)incident_row_cmax[ii]; if (!(os > cmax && ns > cmax)) out_incidence[n_out++] = ii; @@ -147,7 +147,6 @@ void PatchRowScalar(const int32_t* HWY_RESTRICT variables, int32_t* HWY_RESTRICT var_score, int32_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var) @@ -156,7 +155,7 @@ void PatchRowScalar(const int32_t* HWY_RESTRICT variables, const int32_t v = variables[k]; if (v == skip_var) continue; const int32_t flip = 1 - 2 * assign_i32[v]; - const int32_t ns = os_new - sign * ((int32_t)coefficients[k] * flip); + const int32_t ns = os_new - (int32_t)coefficients[k] * flip; const int32_t nc = fj_bin_packed_score_delta(os_new, ns, weight); var_score[v] += nc - nnz_score_delta[k]; nnz_score_delta[k] = nc; @@ -175,7 +174,6 @@ static HWY_INLINE void PatchRowBody(D d, int32_t* HWY_RESTRICT var_score, int32_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var) @@ -189,13 +187,13 @@ static HWY_INLINE void PatchRowBody(D d, if constexpr (!k_mask_remainder) { if ((size_t)(ke - kb) < N) { PatchRowScalar(variables, coefficients, kb, ke, var_score, nnz_score_delta, - assign_i32, sign, weight, os_new, skip_var); + assign_i32, weight, os_new, skip_var); return; } } const V vone = hn::Set(d, 1), vzero = hn::Zero(d); - const V vsign = hn::Set(d, sign), vskip = hn::Set(d, skip_var); + const V vskip = hn::Set(d, skip_var); const V vos = hn::Set(d, os_new); const V vw = hn::Set(d, weight), vw2 = hn::Set(d, weight / 2); @@ -223,7 +221,7 @@ static HWY_INLINE void PatchRowBody(D d, const V flip = vone - hn::ShiftLeft<1>(a01); const V coef = hn::PromoteTo(d, hn::LoadU(dc, coefficients + k)); - const V ns = vos - vsign * coef * flip; + const V ns = vos - coef * flip; const V nsat = hn::IfThenElseZero(hn::Ge(ns, vzero), vone); const V nst = hn::IfThenElseZero(hn::Gt(ns, vzero), vone); @@ -260,7 +258,7 @@ static HWY_INLINE void PatchRowBody(D d, if constexpr (!k_mask_remainder) { PatchRowScalar(variables, coefficients, k, ke, var_score, nnz_score_delta, assign_i32, - sign, weight, os_new, skip_var); + weight, os_new, skip_var); } } @@ -273,13 +271,12 @@ void PatchRowImpl(const int32_t* HWY_RESTRICT variables, int32_t* HWY_RESTRICT var_score, int32_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var) { PatchRowBody(hn::ScalableTag(), variables, coefficients, kb, ke, var_score, - nnz_score_delta, assign_i32, sign, weight, os_new, skip_var); + nnz_score_delta, assign_i32, weight, os_new, skip_var); } template @@ -290,13 +287,12 @@ void PatchRowNarrow8Impl(const int32_t* HWY_RESTRICT variables, int32_t* HWY_RESTRICT var_score, int32_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var) { PatchRowBody(hn::CappedTagIfFixed(), variables, coefficients, kb, ke, - var_score, nnz_score_delta, assign_i32, sign, weight, os_new, skip_var); + var_score, nnz_score_delta, assign_i32, weight, os_new, skip_var); } template @@ -307,13 +303,12 @@ void PatchRowNarrow4Impl(const int32_t* HWY_RESTRICT variables, int32_t* HWY_RESTRICT var_score, int32_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var) { PatchRowBody(hn::CappedTagIfFixed(), variables, coefficients, kb, ke, - var_score, nnz_score_delta, assign_i32, sign, weight, os_new, skip_var); + var_score, nnz_score_delta, assign_i32, weight, os_new, skip_var); } // Longest row worth sending to each narrower kernel, or 0 where that width is not worth having. @@ -345,7 +340,6 @@ void PatchRowDispatchImpl(const int32_t* HWY_RESTRICT variables, int32_t* HWY_RESTRICT var_score, int32_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var) @@ -353,13 +347,13 @@ void PatchRowDispatchImpl(const int32_t* HWY_RESTRICT variables, const int32_t row_len = ke - kb; if (row_len <= k_narrow4_max) { PatchRowNarrow4Impl(variables, coefficients, kb, ke, var_score, nnz_score_delta, - assign_i32, sign, weight, os_new, skip_var); + assign_i32, weight, os_new, skip_var); } else if (row_len <= k_narrow8_max) { PatchRowNarrow8Impl(variables, coefficients, kb, ke, var_score, nnz_score_delta, - assign_i32, sign, weight, os_new, skip_var); + assign_i32, weight, os_new, skip_var); } else { PatchRowImpl(variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, - sign, weight, os_new, skip_var); + weight, os_new, skip_var); } } @@ -463,7 +457,6 @@ using fj_bin_patch_fn_t = void (*)(const int32_t*, const int32_t*, int32_t, int32_t, - int32_t, int32_t); // The vector width is chosen inside the target (see PatchRowDispatchImpl), so the seam carries one @@ -494,7 +487,7 @@ static const auto fj_bin_argmax_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTE template int32_t fj_bin_walk_rows(int32_t* row_slack, const int32_t* incident_row, - const coef_t* signed_coefficient, + const coef_t* reverse_coefficients, const coef_t* incident_row_cmax, int32_t incidence_begin, int32_t incidence_end, @@ -503,7 +496,7 @@ int32_t fj_bin_walk_rows(int32_t* row_slack, { return fj_bin_walk_fn(coef_t{})(row_slack, incident_row, - signed_coefficient, + reverse_coefficients, incident_row_cmax, incidence_begin, incidence_end, @@ -524,13 +517,12 @@ void fj_bin_patch_row(const int32_t* variables, int32_t* var_score, int32_t* nnz_score_delta, const int32_t* assign_i32, - int32_t sign, int32_t weight, int32_t os_new, int32_t skip_var) { fj_bin_patch_fn(coef_t{})(variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, - sign, weight, os_new, skip_var); + weight, os_new, skip_var); } template void fj_bin_patch_row(const int32_t*, @@ -542,7 +534,6 @@ template void fj_bin_patch_row(const int32_t*, const int32_t*, int32_t, int32_t, - int32_t, int32_t); template void fj_bin_patch_row(const int32_t*, @@ -554,7 +545,6 @@ template void fj_bin_patch_row(const int32_t*, const int32_t*, int32_t, int32_t, - int32_t, int32_t); void fj_bin_argmax(const int32_t* var_score, From b0c192429c32bd47ccfee61084c70763330aca06 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 08:29:06 -0700 Subject: [PATCH 10/61] refactor: replace the row record with a plain weight array Once the sign was normalized away, fj_bin_row_t was a single int32 field in a struct still templated on a coef_t it never used. It is now a std::vector row_weight on the engine, which drops the type, its template parameter, and the reference-into-the-array idiom at every use site. Storage is unchanged -- four bytes per row either way -- and the trajectory is bit-identical: 88030 common (climber, iteration) samples across all 16 climbers match on viol, best and maxw. This costs 0.57% on supportcase22: 4.400 -> 4.375 M iters/s, both means of four runs measured back to back to rule out machine drift. The cause is not in the work done. apply_move comes out smaller and no busier -- 822 instructions against 832, 37 spill stores against 38, identical reload and branch counts -- and the diff is a register allocation and stack slot reshuffle across the whole function rather than anything localized. Same class of second-order codegen effect as the ArgmaxImpl relocation that cost 123 cycles per iteration earlier with a byte-identical instruction stream. Taken deliberately: the simpler type is worth more than 0.5% here. Signed-off-by: Alice Boucher --- .../feasibility_jump/fj_cpu_binary.cu | 56 ++++++++----------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index f43a37aec0..49829ba59f 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -146,19 +146,6 @@ struct fj_bin_tabu_t { } }; -// What the apply path still needs per row once the row walk is vectorized: everything else it used -// to read from here now reaches it at unit stride. -// -// The mutable state moved out to the engine's row_slack, which holds bound - lhs rather than lhs, -// because that is what turns the walk's update into a subtraction and lets it gather one array -// instead of four. bound went with it -- only the rebuild paths need it, and they read pb.bound. -// cmax went to pb.incident_row_cmax, replicated per incidence. Every row is stored as a'x <= b, so -// there is no sign to carry. Only the weight is left, and it is genuinely mutable. -template -struct fj_bin_row_t { - int32_t weight; -}; - // Narrowed problem: one-sided rows, integer coefficients, CSR plus its transpose. template struct fj_bin_problem_t { @@ -453,7 +440,10 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, template struct fj_bin_engine_t { fj_bin_problem_t pb; - std::vector> rows; + // The only mutable per-row state besides the slack. Everything else the apply path once read + // per row now reaches it at unit stride: bound stayed in pb, where only the rebuild paths need + // it, and cmax went to pb.incident_row_cmax, replicated per incidence. + std::vector row_weight; // Per row, bound - lhs: negative exactly when the row is violated, and moved by a flip by exactly // -reverse_coefficients. The only mutable state the vectorized walk gathers. @@ -577,11 +567,11 @@ struct fj_bin_engine_t { const int8_t flip = (int8_t)(1 - 2 * assign[v]); int32_t agg_base = 0, agg_bonus = 0; for (int32_t i = pb.reverse_offsets[v]; i < pb.reverse_offsets[v + 1]; ++i) { - const fj_bin_row_t& h = rows[pb.reverse_constraints[i]]; - const int32_t os = row_slack[pb.reverse_constraints[i]]; + const int32_t r = pb.reverse_constraints[i]; + const int32_t os = row_slack[r]; const int32_t ns = os - (int32_t)pb.reverse_coefficients[i] * flip; int32_t base = 0, bonus = 0; - fj_bin_score_delta_parts(os, ns, h.weight, base, bonus); + fj_bin_score_delta_parts(os, ns, row_weight[r], base, bonus); agg_base += base; agg_bonus += bonus; } @@ -620,7 +610,7 @@ struct fj_bin_engine_t { { std::fill(var_score.begin(), var_score.end(), 0); for (int32_t r = 0; r < pb.n_constraints; ++r) { - const int32_t weight = rows[r].weight; + const int32_t weight = row_weight[r]; const int32_t os = row_slack[r]; for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { const int32_t v = pb.variables[k]; @@ -686,7 +676,7 @@ struct fj_bin_engine_t { // The tail writes a score delta through int32_t* and calls out to the patch, either of which may // alias a vector's internal pointer as far as the compiler can prove. Without these locals it // reloads every base pointer below out of `this` on each visit. - fj_bin_row_t* const rows_p = rows.data(); + int32_t* const weight_p = row_weight.data(); int32_t* const slack_p = row_slack.data(); const int32_t* const rcon_p = pb.reverse_constraints.data(); const coef_t* const skv_p = pb.reverse_coefficients.data(); @@ -703,7 +693,7 @@ struct fj_bin_engine_t { // so the walk's shape is the only thing that differs between them. auto finish = [&](int32_t ii) { const int32_t r = rcon_p[ii]; - fj_bin_row_t& h = rows_p[r]; + const int32_t weight = weight_p[r]; const int32_t skv = (int32_t)skv_p[ii]; const int32_t new_slack = slack_p[r]; const int32_t old_slack = new_slack + skv * delta; @@ -730,7 +720,7 @@ struct fj_bin_engine_t { var_score_p, nnz_delta_p, assign_p, - h.weight, + weight, new_slack, var); nnz_touched += ke - kb; @@ -739,7 +729,7 @@ struct fj_bin_engine_t { // The flipped variable's own score delta. Zero on the rows the walk absorbed -- deeply // satisfied both ways -- and already stored as zero there. - const int32_t pv = fj_bin_packed_score_delta(new_slack, new_slack - skv * new_flip, h.weight); + const int32_t pv = fj_bin_packed_score_delta(new_slack, new_slack - skv * new_flip, weight); own_score += pv; nnz_delta_p[rcsr_p[ii]] = pv; }; @@ -804,9 +794,8 @@ struct fj_bin_engine_t { void reweight_constraint(int32_t r, int32_t new_weight) { - fj_bin_row_t& h = rows[r]; - if (new_weight == h.weight) return; - h.weight = new_weight; + if (new_weight == row_weight[r]) return; + row_weight[r] = new_weight; if (new_weight > max_weight) max_weight = new_weight; // The slack is unchanged here, and no variable is excluded, so skip_var matches no index. const int32_t kb = pb.offsets[r], ke = pb.offsets[r + 1]; @@ -817,7 +806,7 @@ struct fj_bin_engine_t { var_score.data(), nnz_score_delta.data(), assign_i32.data(), - h.weight, + new_weight, row_slack[r], -1); nnz_touched += ke - kb; @@ -829,7 +818,7 @@ struct fj_bin_engine_t { void update_weights() { for (int32_t cf : violated_list) { - reweight_constraint(cf, rows[cf].weight + fj_bin_ddfw_transfer); + reweight_constraint(cf, row_weight[cf] + fj_bin_ddfw_transfer); const int32_t vo = pb.offsets[cf], ve = pb.offsets[cf + 1]; if (ve <= vo) continue; int32_t best_donor = -1, best_w = fj_bin_ddfw_init; @@ -839,12 +828,13 @@ struct fj_bin_engine_t { if (ne <= no) continue; const int32_t d = pb.reverse_constraints[no + (int32_t)(rng.next_u32() % (uint32_t)(ne - no))]; - if (d != cf && !is_violated[d] && rows[d].weight > best_w) { - best_w = rows[d].weight; + if (d != cf && !is_violated[d] && row_weight[d] > best_w) { + best_w = row_weight[d]; best_donor = d; } } - if (best_donor >= 0) reweight_constraint(best_donor, rows[best_donor].weight - fj_bin_ddfw_transfer); + if (best_donor >= 0) + reweight_constraint(best_donor, row_weight[best_donor] - fj_bin_ddfw_transfer); } if (violated_list.empty()) objective_weight += 1; } @@ -983,7 +973,7 @@ struct fj_bin_engine_t { { assign = seed_assign; for (int32_t v = 0; v < pb.n_variables; ++v) assign_i32[v] = assign[v]; - for (int32_t r = 0; r < pb.n_constraints; ++r) rows[r].weight = pb.initial_weight[r]; + for (int32_t r = 0; r < pb.n_constraints; ++r) row_weight[r] = pb.initial_weight[r]; max_weight = fj_bin_ddfw_init; objective_weight = 0; tabu.clear(iters); @@ -1025,9 +1015,7 @@ struct fj_bin_engine_t { assign_i32.assign(n, 0); for (int32_t v = 0; v < n; ++v) assign_i32[v] = assign[v]; - rows.resize(m); - for (int32_t r = 0; r < m; ++r) - rows[r] = fj_bin_row_t{pb.initial_weight[r]}; + row_weight.assign(pb.initial_weight.begin(), pb.initial_weight.end()); row_slack.assign(m, 0); var_score.assign(n, 0); From 2f63ab861c1140b41197e4d46477884c70f1e107 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 09:39:03 -0700 Subject: [PATCH 11/61] fix: derive the argmax tile from the L1 data cache size The tile width of the argmax sweep was ported from the prototype as a bare constant, dropping the residency guard that computed it. The target of 256 is about the shape of the sweep -- it sets how often the running maximum is raised, which is what bounds the index re-scan -- but the re-scan only pays off because it revisits a tile that is still L1-hot. On a small L1 an unguarded 256 pushes that re-read out to L2 and costs more than the split saves, which is what the cap the prototype applied was for. Bytes per variable is the score array alone. The prototype also counted its u16 flip_until, because its sweep tested tabu per lane; this engine blocks the handful of tabu variables at the invalid sentinel before the sweep and restores after, so flip_until is never touched here and the divisor is 4 rather than 6. No behaviour change on the machines we run: at 32 KiB the cap is 2048 and the target binds. supportcase22, 20s x 16 climbers, mean of 3: 4.375 -> 4.364 M iters/s, within the layout noise seen across today's commits, with 1/16 crossed as before. Signed-off-by: Alice Boucher --- .../feasibility_jump/fj_cpu_binary.cu | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 49829ba59f..4954554263 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -203,6 +204,36 @@ static inline bool fj_bin_in_int32(double v) return v >= (double)INT32_MIN && v <= (double)INT32_MAX; } +// Tile width of the argmax sweep, in variables: min(algorithm target, L1-residency cap). +// +// The target is about the shape of the sweep rather than cache capacity -- it sets how often the +// running maximum is raised, which is what bounds the index re-scan -- and 256 is the measured +// optimum. The cap is a residency guard, and it is the reason this is not simply a constant: the +// re-scan pays off only because it revisits a tile that is still L1-hot, so the tile must not be +// wide enough to spill. It bites only on a small L1, where an unguarded 256 would push the re-scan +// out to L2 and cost more than the split saves. +// +// Bytes per variable is the score array alone. Tabu does not appear: the sweep reads var_score +// only, with the handful of tabu variables held at the invalid sentinel across it, so flip_until is +// never touched here. +constexpr int32_t fj_bin_argmax_tile_target = 256; +constexpr int32_t fj_bin_argmax_tile_cap_k = 4; + +static int32_t fj_bin_argmax_tile() +{ +#ifdef _SC_LEVEL1_DCACHE_SIZE + long l1 = sysconf(_SC_LEVEL1_DCACHE_SIZE); +#else + long l1 = 0; +#endif + if (l1 <= 0) l1 = 32768; // fallback: 32 KiB, the common x86 L1d + const int32_t bpv = (int32_t)sizeof(int32_t); + const int32_t cap = (int32_t)(l1 / (fj_bin_argmax_tile_cap_k * bpv)); + int32_t t = fj_bin_argmax_tile_target < cap ? fj_bin_argmax_tile_target : cap; + t &= ~15; // whole vectors + return t < 16 ? 16 : t; +} + // Width-independent eligibility scan over the climber's host mirrors. Mutates nothing. template static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) @@ -488,9 +519,8 @@ struct fj_bin_engine_t { int64_t nnz_patched{0}; int64_t rows_walked{0}; - // Tile width for the argmax sweep. Governs how often the running maximum is raised, which is - // what bounds the index re-scan, so it is about the shape of the sweep and not cache capacity. - int32_t argmax_tile{256}; + // Tile width for the argmax sweep, in variables. Set at init from fj_bin_argmax_tile(). + int32_t argmax_tile{fj_bin_argmax_tile_target}; // Settings read at solve entry, where the climber carries populated values. int32_t seed{0}; @@ -1026,6 +1056,7 @@ struct fj_bin_engine_t { violated_list.clear(); var_bitmap.assign(n, 0); + argmax_tile = fj_bin_argmax_tile(); objective_weight = 0; max_weight = fj_bin_ddfw_init; incumbent_objective = 0; From 9bcb1b79db1202537d69fe3b4198da825764a548 Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 09:40:51 -0700 Subject: [PATCH 12/61] debug: add CUOPT_NO_BINFJ to force the general path The binary fast path and the general path are meant to search identically, so a divergence between them is a bug in the fast path. Setting this env var makes try_cpufj_binary_solve decline unconditionally, which is what turns that comparison into one command on an instance the fast path would otherwise take. Found its first bug already: on chromaticindex1024-7 the general path crosses 16/16 in ~30s with 65k iterations per climber, while the fast path runs 1M iterations per climber and crosses 0/16, never reaching a local minimum and so never engaging DDFW. The cause is packed-score saturation -- the engine's own report gives aggregate bonus 122880 against a limit of 16384 -- which corrupts the lexicographic order the packing encodes. Signed-off-by: Alice Boucher --- cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 4954554263..490bfb6946 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -1157,6 +1158,12 @@ bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, f_t time_limit, double work_unit_limit) { + // Escape hatch for A/B against the general path on an instance the fast path would take. The two + // paths are meant to search identically, so any divergence is a bug in this one; setting this is + // how that gets bisected without editing the eligibility scan. + static const bool disabled = std::getenv("CUOPT_NO_BINFJ") != nullptr; + if (disabled) return false; + const fj_bin_scan_t scan = fj_bin_scan(climber); if (scan.reject != fj_binary_reject_t::none) { CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s (row %d, var %d)", From bd9c3834498757733e2b81913fd042fd4551544b Mon Sep 17 00:00:00 2001 From: Alice Boucher Date: Fri, 7 Aug 2026 10:57:57 -0700 Subject: [PATCH 13/61] fix: widen the packed staged score to int64 The packed score encodes the general path's lexicographic (base, bonus) comparison as one arithmetic comparison, and at 15 bits the bonus field was too narrow to do it. Both fields aggregate over the rows a variable appears in, so each is bounded by max_var_degree * max_weight -- unbounded at build time, since DDFW grows the weights. Where the bonus exceeded 2^14 it carried into the base, manufacturing improvement that was not there, so the argmax ranked moves by a corrupted key, no local minimum was ever detected, and DDFW never engaged. Measured aggregate bonus against the old 16384 limit: chromaticindex1024-7 122880 (7.5x), 30n20b8 binarized 66396 (4.1x), crypt16 20229 (1.23x), bnatt400_reduced 17611 (1.07x). Only supportcase22 stayed inside, at 5751. Shift becomes 32, so base has the whole int32 range before the encoding can break. The two fields are per-row where they are computed, so the kernel still evaluates them as int32 at full lane count and widens only for the pack; the pack, the previous value, the difference and the store back all stay in the vector, leaving the scalar lane loop at one add per nonzero as it was before. That loop is 38% of all cycles, so it is the wrong place for work: a first version that packed there instead cost 26.7% rather than 17.8%, and a variant that also encoded the per-nnz delta as two int16 fields to halve that array cost 31.5% -- the patch path is instruction-bound, not bandwidth-bound. chromaticindex1024-7 goes from 0/16 crossings to 16/16, first crossing at 0.13s against 23-33s on the general path, same objective 4. supportcase22 costs 17.8% (4.364 -> 3.589 M iters/s) and is the one instance whose trajectory is unchanged, which is what the saturation numbers predict. crypt16 11/16 and bnatt400_reduced 16/16 objective 1 both hold. Not explained: on supportcase22 two of sixteen climbers diverge from the int32 trajectory past iteration 4.25M (887 of 72338 sampled points); the other fourteen are bit-identical. Left open deliberately rather than dismissed. Signed-off-by: Alice Boucher --- .../feasibility_jump/fj_cpu_binary.cu | 64 +++++----- .../feasibility_jump/fj_cpu_binary.cuh | 28 +++-- .../fj_cpu_binary_kernels.cpp | 113 ++++++++++++------ 3 files changed, 127 insertions(+), 78 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 490bfb6946..b65a1d8bf9 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -105,9 +105,9 @@ struct fj_bin_tabu_t { // replace the scores of tabu'd variable with sentinel values int32_t block_tabu(int32_t iter, - int32_t* var_score, + int64_t* var_score, int32_t (&saved_var)[ring_size], - int32_t (&saved_score)[ring_size]) const + int64_t (&saved_score)[ring_size]) const { int32_t k = 0; for (int32_t i = 0; i < ring_size; ++i) { @@ -124,9 +124,9 @@ struct fj_bin_tabu_t { // reverse the above operation. static void unblock_tabu(int32_t k, - int32_t* var_score, + int64_t* var_score, const int32_t (&saved_var)[ring_size], - const int32_t (&saved_score)[ring_size]) + const int64_t (&saved_score)[ring_size]) { for (int32_t i = k - 1; i >= 0; --i) var_score[saved_var[i]] = saved_score[i]; } @@ -486,8 +486,8 @@ struct fj_bin_engine_t { std::vector seed_assign; // restart target std::vector assign_i32; // gather mirror for the SIMD patch (Batch B) - std::vector var_score; // live feasibility score of flipping each variable - std::vector nnz_score_delta; // per CSR nnz: last score delta of variables[k] in its row + std::vector var_score; // live feasibility score of flipping each variable + std::vector nnz_score_delta; // per CSR nnz: last score delta of variables[k] in its row fj_bin_tabu_t tabu; @@ -647,7 +647,7 @@ struct fj_bin_engine_t { const int32_t v = pb.variables[k]; const int32_t flip = 1 - 2 * assign[v]; const int32_t ns = os - (int32_t)pb.coefficients[k] * flip; - const int32_t p = fj_bin_packed_score_delta(os, ns, weight); + const int64_t p = fj_bin_packed_score_delta(os, ns, weight); nnz_score_delta[k] = p; var_score[v] += p; } @@ -673,7 +673,7 @@ struct fj_bin_engine_t { rebuild_scores(); } - int32_t objective_terms(int32_t v, int8_t delta) const + int64_t objective_terms(int32_t v, int8_t delta) const { const double obj_diff = pb.objective[v] * delta; const int32_t base = obj_diff < 0 ? objective_weight : (obj_diff > 0 ? -objective_weight : 0); @@ -685,10 +685,10 @@ struct fj_bin_engine_t { } else if (old_better && !new_better) { bonus -= objective_weight; } - return base * fj_bin_score_k + bonus; + return (int64_t)base * fj_bin_score_k + bonus; } - int32_t full_score(int32_t v, int8_t delta) const + int64_t full_score(int32_t v, int8_t delta) const { if (objective_weight == 0) return var_score[v]; return var_score[v] + objective_terms(v, delta); @@ -702,7 +702,7 @@ struct fj_bin_engine_t { const int8_t new_flip = (int8_t)(1 - 2 * new_val); const int32_t ob = pb.reverse_offsets[var], oe = pb.reverse_offsets[var + 1]; const int32_t prev_violated = (int32_t)violated_list.size(); - int32_t own_score = 0; + int64_t own_score = 0; // The tail writes a score delta through int32_t* and calls out to the patch, either of which may // alias a vector's internal pointer as far as the compiler can prove. Without these locals it @@ -716,8 +716,8 @@ struct fj_bin_engine_t { const int32_t* const offsets_p = pb.offsets.data(); const int32_t* const vars_p = pb.variables.data(); const coef_t* const coefs_p = pb.coefficients.data(); - int32_t* const var_score_p = var_score.data(); - int32_t* const nnz_delta_p = nnz_score_delta.data(); + int64_t* const var_score_p = var_score.data(); + int64_t* const nnz_delta_p = nnz_score_delta.data(); const int32_t* const assign_p = assign_i32.data(); // Everything a visit still needs once its slack has been advanced. Shared by the two arms below @@ -760,7 +760,7 @@ struct fj_bin_engine_t { // The flipped variable's own score delta. Zero on the rows the walk absorbed -- deeply // satisfied both ways -- and already stored as zero there. - const int32_t pv = fj_bin_packed_score_delta(new_slack, new_slack - skv * new_flip, weight); + const int64_t pv = fj_bin_packed_score_delta(new_slack, new_slack - skv * new_flip, weight); own_score += pv; nnz_delta_p[rcsr_p[ii]] = pv; }; @@ -873,25 +873,28 @@ struct fj_bin_engine_t { // Global argmax over every variable, affordable because var_score is maintained live. While the // objective weight is zero the full score is exactly var_score, which is the vectorized sweep's // precondition; the objective and local-minimum paths fall to the scalar loop. - std::pair find_move_global(bool localmin) + std::pair find_move_global(bool localmin) { if (!localmin && objective_weight == 0) { // The sweep reads var_score alone; the handful of tabu variables are held at the invalid // sentinel across it rather than tested per variable. - int32_t saved_var[fj_bin_tabu_t::ring_size], saved_score[fj_bin_tabu_t::ring_size]; + int32_t saved_var[fj_bin_tabu_t::ring_size]; + int64_t saved_score[fj_bin_tabu_t::ring_size]; const int32_t blocked = tabu.block_tabu(iters, var_score.data(), saved_var, saved_score); - int32_t v = -1, s = fj_bin_score_invalid; + int32_t v = -1; + int64_t s = fj_bin_score_invalid; fj_bin_argmax(var_score.data(), pb.n_variables, argmax_tile, v, s); fj_bin_tabu_t::unblock_tabu(blocked, var_score.data(), saved_var, saved_score); return {v, s}; } - int32_t best_v = -1, best_s = fj_bin_score_invalid; + int32_t best_v = -1; + int64_t best_s = fj_bin_score_invalid; for (int32_t v = 0; v < pb.n_variables; ++v) { if (tabu_blocked(v, localmin)) continue; - const int32_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); + const int64_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); if (s > best_s) { best_s = s; best_v = v; @@ -900,17 +903,18 @@ struct fj_bin_engine_t { return {best_v, best_s}; } - std::pair find_move_in_rows(const std::vector& target_rows, + std::pair find_move_in_rows(const std::vector& target_rows, bool localmin) { - int32_t best_v = -1, best_s = fj_bin_score_invalid; + int32_t best_v = -1; + int64_t best_s = fj_bin_score_invalid; for (int32_t r : target_rows) { for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { const int32_t v = pb.variables[k]; if (var_bitmap[v]) continue; var_bitmap[v] = 1; if (tabu_blocked(v, localmin)) continue; - const int32_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); + const int64_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); if (s > best_s) { best_s = s; best_v = v; @@ -925,7 +929,7 @@ struct fj_bin_engine_t { return {best_v, best_s}; } - std::pair find_move_violated(int32_t sample_size, bool localmin) + std::pair find_move_violated(int32_t sample_size, bool localmin) { // Draw the rows directly instead of reservoir-sampling the violated list: `std::sample` is // linear in the population, so it walked every violated row to keep a handful. Sampling with @@ -952,14 +956,14 @@ struct fj_bin_engine_t { if (target > 1) target = 1; if ((int8_t)target == assign[v]) continue; if (tabu_blocked(v, false)) continue; - const int32_t s = full_score(v, (int8_t)((int8_t)target - assign[v])); + const int64_t s = full_score(v, (int8_t)((int8_t)target - assign[v])); if (s > move.second) move = {v, s}; } } return move; } - std::pair find_move_satisfied(int32_t sample_size) + std::pair find_move_satisfied(int32_t sample_size) { sample_buf.clear(); for (int32_t tries = 0; (int32_t)sample_buf.size() < sample_size && tries < sample_size * 8; @@ -970,14 +974,15 @@ struct fj_bin_engine_t { return find_move_in_rows(sample_buf, false); } - std::pair find_lift_move() const + std::pair find_lift_move() const { - int32_t best_v = -1, best_s = 0; + int32_t best_v = -1; + int64_t best_s = 0; for (int32_t v : pb.objective_vars) { const int8_t delta = (int8_t)(1 - 2 * assign[v]); if ((double)delta * pb.objective[v] >= 0) continue; if (tabu_blocked(v, false)) continue; - const int32_t s = (int32_t)(-std::llround(pb.objective[v] * delta)) * fj_bin_score_k; + const int64_t s = (int64_t)(-std::llround(pb.objective[v] * delta)) * fj_bin_score_k; if (s > best_s) { best_s = s; best_v = v; @@ -1083,7 +1088,8 @@ struct fj_bin_engine_t { if (iters - last_restart_iter >= fj_bin_restart_period) do_restart(); tabu.maybe_rebase(iters); - int32_t move_var = -1, score = fj_bin_score_invalid; + int32_t move_var = -1; + int64_t score = fj_bin_score_invalid; if (violated_list.empty()) std::tie(move_var, score) = find_lift_move(); if (score <= 0) std::tie(move_var, score) = find_move_global(false); if (feasible_found && score <= 0) std::tie(move_var, score) = find_move_satisfied(mtm_sat_samples); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh index 9b45af515d..524ead8780 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -39,10 +39,18 @@ bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, f_t time_limit, double work_unit_limit); -// Packed staged score: one int32 holding base * K + bonus -constexpr int32_t fj_bin_score_shift = 15; -constexpr int32_t fj_bin_score_k = 1 << fj_bin_score_shift; -constexpr int32_t fj_bin_score_invalid = INT32_MIN; +// Packed staged score: one int64 holding base * K + bonus, encoding the general path's +// lexicographic (base, bonus) comparison as a single arithmetic one. +// +// The width is what makes the encoding faithful. Both fields aggregate over the rows a variable +// appears in, so each is bounded by max_var_degree * max_weight -- unbounded above at build time, +// since DDFW grows the weights. At 15 bits the bonus field overflowed into the base on real +// instances (chromaticindex1024-7 reaches an aggregate bonus of 122880 against 16384), which +// silently corrupts the ordering the argmax depends on. 32 bits leaves the base free to use the +// whole int32 range before the encoding can break. +constexpr int32_t fj_bin_score_shift = 32; +constexpr int64_t fj_bin_score_k = (int64_t)1 << fj_bin_score_shift; +constexpr int64_t fj_bin_score_invalid = INT64_MIN; // Change in one row's weighted score when one variable flips, from the row's signed slack before // (os) and after (ns) that flip. base is the weighted change in satisfaction; bonus is the @@ -59,11 +67,11 @@ static inline void fj_bin_score_delta_parts( bonus = weight * (nst - ost); } -static inline int32_t fj_bin_packed_score_delta(int32_t os, int32_t ns, int32_t weight) +static inline int64_t fj_bin_packed_score_delta(int32_t os, int32_t ns, int32_t weight) { int32_t base = 0, bonus = 0; fj_bin_score_delta_parts(os, ns, weight, base, bonus); - return base * fj_bin_score_k + bonus; + return (int64_t)base * fj_bin_score_k + bonus; } // Padding margin to prevent faults on tail SIMD loads @@ -77,8 +85,8 @@ void fj_bin_patch_row(const int32_t* variables, const coef_t* coefficients, int32_t kb, int32_t ke, - int32_t* var_score, - int32_t* nnz_score_delta, + int64_t* var_score, + int64_t* nnz_score_delta, const int32_t* assign_i32, int32_t weight, int32_t os_new, @@ -107,10 +115,10 @@ int32_t fj_bin_walk_rows(int32_t* row_slack, // the full score is exactly var_score. Yields best_var of -1 only if n is 0. // Tabu is handled by "blocking" the scores corresponding to the tabu vars, and restoring them after the argmax // affordable since max_tenure is small -void fj_bin_argmax(const int32_t* var_score, +void fj_bin_argmax(const int64_t* var_score, int32_t n, int32_t tile, int32_t& best_var, - int32_t& best_score); + int64_t& best_score); } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index f2da24c580..915a98f9d4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -144,8 +144,8 @@ void PatchRowScalar(const int32_t* HWY_RESTRICT variables, const coef_t* HWY_RESTRICT coefficients, int32_t kb, int32_t ke, - int32_t* HWY_RESTRICT var_score, - int32_t* HWY_RESTRICT nnz_score_delta, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, int32_t weight, int32_t os_new, @@ -156,7 +156,7 @@ void PatchRowScalar(const int32_t* HWY_RESTRICT variables, if (v == skip_var) continue; const int32_t flip = 1 - 2 * assign_i32[v]; const int32_t ns = os_new - (int32_t)coefficients[k] * flip; - const int32_t nc = fj_bin_packed_score_delta(os_new, ns, weight); + const int64_t nc = fj_bin_packed_score_delta(os_new, ns, weight); var_score[v] += nc - nnz_score_delta[k]; nnz_score_delta[k] = nc; } @@ -171,16 +171,20 @@ static HWY_INLINE void PatchRowBody(D d, const coef_t* HWY_RESTRICT coefficients, int32_t kb, int32_t ke, - int32_t* HWY_RESTRICT var_score, - int32_t* HWY_RESTRICT nnz_score_delta, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, int32_t weight, int32_t os_new, int32_t skip_var) { - const hn::Rebind dc; // same lane count, narrower lanes - using V = hn::Vec; - const size_t N = hn::Lanes(d); + const hn::Rebind dc; // same lane count, narrower lanes + const hn::Repartition dw; // half the lanes, twice as wide: the packed score + const hn::Half dh; // int32 half, the source of each promotion + using V = hn::Vec; + using VW = hn::Vec; + const size_t N = hn::Lanes(d); + const size_t NW = hn::Lanes(dw); // When the remainder is peeled, a row below one vector never reaches the body, so it skips the // ten broadcasts below as well. @@ -231,17 +235,45 @@ static HWY_INLINE void PatchRowBody(D d, const V both_violated = v_not_osat * (vone - nsat); const V base = vw * (nsat - vosat) + both_violated * improving * vw2; const V bonus = vw * (nst - vost); - const V packed_new = hn::ShiftLeft(base) + bonus; - const V delta = packed_new - hn::LoadU(d, nnz_score_delta + k); + // The score is int64, so packing it costs two vectors where the fields took one. Both fields + // are per-row here and fit int32, so they are computed at full lane count above and widened + // only for the pack. Everything below stays in the vector: the pack, the old value, the + // difference and the store back. What reaches the scalar loop is one add per nonzero, which is + // what it was before the score widened -- that loop is 38% of all cycles, so work belongs + // anywhere but there. + const VW base_lo = hn::PromoteTo(dw, hn::LowerHalf(dh, base)); + const VW base_hi = hn::PromoteTo(dw, hn::UpperHalf(dh, base)); + const VW bonus_lo = hn::PromoteTo(dw, hn::LowerHalf(dh, bonus)); + const VW bonus_hi = hn::PromoteTo(dw, hn::UpperHalf(dh, bonus)); + + const VW packed_lo = hn::ShiftLeft(base_lo) + bonus_lo; + const VW packed_hi = hn::ShiftLeft(base_hi) + bonus_hi; + + const VW delta_lo = packed_lo - hn::LoadU(dw, nnz_score_delta + k); + const VW delta_hi = packed_hi - hn::LoadU(dw, nnz_score_delta + k + NW); + + // The store mask is rebuilt at int64 width rather than narrowed from `active`: the same two + // conditions, on the promoted indices. FirstN is applied on every target because where the + // remainder is peeled the body never runs short, so it is all-true there anyway. + const size_t rem = (size_t)(ke - k); + const VW v_lo = hn::PromoteTo(dw, hn::LowerHalf(dh, v)); + const VW v_hi = hn::PromoteTo(dw, hn::UpperHalf(dh, v)); + const VW vskip_w = hn::Set(dw, skip_var); + const auto act_lo = hn::And(hn::Ne(v_lo, vskip_w), hn::FirstN(dw, rem)); + const auto act_hi = hn::And(hn::Ne(v_hi, vskip_w), hn::FirstN(dw, rem > NW ? rem - NW : 0)); + hn::BlendedStore(packed_lo, act_lo, dw, nnz_score_delta + k); + hn::BlendedStore(packed_hi, act_hi, dw, nnz_score_delta + k + NW); #if HWY_TARGET == HWY_AVX3_ZEN4 // zmm VSIB is microcode on Zen 4: VPGATHERDD ~76-80 uops / ~21 CPI and VPSCATTERDD 89 / 24, // against ~5 / ~10 and ~19 / ~11 on SPR-class Intel (Agner Fog, uops.info). So read-modify-write // by lane here; measured +5.8% over the arm below on an EPYC 9554 (supportcase22, 16 climbers). - HWY_ALIGN int32_t idx[hn::MaxLanes(d)], dl[hn::MaxLanes(d)]; + HWY_ALIGN int32_t idx[hn::MaxLanes(d)]; + HWY_ALIGN int64_t dl[hn::MaxLanes(d)]; hn::Store(v, d, idx); - hn::Store(delta, d, dl); + hn::Store(delta_lo, dw, dl); + hn::Store(delta_hi, dw, dl + NW); // Bounded by the row, not the vector: the lanes past it hold padding, whose zero index would // otherwise be applied to variable 0. const size_t lanes = HWY_MIN(N, (size_t)(ke - k)); @@ -249,11 +281,13 @@ static HWY_INLINE void PatchRowBody(D d, if (idx[i] != skip_var) var_score[idx[i]] += dl[i]; } #else - const V current = hn::MaskedGatherIndex(active, d, var_score, v); - hn::MaskedScatterIndex(current + delta, active, d, var_score, v); + // The score is int64, so the gather and scatter run at the promoted width against the promoted + // indices, in the two halves the pack already produced. + const VW cur_lo = hn::MaskedGatherIndex(act_lo, dw, var_score, v_lo); + const VW cur_hi = hn::MaskedGatherIndex(act_hi, dw, var_score, v_hi); + hn::MaskedScatterIndex(cur_lo + delta_lo, act_lo, dw, var_score, v_lo); + hn::MaskedScatterIndex(cur_hi + delta_hi, act_hi, dw, var_score, v_hi); #endif - - hn::BlendedStore(packed_new, active, d, nnz_score_delta + k); } if constexpr (!k_mask_remainder) { @@ -268,8 +302,8 @@ void PatchRowImpl(const int32_t* HWY_RESTRICT variables, const coef_t* HWY_RESTRICT coefficients, int32_t kb, int32_t ke, - int32_t* HWY_RESTRICT var_score, - int32_t* HWY_RESTRICT nnz_score_delta, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, int32_t weight, int32_t os_new, @@ -284,8 +318,8 @@ void PatchRowNarrow8Impl(const int32_t* HWY_RESTRICT variables, const coef_t* HWY_RESTRICT coefficients, int32_t kb, int32_t ke, - int32_t* HWY_RESTRICT var_score, - int32_t* HWY_RESTRICT nnz_score_delta, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, int32_t weight, int32_t os_new, @@ -300,8 +334,8 @@ void PatchRowNarrow4Impl(const int32_t* HWY_RESTRICT variables, const coef_t* HWY_RESTRICT coefficients, int32_t kb, int32_t ke, - int32_t* HWY_RESTRICT var_score, - int32_t* HWY_RESTRICT nnz_score_delta, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, int32_t weight, int32_t os_new, @@ -337,8 +371,8 @@ void PatchRowDispatchImpl(const int32_t* HWY_RESTRICT variables, const coef_t* HWY_RESTRICT coefficients, int32_t kb, int32_t ke, - int32_t* HWY_RESTRICT var_score, - int32_t* HWY_RESTRICT nnz_score_delta, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, const int32_t* HWY_RESTRICT assign_i32, int32_t weight, int32_t os_new, @@ -360,13 +394,13 @@ void PatchRowDispatchImpl(const int32_t* HWY_RESTRICT variables, // Tiled sweep carrying a running maximum. The index re-scan fires only on a tile that raises it, // and that tile is still cache-hot. The tabu window is uint16 against int32 scores, so the mask // crosses a 2:1 width boundary through PromoteMaskTo. -void ArgmaxImpl(const int32_t* HWY_RESTRICT var_score, +void ArgmaxImpl(const int64_t* HWY_RESTRICT var_score, int32_t n, int32_t tile, int32_t* best_var, - int32_t* best_score) + int64_t* best_score) { - const hn::ScalableTag d; + const hn::ScalableTag d; using V = hn::Vec; const int32_t step = (int32_t)hn::Lanes(d); @@ -377,7 +411,8 @@ void ArgmaxImpl(const int32_t* HWY_RESTRICT var_score, int32_t tile_step = tile - (tile % step); if (tile_step < step) tile_step = step; - int32_t bv = -1, bs = fj_bin_score_invalid; + int32_t bv = -1; + int64_t bs = fj_bin_score_invalid; for (int32_t t0 = 0; t0 < nblk; t0 += tile_step) { const int32_t t1 = (t0 + tile_step < nblk) ? t0 + tile_step : nblk; @@ -387,7 +422,7 @@ void ArgmaxImpl(const int32_t* HWY_RESTRICT var_score, tile_max = hn::Max(tile_max, hn::LoadU(d, var_score + v)); } - const int32_t peak = hn::ReduceMax(d, tile_max); + const int64_t peak = hn::ReduceMax(d, tile_max); if (peak > bs) { const V vpeak = hn::Set(d, peak); for (int32_t v = t0; v < t1; v += step) { @@ -452,8 +487,8 @@ using fj_bin_patch_fn_t = void (*)(const int32_t*, const coef_t*, int32_t, int32_t, - int32_t*, - int32_t*, + int64_t*, + int64_t*, const int32_t*, int32_t, int32_t, @@ -514,8 +549,8 @@ void fj_bin_patch_row(const int32_t* variables, const coef_t* coefficients, int32_t kb, int32_t ke, - int32_t* var_score, - int32_t* nnz_score_delta, + int64_t* var_score, + int64_t* nnz_score_delta, const int32_t* assign_i32, int32_t weight, int32_t os_new, @@ -529,8 +564,8 @@ template void fj_bin_patch_row(const int32_t*, const int8_t*, int32_t, int32_t, - int32_t*, - int32_t*, + int64_t*, + int64_t*, const int32_t*, int32_t, int32_t, @@ -540,18 +575,18 @@ template void fj_bin_patch_row(const int32_t*, const int16_t*, int32_t, int32_t, - int32_t*, - int32_t*, + int64_t*, + int64_t*, const int32_t*, int32_t, int32_t, int32_t); -void fj_bin_argmax(const int32_t* var_score, +void fj_bin_argmax(const int64_t* var_score, int32_t n, int32_t tile, int32_t& best_var, - int32_t& best_score) + int64_t& best_score) { fj_bin_argmax_fn(var_score, n, tile, &best_var, &best_score); } From c308ce496594e1c6778fbe0bfce91815f1d9bbf2 Mon Sep 17 00:00:00 2001 From: yboucher Date: Tue, 18 Aug 2026 13:51:49 -0700 Subject: [PATCH 14/61] add row integralize --- .../feasibility_jump/fj_cpu_binary.cu | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index b65a1d8bf9..fe41614cbb 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -12,6 +12,7 @@ #include #include +#include #include @@ -184,8 +185,11 @@ struct fj_bin_scan_t { int32_t n_split_constraints{0}; int32_t bad_row{-1}; int32_t bad_var{-1}; + std::vector row_scale; }; +constexpr int64_t fj_bin_scale_cap = std::numeric_limits::max(); + // DDFW and restart have no general-path equivalent, so their defaults live here until there is a // reason to promote them alongside the other FJ knobs. constexpr int32_t fj_bin_ddfw_init = 10; // initial weight, also the donation floor @@ -266,15 +270,54 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) const auto& cstr_ub = c.h_cstr_ub; double max_abs_coefficient = 0; + std::vector row_values; for (int32_t r = 0; r < m; ++r) { - double row_abs_sum = 0; + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + const bool lb_fin = std::isfinite(lb); + const bool ub_fin = std::isfinite(ub); + const double sides[2] = {lb, ub}; + const bool finite[2] = {lb_fin, ub_fin}; + + bool fractional_coefficient_seen = false; + bool integral = true; for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { - const double a = coeffs[k]; - if (!is_integer(a, tol)) { - out.reject = fj_binary_reject_t::fractional_coefficient; + if (!is_integer(coeffs[k], tol)) { + fractional_coefficient_seen = true; + integral = false; + break; + } + } + for (int s = 0; s < 2 && integral; ++s) { + if (finite[s] && !is_integer(sides[s], tol)) integral = false; + } + + double row_s = 1.0; + if (!integral) { + row_values.clear(); + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) row_values.push_back(coeffs[k]); + for (int s = 0; s < 2; ++s) { + if (finite[s]) row_values.push_back(sides[s]); + } + row_s = find_scaling_rational(row_values, + /*maxscale=*/1.0 / tol, + /*maxdnom=*/fj_bin_scale_cap, + /*maxfinal=*/(double)fj_bin_scale_cap, + /*intcheck_tol=*/tol); + if (!std::isfinite(row_s) || row_s <= 0.0) { + out.reject = fractional_coefficient_seen ? fj_binary_reject_t::fractional_coefficient + : fj_binary_reject_t::fractional_row_bound; out.bad_row = r; return out; } + if (out.row_scale.empty()) out.row_scale.assign(m, 1.0); + out.row_scale[r] = row_s; + } + + double row_abs_sum = 0; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + const double a = row_s * coeffs[k]; + cuopt_assert(is_integer(a, tol), "row scaling left a fractional coefficient"); const double abs_a = std::fabs(std::round(a)); row_abs_sum += abs_a; if (abs_a > max_abs_coefficient) max_abs_coefficient = abs_a; @@ -288,20 +331,11 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) return out; } - const double lb = cstr_lb[r]; - const double ub = cstr_ub[r]; - const bool lb_fin = std::isfinite(lb); - const bool ub_fin = std::isfinite(ub); - const double sides[2] = {lb, ub}; - const bool finite[2] = {lb_fin, ub_fin}; for (int s = 0; s < 2; ++s) { if (!finite[s]) continue; - if (!is_integer(sides[s], tol)) { - out.reject = fj_binary_reject_t::fractional_row_bound; - out.bad_row = r; - return out; - } - if (!fj_bin_in_int32(std::round(sides[s]))) { + const double scaled_side = row_s * sides[s]; + cuopt_assert(is_integer(scaled_side, tol), "row scaling left a fractional row bound"); + if (!fj_bin_in_int32(std::round(scaled_side))) { out.reject = fj_binary_reject_t::row_bound_out_of_range; out.bad_row = r; return out; @@ -330,9 +364,10 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) // failing check here is a self-consistency bug and refuses the fast path rather than truncating. template static bool fj_bin_narrow(const fj_cpu_climber_t& c, - int32_t n_split, + const fj_bin_scan_t& scan, fj_bin_problem_t& pb) { + const int32_t n_split = scan.n_split_constraints; const int32_t n = c.view.pb.n_variables; const int32_t m = c.view.pb.n_constraints; const double tol = c.view.pb.tolerances.integrality_tolerance; @@ -366,9 +401,10 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, // fields: the scan admits |coef| up to 127 for int8 and 32767 for int16, and the bound is checked // for int32 range after negating. auto emit = [&](int32_t r, double side_bound, long side, double weight) -> bool { + const double s = scan.row_scale.empty() ? 1.0 : scan.row_scale[r]; coef_t row_cmax = 1; for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { - const double a = coeffs[k]; + const double a = s * coeffs[k]; const long ai = side * std::lround(a); if (!is_integer(a, tol) || ai < std::numeric_limits::min() || ai > std::numeric_limits::max()) { @@ -379,7 +415,7 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, const coef_t abs_a = (coef_t)std::labs(ai); if (abs_a > row_cmax) row_cmax = abs_a; } - const long b = side * std::lround(side_bound); + const long b = side * std::lround(s * side_bound); if (!fj_bin_in_int32((double)b)) return false; pb.offsets.push_back((int32_t)pb.variables.size()); pb.bound.push_back((int32_t)b); @@ -1181,7 +1217,7 @@ bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, } auto run = [&](auto& engine) -> bool { - if (!fj_bin_narrow(climber, scan.n_split_constraints, engine.pb)) { + if (!fj_bin_narrow(climber, scan, engine.pb)) { CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s", climber.log_prefix.c_str(), fj_binary_reject_name(fj_binary_reject_t::narrow_check_failed)); From 2f7cf86e632db279a592d4d13fba68ab1becd7c0 Mon Sep 17 00:00:00 2001 From: yboucher Date: Wed, 19 Aug 2026 01:09:47 -0700 Subject: [PATCH 15/61] fix solve_CPUFJ link --- cpp/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0fe2bf064b..72d03b3d2d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,6 +49,7 @@ rapids_cmake_build_type(Release) option(CMAKE_CUDA_LINEINFO "Enable the -lineinfo option for nvcc useful for cuda-memcheck / profiler" ON) option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_LP_ONLY "Build only linear programming components, exclude routing and MIP-specific files" OFF) +option(BUILD_MIP_BENCHMARKS "Build MIP benchmarks" OFF) option(SKIP_C_PYTHON_ADAPTERS "Skip building C and Python adapter files (cython_solve.cu and cuopt_c.cpp)" OFF) option(SKIP_ROUTING_BUILD "Skip building routing components" OFF) option(SKIP_GRPC_BUILD "Skip building gRPC and protobuf components" OFF) @@ -800,6 +801,9 @@ target_link_libraries(cuopt_objs # - generate tests -------------------------------------------------------------------------------- if (BUILD_TESTS) include(CTest) +endif () + +if (BUILD_TESTS OR (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY)) add_library(cuopt_static STATIC $) target_link_libraries(cuopt_static PUBLIC @@ -842,6 +846,9 @@ if (BUILD_TESTS) if (TARGET KaMinPar) add_dependencies(cuopt_static KaMinPar) endif () +endif () + +if (BUILD_TESTS) add_subdirectory(tests) endif (BUILD_TESTS) @@ -1058,7 +1065,6 @@ if (NOT BUILD_LP_ONLY) endif () -option(BUILD_MIP_BENCHMARKS "Build MIP benchmarks" OFF) if (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY) add_executable(solve_MIP ../benchmarks/linear_programming/cuopt/run_mip.cpp) target_include_directories(solve_MIP @@ -1102,7 +1108,7 @@ if (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY) ) target_link_libraries(solve_CPUFJ PUBLIC - cuopt + cuopt_static OpenMP::OpenMP_CXX OpenMP::OpenMP_CUDA ) From 5805269d024f947ba6acf21df4dcc817e645b867 Mon Sep 17 00:00:00 2001 From: yboucher Date: Wed, 19 Aug 2026 01:14:10 -0700 Subject: [PATCH 16/61] latency tweaks --- .../cpu_optimization_problem.hpp | 7 + .../optimization_problem.hpp | 7 + cpp/src/mip_heuristics/early_heuristic.cuh | 64 +--- .../feasibility_jump/early_cpufj.cu | 14 +- .../feasibility_jump/early_cpufj.cuh | 7 + .../feasibility_jump/early_gpufj.cu | 41 ++- .../feasibility_jump/early_gpufj.cuh | 15 + .../mip_heuristics/feasibility_jump/fj_cpu.cu | 188 +++++++++- .../feasibility_jump/fj_cpu.cuh | 7 + .../presolve/semi_continuous.cu | 2 + cpp/src/mip_heuristics/solve.cu | 175 +++++----- cpp/src/pdlp/cpu_optimization_problem.cpp | 58 +++- cpp/src/pdlp/optimization_problem.cu | 47 ++- cpp/src/utilities/version_info.cpp | 323 +++++++++++------- 14 files changed, 668 insertions(+), 287 deletions(-) diff --git a/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp b/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp index 28aa91a82f..f0673a4f66 100644 --- a/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp +++ b/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp @@ -123,6 +123,12 @@ class cpu_optimization_problem_t : public optimization_problem_interface_t& get_variable_names() const override; const std::vector& get_row_names() const override; const std::vector& get_quadratic_objective_offsets() const override; @@ -208,6 +214,7 @@ class cpu_optimization_problem_t : public optimization_problem_interface_t std::string get_objective_name() const override; std::string get_problem_name() const override; problem_category_t get_problem_category() const override; + /** + * @brief Whether any variable type is SEMI_CONTINUOUS. + * + * Cached in set_variable_types(); used to skip SC reformulation host probes. + */ + bool has_semi_continuous_variables() const noexcept; const std::vector& get_variable_names() const override; const std::vector& get_row_names() const override; const std::vector& get_quadratic_objective_offsets() const override; @@ -391,6 +397,7 @@ class optimization_problem_t : public optimization_problem_interface_t rmm::cuda_stream_view stream_view_; problem_category_t problem_category_ = problem_category_t::LP; + bool has_semi_continuous_variables_{false}; bool maximize_{false}; i_t n_vars_{0}; i_t n_constraints_{0}; diff --git a/cpp/src/mip_heuristics/early_heuristic.cuh b/cpp/src/mip_heuristics/early_heuristic.cuh index 6654470732..84d5f86f7c 100644 --- a/cpp/src/mip_heuristics/early_heuristic.cuh +++ b/cpp/src/mip_heuristics/early_heuristic.cuh @@ -7,18 +7,13 @@ #pragma once -#include -#include - #include - -#include - -#include +#include #include #include #include +#include #include namespace cuopt::mathematical_optimization::mip { @@ -34,25 +29,13 @@ template class early_heuristic_t { public: early_heuristic_t(const optimization_problem_t& op_problem, - const typename mip_solver_settings_t::tolerances_t& tolerances, early_incumbent_callback_t incumbent_callback) - : incumbent_callback_(std::move(incumbent_callback)) + : objective_scaling_factor_(op_problem.get_sense() ? -op_problem.get_objective_scaling_factor() + : op_problem.get_objective_scaling_factor()), + objective_offset_(op_problem.get_sense() ? -op_problem.get_objective_offset() + : op_problem.get_objective_offset()), + incumbent_callback_(std::move(incumbent_callback)) { - RAFT_CUDA_TRY(cudaGetDevice(&device_id_)); - - // Build and preprocess on the original handle, then copy onto our own handle - // so the derived solver can run on a dedicated stream (prevents graph capture conflicts). - problem_t temp_problem(op_problem, tolerances, false); - temp_problem.preprocess_problem(); - temp_problem.handle_ptr->sync_stream(); - problem_ptr_ = std::make_unique>(temp_problem, &handle_); - - solution_ptr_ = std::make_unique>(*problem_ptr_); - thrust::fill(handle_.get_thrust_policy(), - solution_ptr_->assignment.begin(), - solution_ptr_->assignment.end(), - f_t{0}); - solution_ptr_->clamp_within_bounds(); } bool solution_found() const { return solution_found_; } @@ -60,12 +43,12 @@ class early_heuristic_t { // Return the best objective converted to user-space (sense-aware, offset-aware). f_t get_best_user_objective() const { - return problem_ptr_->get_user_obj_from_solver_obj(best_objective_); + return objective_scaling_factor_ * (best_objective_ + objective_offset_); } // Set the incumbent threshold. `obj` must be in THIS heuristic's solver-space - // (i.e. the space of problem_ptr_). Callers that hold a value from a different - // problem representation (e.g., the original pre-presolve problem) must convert - // it first, otherwise try_update_best will reject valid solutions. + // (i.e. the space of its input problem). Callers that hold a value from a + // different problem representation (e.g., the original pre-presolve problem) + // must convert it first, otherwise try_update_best will reject valid solutions. void set_best_objective(f_t obj) { best_objective_ = obj; } const std::vector& get_best_assignment() const { return best_assignment_; } @@ -73,40 +56,25 @@ class early_heuristic_t { ~early_heuristic_t() = default; // NOT thread-safe. solver_obj is in solver-space (always minimization). - // Uses a private CUDA stream to avoid racing with the FJ solver's stream. void try_update_best(f_t solver_obj, const std::vector& assignment) { if (solver_obj >= best_objective_) { return; } best_objective_ = solver_obj; - RAFT_CUDA_TRY(cudaSetDevice(device_id_)); - auto stream = handle_.get_stream(); - rmm::device_uvector d_assignment(assignment.size(), stream); - raft::copy(d_assignment.data(), assignment.data(), assignment.size(), stream); - problem_ptr_->post_process_assignment(d_assignment, true, stream); - auto user_assignment = cuopt::host_copy(d_assignment, stream); - - best_assignment_ = user_assignment; + best_assignment_ = ((Derived*)this)->to_user_assignment(assignment); solution_found_ = true; - f_t user_obj = problem_ptr_->get_user_obj_from_solver_obj(solver_obj); + f_t user_obj = get_best_user_objective(); // Log and callback are deferred to the shared incumbent_callback_ which enforces // global monotonicity across all early heuristic instances. if (incumbent_callback_) { - incumbent_callback_(solver_obj, user_obj, user_assignment, Derived::name()); + incumbent_callback_(solver_obj, user_obj, best_assignment_, Derived::name()); } } - int device_id_{0}; - - // handle_ must be declared before problem_ptr_/solution_ptr_ so it outlives them - // (C++ destroys members in reverse declaration order) - raft::handle_t handle_; - - std::unique_ptr> problem_ptr_; - std::unique_ptr> solution_ptr_; - bool solution_found_{false}; f_t best_objective_{std::numeric_limits::infinity()}; + f_t objective_scaling_factor_; + f_t objective_offset_; std::vector best_assignment_; early_incumbent_callback_t incumbent_callback_; diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu index ba14e657d5..e4db66ed1f 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu @@ -16,8 +16,9 @@ early_cpufj_t::early_cpufj_t( const optimization_problem_t& op_problem, const typename mip_solver_settings_t::tolerances_t& tolerances, early_incumbent_callback_t incumbent_callback) - : early_heuristic_t>( - op_problem, tolerances, std::move(incumbent_callback)) + : early_heuristic_t>(op_problem, std::move(incumbent_callback)), + problem_ptr_(&op_problem), + tolerances_(tolerances) { } @@ -36,7 +37,8 @@ void early_cpufj_t::start() this->preemption_flag_.store(false); this->start_time_ = std::chrono::steady_clock::now(); - fj_cpu_ = init_fj_cpu_standalone(*this->problem_ptr_, *this->solution_ptr_, preemption_flag_); + fj_cpu_ = + init_fj_cpu_from_optimization_problem(*this->problem_ptr_, tolerances_, preemption_flag_); fj_cpu_->log_prefix = "[Early CPUFJ] "; @@ -67,6 +69,12 @@ void early_cpufj_t::stop() fj_cpu_.reset(); } +template +std::vector early_cpufj_t::to_user_assignment(const std::vector& assignment) +{ + return assignment; +} + #if MIP_INSTANTIATE_FLOAT template class early_cpufj_t; #endif diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh index e2bb2c07b2..a6cf3057e0 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh @@ -12,6 +12,7 @@ #include #include +#include namespace cuopt::mathematical_optimization::mip { @@ -30,6 +31,12 @@ class early_cpufj_t : public early_heuristic_t void stop(); private: + friend class early_heuristic_t>; + + std::vector to_user_assignment(const std::vector& assignment); + + const optimization_problem_t* problem_ptr_; + typename mip_solver_settings_t::tolerances_t tolerances_; std::unique_ptr> fj_cpu_; std::atomic preemption_flag_{false}; }; diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu index 463f074f59..c9d787a236 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu @@ -10,9 +10,15 @@ #include #include #include +#include #include #include +#include +#include + +#include +#include #include @@ -22,11 +28,26 @@ template early_gpufj_t::early_gpufj_t(const optimization_problem_t& op_problem, const mip_solver_settings_t& settings, early_incumbent_callback_t incumbent_callback) - : early_heuristic_t>( - op_problem, settings.get_tolerances(), std::move(incumbent_callback)) + : early_heuristic_t>(op_problem, std::move(incumbent_callback)) { - context_ptr_ = std::make_unique>( - &this->handle_, this->problem_ptr_.get(), settings); + RAFT_CUDA_TRY(cudaGetDevice(&device_id_)); + + // Build and preprocess on the original handle, then copy onto our own handle + // so the derived solver can run on a dedicated stream (prevents graph capture conflicts). + problem_t temp_problem(op_problem, settings.get_tolerances(), false); + temp_problem.preprocess_problem(); + temp_problem.handle_ptr->sync_stream(); + problem_ptr_ = std::make_unique>(temp_problem, &handle_); + + solution_ptr_ = std::make_unique>(*problem_ptr_); + thrust::fill(handle_.get_thrust_policy(), + solution_ptr_->assignment.begin(), + solution_ptr_->assignment.end(), + f_t{0}); + solution_ptr_->clamp_within_bounds(); + + context_ptr_ = + std::make_unique>(&handle_, problem_ptr_.get(), settings); } template @@ -81,6 +102,18 @@ void early_gpufj_t::stop() fj_ptr_.reset(); } +template +std::vector early_gpufj_t::to_user_assignment(const std::vector& assignment) +{ + // Uses a private CUDA stream to avoid racing with the FJ solver's stream. + RAFT_CUDA_TRY(cudaSetDevice(device_id_)); + auto stream = handle_.get_stream(); + rmm::device_uvector d_assignment(assignment.size(), stream); + raft::copy(d_assignment.data(), assignment.data(), assignment.size(), stream); + problem_ptr_->post_process_assignment(d_assignment, true, stream); + return cuopt::host_copy(d_assignment, stream); +} + #if MIP_INSTANTIATE_FLOAT template class early_gpufj_t; #endif diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh index 99e8579d31..ed8d17206e 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh @@ -8,8 +8,11 @@ #pragma once #include +#include +#include #include +#include namespace cuopt::mathematical_optimization::mip { @@ -34,6 +37,18 @@ class early_gpufj_t : public early_heuristic_t void stop(); private: + friend class early_heuristic_t>; + + std::vector to_user_assignment(const std::vector& assignment); + + int device_id_{0}; + + // handle_ must be declared before problem_ptr_/solution_ptr_ so it outlives them + // (C++ destroys members in reverse declaration order) + raft::handle_t handle_; + + std::unique_ptr> problem_ptr_; + std::unique_ptr> solution_ptr_; std::unique_ptr> context_ptr_; std::unique_ptr> fj_ptr_; }; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index da009c13a6..4cce1df459 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -607,7 +607,6 @@ static inline std::pair compute_score(fj_cpu_climber_t& fj_cpu, auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[i].get(); auto cstr_idx = fj_cpu.h_reverse_constraints[i]; - fj_cpu.unique_cstrs_accessed_window.insert(cstr_idx); auto cstr_coeff = fj_cpu.h_reverse_coefficients[i]; f_t old_lhs = fj_cpu.h_lhs[cstr_idx]; @@ -2072,6 +2070,182 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w #endif } +template +static std::vector copy_to_host_async(const rmm::device_uvector& input, + rmm::cuda_stream_view stream) +{ + std::vector output(input.size()); + raft::copy(output.data(), input.data(), input.size(), stream); + return output; +} + +template +std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings) +{ + using f_t2 = typename type_2::type; + + raft::common::nvtx::range scope("init_fj_cpu_from_optimization_problem"); + + const i_t n_variables = problem.get_n_variables(); + const i_t n_constraints = problem.get_n_constraints(); + const i_t nnz = problem.get_nnz(); + auto stream = problem.get_handle_ptr()->get_stream(); + + auto coefficients = copy_to_host_async(problem.get_constraint_matrix_values(), stream); + auto variables = copy_to_host_async(problem.get_constraint_matrix_indices(), stream); + auto offsets = copy_to_host_async(problem.get_constraint_matrix_offsets(), stream); + auto objective_coefficients = copy_to_host_async(problem.get_objective_coefficients(), stream); + auto variable_lower_bounds = copy_to_host_async(problem.get_variable_lower_bounds(), stream); + auto variable_upper_bounds = copy_to_host_async(problem.get_variable_upper_bounds(), stream); + auto constraint_lower_bounds = copy_to_host_async(problem.get_constraint_lower_bounds(), stream); + auto constraint_upper_bounds = copy_to_host_async(problem.get_constraint_upper_bounds(), stream); + auto constraint_bounds = copy_to_host_async(problem.get_constraint_bounds(), stream); + auto row_types = copy_to_host_async(problem.get_row_types(), stream); + auto variable_types = copy_to_host_async(problem.get_variable_types(), stream); + problem.get_handle_ptr()->sync_stream(); + + cuopt_assert(coefficients.size() == (size_t)nnz, "coefficient size mismatch"); + cuopt_assert(variables.size() == (size_t)nnz, "variable index size mismatch"); + cuopt_assert(offsets.size() == (size_t)(n_constraints + 1), + "constraint offset size mismatch"); + cuopt_assert(!offsets.empty() && offsets.front() == 0, "invalid first constraint offset"); + cuopt_assert(offsets.back() == nnz, "invalid final constraint offset"); + cuopt_assert(std::is_sorted(offsets.begin(), offsets.end()), "unsorted constraint offsets"); + cuopt_assert( + std::all_of(variables.begin(), + variables.end(), + [n_variables](i_t variable) { return variable >= 0 && variable < n_variables; }), + "variable index out of range"); + cuopt_assert(objective_coefficients.size() == (size_t)n_variables, + "objective size mismatch"); + cuopt_assert(variable_lower_bounds.empty() || + variable_lower_bounds.size() == (size_t)n_variables, + "variable lower bound size mismatch"); + cuopt_assert(variable_upper_bounds.empty() || + variable_upper_bounds.size() == (size_t)n_variables, + "variable upper bound size mismatch"); + + if (constraint_lower_bounds.empty() && constraint_upper_bounds.empty()) { + cuopt_assert(row_types.size() == (size_t)n_constraints, "row type size mismatch"); + cuopt_assert(constraint_bounds.size() == (size_t)n_constraints, + "constraint bound size mismatch"); + constraint_lower_bounds.resize(n_constraints); + constraint_upper_bounds.resize(n_constraints); + for (i_t row = 0; row < n_constraints; ++row) { + const f_t bound = constraint_bounds[row]; + if (row_types[row] == 'E') { + constraint_lower_bounds[row] = bound; + constraint_upper_bounds[row] = bound; + } else if (row_types[row] == 'G') { + constraint_lower_bounds[row] = bound; + constraint_upper_bounds[row] = std::numeric_limits::infinity(); + } else { + cuopt_assert(row_types[row] == 'L', "invalid row type"); + constraint_lower_bounds[row] = -std::numeric_limits::infinity(); + constraint_upper_bounds[row] = bound; + } + } + } else { + cuopt_assert(constraint_lower_bounds.size() == (size_t)n_constraints, + "constraint lower bound size mismatch"); + cuopt_assert(constraint_upper_bounds.size() == (size_t)n_constraints, + "constraint upper bound size mismatch"); + } + + if (variable_lower_bounds.empty()) { variable_lower_bounds.assign(n_variables, f_t{0}); } + if (variable_upper_bounds.empty()) { + variable_upper_bounds.assign(n_variables, std::numeric_limits::infinity()); + } + if (variable_types.empty()) { variable_types.assign(n_variables, var_t::CONTINUOUS); } + cuopt_assert(variable_types.size() == (size_t)n_variables, + "variable type size mismatch"); + + if (problem.get_sense()) { + std::transform(objective_coefficients.begin(), + objective_coefficients.end(), + objective_coefficients.begin(), + std::negate{}); + } + + std::vector variable_bounds(n_variables); + std::vector is_binary_variable(n_variables, 0); + std::vector binary_indices; + binary_indices.reserve(n_variables); + i_t n_integer_vars = 0; + for (i_t variable = 0; variable < n_variables; ++variable) { + f_t lower = variable_lower_bounds[variable]; + f_t upper = variable_upper_bounds[variable]; + const bool is_integer = variable_types[variable] == var_t::INTEGER; + if (is_integer) { + lower = std::ceil(lower); + upper = std::floor(upper); + ++n_integer_vars; + } + cuopt_assert(lower <= upper, "crossing variable bounds"); + variable_bounds[variable] = f_t2{lower, upper}; + if (is_integer && lower == f_t{0} && upper == f_t{1}) { + is_binary_variable[variable] = 1; + binary_indices.push_back(variable); + } + } + + csr_matrix_t csr(n_constraints, n_variables, nnz); + csr.x = coefficients; + csr.j = variables; + csr.row_start = offsets; + csc_matrix_t csc(n_constraints, n_variables, nnz); + csr.to_compressed_col(csc); + + std::vector assignment(n_variables, f_t{0}); + for (i_t variable = 0; variable < n_variables; ++variable) { + f_t value = std::clamp( + f_t{0}, get_lower(variable_bounds[variable]), get_upper(variable_bounds[variable])); + if (variable_types[variable] == var_t::INTEGER) { value = std::round(value); } + assignment[variable] = value; + } + + auto fj_cpu = std::make_unique>(preemption_flag); + fj_cpu->view = typename fj_t::climber_data_t::view_t{}; + fj_cpu->pb_ptr = nullptr; + fj_cpu->settings = settings; + + fj_cpu->h_reverse_coefficients = std::move(csc.x); + fj_cpu->h_reverse_constraints = std::move(csc.i); + fj_cpu->h_reverse_offsets = std::move(csc.col_start); + fj_cpu->h_coefficients = std::move(coefficients); + fj_cpu->h_offsets = std::move(offsets); + fj_cpu->h_variables = std::move(variables); + fj_cpu->h_obj_coeffs = std::move(objective_coefficients); + fj_cpu->h_var_bounds = std::move(variable_bounds); + fj_cpu->h_cstr_lb = std::move(constraint_lower_bounds); + fj_cpu->h_cstr_ub = std::move(constraint_upper_bounds); + fj_cpu->h_var_types = std::move(variable_types); + fj_cpu->h_is_binary_variable = std::move(is_binary_variable); + fj_cpu->h_binary_indices = std::move(binary_indices); + fj_cpu->h_cstr_left_weights.resize(n_constraints, f_t{1}); + fj_cpu->h_cstr_right_weights.resize(n_constraints, f_t{1}); + fj_cpu->max_weight = f_t{1}; + fj_cpu->h_objective_weight = f_t{0}; + fj_cpu->h_assignment = assignment; + fj_cpu->h_best_assignment = std::move(assignment); + fj_cpu->h_lhs.resize(n_constraints); + fj_cpu->h_lhs_sumcomp.resize(n_constraints, f_t{0}); + fj_cpu->h_tabu_nodec_until.resize(n_variables, 0); + fj_cpu->h_tabu_noinc_until.resize(n_variables, 0); + fj_cpu->h_tabu_lastdec.resize(n_variables, 0); + fj_cpu->h_tabu_lastinc.resize(n_variables, 0); + fj_cpu->iterations = 0; + fj_cpu->settings.seed = cuopt::seed_generator::get_seed(); + + finalize_fj_cpu_host_initialization( + *fj_cpu, n_variables, n_constraints, n_integer_vars, nnz, tolerances); + return fj_cpu; +} + template std::unique_ptr> init_fj_cpu_standalone( problem_t& problem, @@ -2170,6 +2344,11 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings); template void finalize_fj_cpu_host_initialization( fj_cpu_climber_t& fj_cpu, int n_variables, @@ -2190,6 +2369,11 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings); template void finalize_fj_cpu_host_initialization( fj_cpu_climber_t& fj_cpu, int n_variables, diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index bb528f54ea..89db669dd0 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -232,4 +232,11 @@ std::unique_ptr> init_fj_cpu_standalone( std::atomic& preemption_flag, fj_settings_t settings = fj_settings_t{}); +template +std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings = fj_settings_t{}); + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/presolve/semi_continuous.cu b/cpp/src/mip_heuristics/presolve/semi_continuous.cu index 33b7efff0e..51d2552746 100644 --- a/cpp/src/mip_heuristics/presolve/semi_continuous.cu +++ b/cpp/src/mip_heuristics/presolve/semi_continuous.cu @@ -115,6 +115,8 @@ bool reformulate_semi_continuous(optimization_problem_t& op_problem, std::vector* used_fallback_big_m, std::vector* semi_continuous_binary_to_original_indices) { + if (!op_problem.has_semi_continuous_variables()) { return false; } + // 1. Identify semi-continuous variables auto var_types = op_problem.get_variable_types_host(); auto var_lb = op_problem.get_variable_lower_bounds_host(); diff --git a/cpp/src/mip_heuristics/solve.cu b/cpp/src/mip_heuristics/solve.cu index 162a5ba291..0fc4230192 100644 --- a/cpp/src/mip_heuristics/solve.cu +++ b/cpp/src/mip_heuristics/solve.cu @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -254,7 +255,6 @@ mip_solution_t run_mip_solver( settings.determinism_mode != CUOPT_MODE_DETERMINISTIC && problem.original_problem_ptr->get_n_integers() > 0; if (run_early_cpufj) { - auto early_fj_start = std::chrono::steady_clock::now(); auto* presolver_ptr = problem.presolve_data.papilo_presolve_ptr; auto mip_callbacks = settings.get_mip_callbacks(); f_t no_bound = problem.presolve_data.objective_scaling_factor >= 0 ? (f_t)-1e20 : (f_t)1e20; @@ -269,22 +269,19 @@ mip_solution_t run_mip_solver( mip_solver_settings_accessor::get_semi_continuous_original_num_variables( settings), ctx_ptr = &solver.context, - early_fj_start](f_t solver_obj, - f_t user_obj, - const std::vector& assignment, - const char* heuristic_name) { + &timer](f_t solver_obj, + f_t user_obj, + const std::vector& assignment, + const char* heuristic_name) { std::vector user_assignment; presolver_ptr->uncrush_primal_solution(assignment, user_assignment); ctx_ptr->initial_incumbent_assignment = user_assignment; ctx_ptr->initial_upper_bound = user_obj; - double elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - early_fj_start) - .count(); CUOPT_LOG_INFO( - "New solution from early primal heuristics (%s). Objective %+.6e. Time %.2f", + "New solution from early primal heuristics (%s). Objective %+.6e. Time %.3f", heuristic_name, user_obj, - elapsed); + timer.elapsed_time()); invoke_solution_callbacks(mip_callbacks, has_semi_continuous_callback_translation, semi_continuous_original_num_variables, @@ -376,16 +373,6 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p raft::common::nvtx::range fun_scope("Running solver"); auto timer = timer_t(time_limit); - problem_checking_t::check_problem_representation(op_problem); - problem_checking_t::check_initial_solution_representation(op_problem, settings); - - CUOPT_LOG_INFO( - "Solving a problem with %d constraints, %d variables (%d integers), and %d nonzeros", - op_problem.get_n_constraints(), - op_problem.get_n_variables(), - op_problem.get_n_integers(), - op_problem.get_nnz()); - // Reformulate semi-continuous variables (x = 0 OR L <= x <= U) before Papilo presolve. // Uses deterministic CPU bounds strengthening to derive tight upper bounds for SC vars with // infinite UB. @@ -407,15 +394,6 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p settings, n_orig_before_sc, semi_continuous_binary_to_original_indices); } - op_problem.print_scaling_information(); - - // Check for crossing bounds. Return infeasible if there are any - if (problem_checking_t::has_crossing_bounds(op_problem)) { - return mip_solution_t(mip_termination_status_t::Infeasible, - solver_stats_t{}, - op_problem.get_handle_ptr()->get_stream()); - } - for (auto callback : settings.get_mip_callbacks()) { auto callback_num_variables = op_problem.get_n_variables(); if (mip_solver_settings_accessor::has_semi_continuous_callback_translation( @@ -444,16 +422,6 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p } #endif - if (settings.mip_scaling != CUOPT_MIP_SCALING_OFF) { - mip::mip_scaling_strategy_t scaling(op_problem); - scaling.scale_problem(settings.mip_scaling != CUOPT_MIP_SCALING_NO_OBJECTIVE); - } - double presolve_time = 0.0; - std::unique_ptr> presolver; - std::optional> presolve_result_opt; - mip::problem_t problem( - op_problem, settings.get_tolerances(), settings.determinism_mode == CUOPT_MODE_DETERMINISTIC); - auto run_presolve = settings.presolver != presolver_t::None; bool has_set_solution_callback = false; for (auto callback : settings.get_mip_callbacks()) { @@ -481,8 +449,8 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p std::vector early_incumbent_pool; // Track best incumbent found during presolve (shared across CPU and GPU FJ). - // early_best_objective is in the original problem's solver-space (always minimization), - // used for fast comparison in the callback. + // The CPU and GPU heuristics can use differently scaled solver spaces, so compare their + // objectives in a common minimization-oriented user space. // early_best_user_obj is the corresponding user-space objective, // passed to run_mip for correct cross-space conversion. // We attempt to crush early-heuristics solutions into the presolved space. @@ -491,7 +459,7 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p // but is dropped due to these dual reductions, and we lose a good solution. // This is why we still keep the solution around in original-space // and later extract it at the end of the solve. - std::atomic early_best_objective{std::numeric_limits::infinity()}; + std::atomic early_best_user_score{std::numeric_limits::infinity()}; f_t early_best_user_obj{std::numeric_limits::infinity()}; std::vector early_best_user_assignment; std::mutex early_callback_mutex; @@ -500,57 +468,92 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p std::unique_ptr> early_gpufj; bool run_early_fj = run_presolve && settings.determinism_mode != CUOPT_MODE_DETERMINISTIC && - op_problem.get_n_integers() > 0 && op_problem.get_n_constraints() > 0; - f_t no_bound = problem.presolve_data.objective_scaling_factor >= 0 ? (f_t)-1e20 : (f_t)1e20; - if (run_early_fj) { - auto early_fj_start = std::chrono::steady_clock::now(); - auto early_fj_callback = - [&early_best_objective, - &early_best_user_obj, - &early_best_user_assignment, - &early_incumbent_pool, - &early_callback_mutex, - early_fj_start, - mip_callbacks = settings.get_mip_callbacks(), - has_semi_continuous_callback_translation = - mip_solver_settings_accessor::has_semi_continuous_callback_translation( - settings), - semi_continuous_original_num_variables = - mip_solver_settings_accessor::get_semi_continuous_original_num_variables( - settings), - no_bound](f_t solver_obj, - f_t user_obj, - const std::vector& assignment, - const char* heuristic_name) { - std::lock_guard lock(early_callback_mutex); - if (solver_obj >= early_best_objective.load()) { return; } - early_best_objective.store(solver_obj); - early_best_user_obj = user_obj; - early_best_user_assignment = assignment; - early_incumbent_pool.push_back({user_obj, assignment}); - double elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - early_fj_start) - .count(); - CUOPT_LOG_INFO( - "New solution from early primal heuristics (%s). Objective %+.6e. Time %.2f", - heuristic_name, - user_obj, - elapsed); - auto user_assignment = assignment; - invoke_solution_callbacks(mip_callbacks, - has_semi_continuous_callback_translation, - semi_continuous_original_num_variables, - user_obj, - user_assignment, - no_bound); - }; + op_problem.get_problem_category() != problem_category_t::LP && + op_problem.get_n_constraints() > 0; + const f_t objective_sense = + op_problem.get_objective_scaling_factor() >= f_t{0} ? f_t{1} : f_t{-1}; + f_t no_bound = objective_sense > f_t{0} ? (f_t)-1e20 : (f_t)1e20; + auto early_fj_callback = + [&early_best_user_score, + &early_best_user_obj, + &early_best_user_assignment, + &early_incumbent_pool, + &early_callback_mutex, + &timer, + objective_sense, + mip_callbacks = settings.get_mip_callbacks(), + has_semi_continuous_callback_translation = + mip_solver_settings_accessor::has_semi_continuous_callback_translation(settings), + semi_continuous_original_num_variables = + mip_solver_settings_accessor::get_semi_continuous_original_num_variables( + settings), + no_bound]( + f_t, f_t user_obj, const std::vector& assignment, const char* heuristic_name) { + std::lock_guard lock(early_callback_mutex); + const f_t objective = objective_sense * user_obj; + if (objective >= early_best_user_score.load()) { return; } + early_best_user_score.store(objective); + early_best_user_obj = user_obj; + early_best_user_assignment = assignment; + early_incumbent_pool.push_back({user_obj, assignment}); + CUOPT_LOG_INFO("New solution from early primal heuristics (%s). Objective %+.6e. Time %.3f", + heuristic_name, + user_obj, + timer.elapsed_time()); + auto user_assignment = assignment; + invoke_solution_callbacks(mip_callbacks, + has_semi_continuous_callback_translation, + semi_continuous_original_num_variables, + user_obj, + user_assignment, + no_bound); + }; + if (run_early_fj) { // Start early CPUFJ on original problem (will restart on presolved problem after Papilo) early_cpufj = std::make_unique>( op_problem, settings.get_tolerances(), early_fj_callback); early_cpufj->start(); CUOPT_LOG_DEBUG("Started early CPUFJ on original problem"); + } + + auto early_cpufj_guard = cuopt::scope_guard([&]() { + if (early_cpufj) { + early_cpufj->stop(); + early_cpufj.reset(); + } + }); + + problem_checking_t::check_problem_representation(op_problem); + problem_checking_t::check_initial_solution_representation(op_problem, settings); + + CUOPT_LOG_INFO( + "Solving a problem with %d constraints, %d variables (%d integers), and %d nonzeros", + op_problem.get_n_constraints(), + op_problem.get_n_variables(), + op_problem.get_n_integers(), + op_problem.get_nnz()); + op_problem.print_scaling_information(); + + // Check for crossing bounds. Return infeasible if there are any + if (problem_checking_t::has_crossing_bounds(op_problem)) { + return mip_solution_t(mip_termination_status_t::Infeasible, + solver_stats_t{}, + op_problem.get_handle_ptr()->get_stream()); + } + + if (settings.mip_scaling != CUOPT_MIP_SCALING_OFF) { + mip::mip_scaling_strategy_t scaling(op_problem); + scaling.scale_problem(settings.mip_scaling != CUOPT_MIP_SCALING_NO_OBJECTIVE); + } + double presolve_time = 0.0; + std::unique_ptr> presolver; + std::optional> presolve_result_opt; + mip::problem_t problem( + op_problem, settings.get_tolerances(), settings.determinism_mode == CUOPT_MODE_DETERMINISTIC); + + if (run_early_fj) { // Start early GPU FJ (uses GPU while CPU is busy with Papilo) early_gpufj = std::make_unique>(op_problem, settings, early_fj_callback); diff --git a/cpp/src/pdlp/cpu_optimization_problem.cpp b/cpp/src/pdlp/cpu_optimization_problem.cpp index 4b970eb6ec..93e86b7da6 100644 --- a/cpp/src/pdlp/cpu_optimization_problem.cpp +++ b/cpp/src/pdlp/cpu_optimization_problem.cpp @@ -29,20 +29,36 @@ namespace cuopt::mathematical_optimization { namespace { -// Classify a problem as LP / MIP / IP from its (enum) variable types. Single source of truth -// shared by set_variable_types() and adopt_from_mps_data_model() so the detection rule lives in -// one place. Empty types (no variables declared) classify as LP, matching the populate path where -// set_variable_types() is skipped and the category keeps its LP default. -problem_category_t problem_category_from_variable_types(const std::vector& variable_types) -{ - if (variable_types.empty()) { return problem_category_t::LP; } - const std::size_t n_discrete = static_cast( - std::count_if(variable_types.begin(), variable_types.end(), [](var_t v) { - return v == var_t::INTEGER || v == var_t::SEMI_CONTINUOUS; - })); - if (n_discrete == variable_types.size()) { return problem_category_t::IP; } - if (n_discrete > 0) { return problem_category_t::MIP; } - return problem_category_t::LP; +// Classify a problem as LP / MIP / IP from its (enum) variable types, and whether any +// SEMI_CONTINUOUS vars are present. Single source of truth shared by set_variable_types() and +// adopt_from_mps_data_model() so the detection rule lives in one place. Empty types (no variables +// declared) classify as LP with no SC, matching the populate path where set_variable_types() is +// skipped and the category keeps its LP default. +struct variable_type_summary_t { + problem_category_t category; + bool has_semi_continuous; +}; + +variable_type_summary_t summarize_variable_types(const std::vector& variable_types) +{ + if (variable_types.empty()) { + return {problem_category_t::LP, false}; + } + size_t n_discrete = 0; + bool has_semi_continuous = false; + for (var_t v : variable_types) { + if (v == var_t::SEMI_CONTINUOUS) { + has_semi_continuous = true; + ++n_discrete; + } else if (v == var_t::INTEGER) { + ++n_discrete; + } + } + if (n_discrete == variable_types.size()) { + return {problem_category_t::IP, has_semi_continuous}; + } + if (n_discrete > 0) { return {problem_category_t::MIP, has_semi_continuous}; } + return {problem_category_t::LP, false}; } } // namespace @@ -232,7 +248,9 @@ void cpu_optimization_problem_t::set_variable_types(const var_t* varia variable_types_.resize(size); std::copy(variable_types, variable_types + size, variable_types_.begin()); - problem_category_ = problem_category_from_variable_types(variable_types_); + const auto summary = summarize_variable_types(variable_types_); + problem_category_ = summary.category; + has_semi_continuous_variables_ = summary.has_semi_continuous; } template @@ -513,6 +531,12 @@ problem_category_t cpu_optimization_problem_t::get_problem_category() return problem_category_; } +template +bool cpu_optimization_problem_t::has_semi_continuous_variables() const noexcept +{ + return has_semi_continuous_variables_; +} + template const std::vector& cpu_optimization_problem_t::get_variable_names() const { @@ -1171,7 +1195,9 @@ void cpu_optimization_problem_t::adopt_from_mps_data_model( for (size_t i = 0; i < model.var_types_.size(); ++i) { variable_types_[i] = char_to_var_type(model.var_types_[i]); } - problem_category_ = problem_category_from_variable_types(variable_types_); + const auto summary = summarize_variable_types(variable_types_); + problem_category_ = summary.category; + has_semi_continuous_variables_ = summary.has_semi_continuous; if (model.has_quadratic_constraints()) { move_quadratic_constraints_from_model(*this, model.quadratic_constraints_); diff --git a/cpp/src/pdlp/optimization_problem.cu b/cpp/src/pdlp/optimization_problem.cu index 95457e2556..18222ed012 100644 --- a/cpp/src/pdlp/optimization_problem.cu +++ b/cpp/src/pdlp/optimization_problem.cu @@ -54,6 +54,8 @@ namespace cuopt::mathematical_optimization { +constexpr size_t host_variable_type_summary_limit = 50'000; + template optimization_problem_t::optimization_problem_t(raft::handle_t const* handle_ptr) : handle_ptr_(handle_ptr), @@ -101,6 +103,7 @@ optimization_problem_t::optimization_problem_t( objective_name_{other.get_objective_name()}, problem_name_{other.get_problem_name()}, problem_category_{other.get_problem_category()}, + has_semi_continuous_variables_{other.has_semi_continuous_variables()}, var_names_{other.get_variable_names()}, row_names_{other.get_row_names()}, quadratic_constraints_{other.get_quadratic_constraints()} @@ -285,14 +288,40 @@ void optimization_problem_t::set_variable_types(const var_t* variable_ variable_types_.resize(size, stream_view_); raft::copy(variable_types_.data(), variable_types, size, stream_view_); - // Auto-detect problem category based on variable types. + // Auto-detect problem category and cache presence of SEMI_CONTINUOUS vars. // SEMI_CONTINUOUS vars will be reformulated into binary + continuous before solving, // so a problem with only SC vars is treated as MIP. - i_t n_discrete = thrust::count_if( - handle_ptr_->get_thrust_policy(), - variable_types_.begin(), - variable_types_.end(), - [] __device__(auto val) { return val == var_t::INTEGER || val == var_t::SEMI_CONTINUOUS; }); + // Prefer host-side for small instances to reduce latency between launch and first-feasible. + i_t n_discrete = 0; + bool has_semi_continuous_variables = false; + if ((size_t)size < host_variable_type_summary_limit) { + for (i_t i = 0; i < size; ++i) { + const var_t val = variable_types[i]; + if (val == var_t::SEMI_CONTINUOUS) { + has_semi_continuous_variables = true; + ++n_discrete; + } else if (val == var_t::INTEGER) { + ++n_discrete; + } + } + } else { + auto is_discrete = [] __host__ __device__(var_t val) { + return val == var_t::INTEGER || val == var_t::SEMI_CONTINUOUS; + }; + auto is_semi_continuous = [] __host__ __device__(var_t val) { + return val == var_t::SEMI_CONTINUOUS; + }; + n_discrete = thrust::count_if(handle_ptr_->get_thrust_policy(), + variable_types_.begin(), + variable_types_.end(), + is_discrete); + has_semi_continuous_variables = + thrust::count_if(handle_ptr_->get_thrust_policy(), + variable_types_.begin(), + variable_types_.end(), + is_semi_continuous) > 0; + } + has_semi_continuous_variables_ = has_semi_continuous_variables; if (n_discrete == size) { problem_category_ = problem_category_t::IP; } else if (n_discrete > 0) { @@ -580,6 +609,12 @@ problem_category_t optimization_problem_t::get_problem_category() cons return problem_category_; } +template +bool optimization_problem_t::has_semi_continuous_variables() const noexcept +{ + return has_semi_continuous_variables_; +} + template const std::vector& optimization_problem_t::get_variable_names() const { diff --git a/cpp/src/utilities/version_info.cpp b/cpp/src/utilities/version_info.cpp index 67ec9e6794..3fe1074f87 100644 --- a/cpp/src/utilities/version_info.cpp +++ b/cpp/src/utilities/version_info.cpp @@ -15,135 +15,208 @@ #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include + +#include +#include +#include +#include namespace cuopt { -static int get_physical_cores() +// Reads up to buf_size-1 bytes, NUL-terminates, strips trailing whitespace/NULs. +// Returns bytes kept (excluding the terminator), or -1 on failure. +static ssize_t read_file_buf(const char* path, char* buf, size_t buf_size) +{ + if (buf_size == 0) return -1; + const int fd = open(path, O_RDONLY); + if (fd < 0) return -1; + const ssize_t n = read(fd, buf, buf_size - 1); + close(fd); + if (n < 0) return -1; + buf[n] = '\0'; + + // Device-tree properties are often NUL-terminated without a trailing newline. + size_t len = 0; + while (len < (size_t)n && buf[len] != '\0') { + ++len; + } + buf[len] = '\0'; + while (len > 0 && + (buf[len - 1] == '\n' || buf[len - 1] == '\r' || buf[len - 1] == ' ' || + buf[len - 1] == '\t')) { + buf[--len] = '\0'; + } + return (ssize_t)len; +} + +// Parses a kernel CPU list ("0-3,8,10-11") into cpus[0..max_cpus). Returns count written. +static int parse_cpu_list(const char* list, int* cpus, int max_cpus) { - std::ifstream cpuinfo("/proc/cpuinfo"); - if (!cpuinfo.is_open()) return 0; - - std::string line; - int physical_id = -1, core_id = -1; - std::set> cores; - - while (std::getline(cpuinfo, line)) { - if (line.find("physical id") != std::string::npos) { - physical_id = std::stoi(line.substr(line.find(":") + 1)); - } else if (line.find("core id") != std::string::npos) { - core_id = std::stoi(line.substr(line.find(":") + 1)); + int count = 0; + const char* p = list; + while (*p && count < max_cpus) { + while (*p == ',' || *p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + ++p; } + if (*p == '\0') break; + + char* end = nullptr; + const long lo = std::strtol(p, &end, 10); + if (end == p) break; + p = end; - if (physical_id != -1 && core_id != -1) { - cores.insert({physical_id, core_id}); - physical_id = -1; - core_id = -1; + if (*p == '-') { + ++p; + const long hi = std::strtol(p, &end, 10); + if (end == p) break; + p = end; + for (long cpu = lo; cpu <= hi && count < max_cpus; ++cpu) { + cpus[count++] = (int)cpu; + } + } else { + cpus[count++] = (int)lo; } } + return count; +} - if (cores.empty()) { - cpuinfo.clear(); - cpuinfo.seekg(0); - while (std::getline(cpuinfo, line)) { - if (line.find("cpu cores") != std::string::npos) { - return std::stoi(line.substr(line.find(":") + 1)); +static void mark_cpus_from_list(const char* list, char visited[CPU_SETSIZE]) +{ + const char* p = list; + while (*p) { + while (*p == ',' || *p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + ++p; + } + if (*p == '\0') break; + + char* end = nullptr; + const long lo = std::strtol(p, &end, 10); + if (end == p) break; + p = end; + + if (*p == '-') { + ++p; + const long hi = std::strtol(p, &end, 10); + if (end == p) break; + p = end; + for (long cpu = lo; cpu <= hi; ++cpu) { + if (cpu >= 0 && cpu < CPU_SETSIZE) { visited[cpu] = 1; } } + } else if (lo >= 0 && lo < CPU_SETSIZE) { + visited[lo] = 1; } - return 1; } - return cores.size(); } -static std::string get_cpu_model_from_proc() +// CPUs this process may run on (respects Slurm/cgroup cpusets, taskset, etc.). +static int get_allowed_cpus(int* cpus, int max_cpus) { - std::ifstream cpuinfo("/proc/cpuinfo"); - if (!cpuinfo.is_open()) return ""; - - std::string line; - while (std::getline(cpuinfo, line)) { - std::size_t pos = line.find("model name"); - if (pos == std::string::npos) pos = line.find("Processor"); - if (pos != std::string::npos) { - std::size_t colon = line.find(':', pos); - if (colon != std::string::npos) return line.substr(colon + 2); // Skip ": " + cpu_set_t set; + CPU_ZERO(&set); + int count = 0; + if (sched_getaffinity(0, sizeof(set), &set) == 0) { + for (int cpu = 0; cpu < CPU_SETSIZE && count < max_cpus; ++cpu) { + if (CPU_ISSET(cpu, &set)) { cpus[count++] = cpu; } } } - return ""; + if (count > 0) return count; + + char buf[256]; + if (read_file_buf("/sys/devices/system/cpu/online", buf, sizeof(buf)) < 0) return 0; + return parse_cpu_list(buf, cpus, max_cpus); } -// From https://gcc.gnu.org/onlinedocs/gcc/x86-Built-in-Functions.html -// Also supported by clang -static std::string get_cpu_model_builtin() +static int get_physical_cores(const int* allowed_cpus, int allowed_count) { -#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) - __builtin_cpu_init(); - return __builtin_cpu_is("amd") ? "AMD CPU" - : __builtin_cpu_is("intel") ? "Intel CPU" - : __builtin_cpu_is("atom") ? "Intel Atom CPU" - : __builtin_cpu_is("slm") ? "Intel Silvermont CPU" - : __builtin_cpu_is("core2") ? "Intel Core 2 CPU" - : __builtin_cpu_is("corei7") ? "Intel Core i7 CPU" - : __builtin_cpu_is("nehalem") ? "Intel Core i7 Nehalem CPU" - : __builtin_cpu_is("westmere") ? "Intel Core i7 Westmere CPU" - : __builtin_cpu_is("sandybridge") ? "Intel Core i7 Sandy Bridge CPU" - : __builtin_cpu_is("ivybridge") ? "Intel Core i7 Ivy Bridge CPU" - : __builtin_cpu_is("haswell") ? "Intel Core i7 Haswell CPU" - : __builtin_cpu_is("broadwell") ? "Intel Core i7 Broadwell CPU" - : __builtin_cpu_is("skylake") ? "Intel Core i7 Skylake CPU" - : __builtin_cpu_is("skylake-avx512") ? "Intel Core i7 Skylake AVX512 CPU" - : __builtin_cpu_is("cannonlake") ? "Intel Core i7 Cannon Lake CPU" - : __builtin_cpu_is("icelake-client") ? "Intel Core i7 Ice Lake Client CPU" - : __builtin_cpu_is("icelake-server") ? "Intel Core i7 Ice Lake Server CPU" - : __builtin_cpu_is("cascadelake") ? "Intel Core i7 Cascadelake CPU" - : __builtin_cpu_is("tigerlake") ? "Intel Core i7 Tigerlake CPU" - : __builtin_cpu_is("cooperlake") ? "Intel Core i7 Cooperlake CPU" - : __builtin_cpu_is("sapphirerapids") ? "Intel Core i7 sapphirerapids CPU" - : __builtin_cpu_is("alderlake") ? "Intel Core i7 Alderlake CPU" - : __builtin_cpu_is("rocketlake") ? "Intel Core i7 Rocketlake CPU" - : __builtin_cpu_is("graniterapids") ? "Intel Core i7 graniterapids CPU" - : __builtin_cpu_is("graniterapids-d") ? "Intel Core i7 graniterapids D CPU" - : __builtin_cpu_is("bonnell") ? "Intel Atom Bonnell CPU" - : __builtin_cpu_is("silvermont") ? "Intel Atom Silvermont CPU" - : __builtin_cpu_is("goldmont") ? "Intel Atom Goldmont CPU" - : __builtin_cpu_is("goldmont-plus") ? "Intel Atom Goldmont Plus CPU" - : __builtin_cpu_is("tremont") ? "Intel Atom Tremont CPU" - : __builtin_cpu_is("sierraforest") ? "Intel Atom Sierra Forest CPU" - : __builtin_cpu_is("grandridge") ? "Intel Atom Grand Ridge CPU" - : __builtin_cpu_is("amdfam10h") ? "AMD Family 10h CPU" - : __builtin_cpu_is("barcelona") ? "AMD Family 10h Barcelona CPU" - : __builtin_cpu_is("shanghai") ? "AMD Family 10h Shanghai CPU" - : __builtin_cpu_is("istanbul") ? "AMD Family 10h Istanbul CPU" - : __builtin_cpu_is("btver1") ? "AMD Family 14h CPU" - : __builtin_cpu_is("amdfam15h") ? "AMD Family 15h CPU" - : __builtin_cpu_is("bdver1") ? "AMD Family 15h Bulldozer version 1" - : __builtin_cpu_is("bdver2") ? "AMD Family 15h Bulldozer version 2" - : __builtin_cpu_is("bdver3") ? "AMD Family 15h Bulldozer version 3" - : __builtin_cpu_is("bdver4") ? "AMD Family 15h Bulldozer version 4" - : __builtin_cpu_is("btver2") ? "AMD Family 16h CPU" - : __builtin_cpu_is("amdfam17h") ? "AMD Family 17h CPU" - : __builtin_cpu_is("znver1") ? "AMD Family 17h Zen version 1" - : __builtin_cpu_is("znver2") ? "AMD Family 17h Zen version 2" - : __builtin_cpu_is("amdfam19h") ? "AMD Family 19h CPU" - : "Unknown"; -#else - return "Unknown"; -#endif + if (allowed_count <= 0) return 0; + + char visited[CPU_SETSIZE]; + std::memset(visited, 0, sizeof(visited)); + int cores = 0; + + for (int i = 0; i < allowed_count; ++i) { + const int cpu = allowed_cpus[i]; + if (cpu < 0 || cpu >= CPU_SETSIZE || visited[cpu]) continue; + + char path[128]; + char buf[256]; + snprintf(path, + sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/core_cpus_list", + cpu); + ssize_t n = read_file_buf(path, buf, sizeof(buf)); + if (n < 0) { + snprintf(path, + sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/thread_siblings_list", + cpu); + n = read_file_buf(path, buf, sizeof(buf)); + } + + if (n >= 0) { + mark_cpus_from_list(buf, visited); + } + visited[cpu] = 1; + ++cores; + } + + return cores > 0 ? cores : allowed_count; +} + +static bool copy_stripped(char* dst, size_t dst_size, const char* src) +{ + if (dst_size == 0) return false; + size_t len = std::strlen(src); + while (len > 0 && (src[len - 1] == '\n' || src[len - 1] == '\r' || src[len - 1] == ' ')) { + --len; + } + if (len >= dst_size) len = dst_size - 1; + std::memcpy(dst, src, len); + dst[len] = '\0'; + return len > 0; +} + +static bool get_cpu_model_from_proc(char* out, size_t out_size) +{ + FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo == nullptr) return false; + + char line[512]; + while (fgets(line, sizeof(line), cpuinfo) != nullptr) { + const char* field = std::strstr(line, "model name"); + if (field == nullptr) field = std::strstr(line, "Processor"); + if (field == nullptr) continue; + + const char* colon = std::strchr(field, ':'); + if (colon == nullptr) continue; + ++colon; + while (*colon == ' ' || *colon == '\t') { + ++colon; + } + const bool ok = copy_stripped(out, out_size, colon); + fclose(cpuinfo); + return ok; + } + fclose(cpuinfo); + return false; } -static std::string get_cpu_model() +static void get_cpu_model(char* out, size_t out_size) { - if (auto model_from_proc = get_cpu_model_from_proc(); !model_from_proc.empty()) { - return model_from_proc; - } else if (auto model_from_builtin = get_cpu_model_builtin(); !model_from_builtin.empty()) { - return model_from_builtin; + if (get_cpu_model_from_proc(out, out_size)) return; + + char buf[256]; + if (read_file_buf("/sys/firmware/devicetree/base/model", buf, sizeof(buf)) >= 0 || + read_file_buf("/proc/device-tree/model", buf, sizeof(buf)) >= 0) { + if (copy_stripped(out, out_size, buf)) return; } - return "Unknown"; + if (read_file_buf("/sys/devices/virtual/dmi/id/product_name", buf, sizeof(buf)) >= 0) { + if (copy_stripped(out, out_size, buf)) return; + } + std::snprintf(out, out_size, "Unknown"); } static const char* get_simd_target() @@ -166,26 +239,28 @@ struct host_memory_info_t { static host_memory_info_t get_host_memory_info() { - std::ifstream meminfo("/proc/meminfo"); - if (!meminfo.is_open()) return {}; + FILE* meminfo = fopen("/proc/meminfo", "r"); + if (meminfo == nullptr) return {}; - std::string line; + char line[256]; long total_kb = 0; long available_kb = 0; long free_kb = 0; - while (std::getline(meminfo, line)) { - std::istringstream fields(line); - std::string key; + int found = 0; + while (found < 3 && fgets(line, sizeof(line), meminfo) != nullptr) { long value_kb = 0; - fields >> key >> value_kb; - if (key == "MemTotal:") { + if (std::sscanf(line, "MemTotal: %ld", &value_kb) == 1) { total_kb = value_kb; - } else if (key == "MemAvailable:") { + ++found; + } else if (std::sscanf(line, "MemAvailable: %ld", &value_kb) == 1) { available_kb = value_kb; - } else if (key == "MemFree:") { + ++found; + } else if (std::sscanf(line, "MemFree: %ld", &value_kb) == 1) { free_kb = value_kb; + ++found; } } + fclose(meminfo); if (available_kb == 0) { available_kb = free_kb; } constexpr double kb_per_gib = 1024.0 * 1024.0; @@ -209,14 +284,18 @@ void print_version_info(int num_devices) CUOPT_GIT_COMMIT_HASH, CUOPT_CPU_ARCHITECTURE, CUOPT_CUDA_ARCHITECTURES); + const auto memory = get_host_memory_info(); - CUOPT_LOG_INFO( - "CPU: %s, threads (physical/logical): %d/%d, RAM (available/total): %.2f / %.2f GiB", - get_cpu_model().c_str(), - get_physical_cores(), - std::thread::hardware_concurrency(), - memory.available_gb, - memory.total_gb); + int allowed_cpus[CPU_SETSIZE]; + const int allowed_count = get_allowed_cpus(allowed_cpus, CPU_SETSIZE); + char cpu_model[256]; + get_cpu_model(cpu_model, sizeof(cpu_model)); + CUOPT_LOG_INFO("CPU: %s, threads: %dC/%dT, RAM usage: %.2f/%.2fGiB", + cpu_model, + get_physical_cores(allowed_cpus, allowed_count), + allowed_count, + std::max(0.0, memory.total_gb - memory.available_gb), + memory.total_gb); CUOPT_LOG_INFO("CPU SIMD target: %s", get_simd_target()); for (int device_id = 0; device_id < num_devices; ++device_id) { From 10ddf0ee2c0c5ef638ba0691aa76690746486410 Mon Sep 17 00:00:00 2001 From: yboucher Date: Wed, 19 Aug 2026 03:10:25 -0700 Subject: [PATCH 17/61] more logs --- cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu | 7 +++++-- cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 4cce1df459..4db353acd9 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1086,8 +1086,11 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; fj_cpu.h_best_assignment = fj_cpu.h_assignment; fj_cpu.iterations_since_best = 0; - CUOPT_LOG_TRACE( - "%sCPUFJ: new best objective: %g", fj_cpu.log_prefix.c_str(), fj_cpu.h_incumbent_objective); + // DEBUG, and reporting the stored best rather than the pre-epsilon incumbent, + // so it matches the binary path and the end-of-solve incumbent audit. + CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", + fj_cpu.log_prefix.c_str(), + fj_cpu.h_best_objective); if (fj_cpu.improvement_callback) { double current_work_units = fj_cpu.work_units_elapsed.load(std::memory_order_acquire); fj_cpu.improvement_callback( diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index fe41614cbb..bf0768ced4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -853,6 +853,12 @@ struct fj_bin_engine_t { climber.h_incumbent_objective = (f_t)incumbent_objective; climber.h_best_objective = (f_t)best_objective; climber.feasible_found = true; + // Emitted once per improvement so the benchmark harness can reconstruct the + // incumbent trajectory exactly, rather than sampling it at log_interval. + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] new incumbent: objective %.17g", + climber.log_prefix.c_str(), + coefficient_bits(), + best_objective); if (climber.improvement_callback) { const double work_units = climber.work_units_elapsed.load(std::memory_order_acquire); climber.improvement_callback((f_t)best_objective, h_best, work_units); From 69ac5a2f83e1de59f91fdcc93caac5677ebe5e1c Mon Sep 17 00:00:00 2001 From: yboucher Date: Wed, 19 Aug 2026 04:03:00 -0700 Subject: [PATCH 18/61] ai review --- .../feasibility_jump/fj_cpu_binary.cu | 47 +++++++++++++++++-- cpp/src/mip_heuristics/solve.cu | 3 +- cpp/src/pdlp/optimization_problem.cu | 4 +- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index bf0768ced4..6010385533 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -16,6 +16,11 @@ #include +#include +#include +#include +#include + #include #include #include @@ -264,10 +269,24 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) } } - const auto& offsets = c.h_offsets; - const auto& coeffs = c.h_coefficients; - const auto& cstr_lb = c.h_cstr_lb; - const auto& cstr_ub = c.h_cstr_ub; + const auto& offsets = c.h_offsets; + const auto& reverse_offsets = c.h_reverse_offsets; + const auto& reverse_constraints = c.h_reverse_constraints; + const auto& coeffs = c.h_coefficients; + const auto& cstr_lb = c.h_cstr_lb; + const auto& cstr_ub = c.h_cstr_ub; + + cuopt_assert( + thrust::all_of( + thrust::host, + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(n), + [&reverse_offsets, &reverse_constraints](int32_t v) { + const auto first = reverse_constraints.begin() + reverse_offsets[v]; + const auto last = reverse_constraints.begin() + reverse_offsets[v + 1]; + return thrust::adjacent_find(thrust::host, first, last) == last; + }), + "duplicate variable in CSR row"); double max_abs_coefficient = 0; std::vector row_values; @@ -315,11 +334,19 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) } double row_abs_sum = 0; + double row_lhs_min = 0; + double row_lhs_max = 0; for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { const double a = row_s * coeffs[k]; cuopt_assert(is_integer(a, tol), "row scaling left a fractional coefficient"); - const double abs_a = std::fabs(std::round(a)); + const double integral_a = std::round(a); + const double abs_a = std::fabs(integral_a); row_abs_sum += abs_a; + if (integral_a < 0) { + row_lhs_min += integral_a; + } else { + row_lhs_max += integral_a; + } if (abs_a > max_abs_coefficient) max_abs_coefficient = abs_a; } @@ -340,6 +367,16 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) out.bad_row = r; return out; } + const double integral_side = std::round(scaled_side); + const double min_slack = + s == 0 ? row_lhs_min - integral_side : integral_side - row_lhs_max; + const double max_slack = + s == 0 ? row_lhs_max - integral_side : integral_side - row_lhs_min; + if (!fj_bin_in_int32(min_slack) || !fj_bin_in_int32(max_slack)) { + out.reject = fj_binary_reject_t::lhs_headroom; + out.bad_row = r; + return out; + } } // Free rows are dropped: trivially satisfied, contributing nothing to the search. out.n_split_constraints += (int32_t)lb_fin + (int32_t)ub_fin; diff --git a/cpp/src/mip_heuristics/solve.cu b/cpp/src/mip_heuristics/solve.cu index 0fc4230192..5371a87026 100644 --- a/cpp/src/mip_heuristics/solve.cu +++ b/cpp/src/mip_heuristics/solve.cu @@ -470,8 +470,7 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p bool run_early_fj = run_presolve && settings.determinism_mode != CUOPT_MODE_DETERMINISTIC && op_problem.get_problem_category() != problem_category_t::LP && op_problem.get_n_constraints() > 0; - const f_t objective_sense = - op_problem.get_objective_scaling_factor() >= f_t{0} ? f_t{1} : f_t{-1}; + const f_t objective_sense = op_problem.get_sense() ? f_t{-1} : f_t{1}; f_t no_bound = objective_sense > f_t{0} ? (f_t)-1e20 : (f_t)1e20; auto early_fj_callback = [&early_best_user_score, diff --git a/cpp/src/pdlp/optimization_problem.cu b/cpp/src/pdlp/optimization_problem.cu index 18222ed012..d4f669a118 100644 --- a/cpp/src/pdlp/optimization_problem.cu +++ b/cpp/src/pdlp/optimization_problem.cu @@ -295,8 +295,8 @@ void optimization_problem_t::set_variable_types(const var_t* variable_ i_t n_discrete = 0; bool has_semi_continuous_variables = false; if ((size_t)size < host_variable_type_summary_limit) { - for (i_t i = 0; i < size; ++i) { - const var_t val = variable_types[i]; + const auto h_variable_types = cuopt::host_copy(variable_types_, stream_view_); + for (const var_t val : h_variable_types) { if (val == var_t::SEMI_CONTINUOUS) { has_semi_continuous_variables = true; ++n_discrete; From 64abba9dd6c96a803bf7d9867d285d6402bb8e9d Mon Sep 17 00:00:00 2001 From: yboucher Date: Wed, 19 Aug 2026 07:13:59 -0700 Subject: [PATCH 19/61] fix build --- cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 6010385533..03685a4e83 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -16,8 +16,8 @@ #include -#include #include +#include #include #include From 0c771a39a808aacfd018efef94e4e4c2c324eba9 Mon Sep 17 00:00:00 2001 From: yboucher Date: Wed, 19 Aug 2026 23:45:36 -0700 Subject: [PATCH 20/61] some optimizing --- .../linear_programming/cuopt/run_cpufj.cu | 10 +- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 240 +++++++++++++++--- .../feasibility_jump/fj_cpu.cuh | 71 +++++- 3 files changed, 285 insertions(+), 36 deletions(-) diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu index e62a8ef224..d071cc0648 100644 --- a/benchmarks/linear_programming/cuopt/run_cpufj.cu +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -129,14 +129,20 @@ int main(int argc, char** argv) mip::clamp_within_var_bounds(solution.assignment, &problem, &handle); handle.sync_stream(); - // Built serially: each climber host-copies the problem off the same stream. + // Built serially: only the first climber host-copies the problem off the stream, the rest clone + // its host data. std::vector> preemption_flags(n_climbers); std::vector>> climbers(n_climbers); for (int k = 0; k < n_climbers; ++k) { preemption_flags[k].store(false); mip::fj_settings_t settings; settings.seed = (int)(base_seed + k); - climbers[k] = mip::init_fj_cpu_standalone(problem, solution, preemption_flags[k], settings); + if (k == 0) { + climbers[k] = mip::init_fj_cpu_standalone(problem, solution, preemption_flags[k], settings); + } else { + climbers[k] = mip::init_fj_cpu_standalone_from_template( + problem, *climbers[0], preemption_flags[k], settings); + } // Portfolio diversification, decorrelated from the value RNG. std::mt19937 rng(base_seed + 7919u * k); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 4db353acd9..9adce2a3f7 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -18,6 +18,7 @@ #include +#include #include #include @@ -64,6 +65,16 @@ void finalize_fj_cpu_host_initialization( i_t nnz, const typename mip_solver_settings_t::tolerances_t& tolerances); +template +static void finalize_fj_cpu_host_initialization_from_template( + fj_cpu_climber_t& fj_cpu, + const fj_cpu_climber_t& tmpl, + i_t n_variables, + i_t n_constraints, + i_t n_integer_vars, + i_t nnz, + const typename mip_solver_settings_t::tolerances_t& tolerances); + template thrust::tuple get_mtm_for_bound(const typename fj_t::climber_data_t::view_t& fj, i_t var_idx, @@ -918,7 +929,7 @@ static void smooth_weights(fj_cpu_climber_t& fj_cpu) CPUFJ_NVTX_RANGE("CPUFJ::smooth_weights"); for (i_t cstr_idx = 0; cstr_idx < fj_cpu.view.pb.n_constraints; cstr_idx++) { // consider only satisfied constraints - if (fj_cpu.violated_constraints.count(cstr_idx)) continue; + if (fj_cpu.violated_constraints.contains(cstr_idx)) continue; f_t weight_l = max((f_t)0, fj_cpu.h_cstr_left_weights[cstr_idx] - 1); f_t weight_r = max((f_t)0, fj_cpu.h_cstr_right_weights[cstr_idx] - 1); @@ -1047,13 +1058,13 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, if (fabs(fj_cpu.h_lhs_sumcomp[cstr_idx]) > BIGVAL_THRESHOLD) fj_cpu.trigger_early_lhs_recomputation = true; - if (new_cost < -cstr_tolerance && !fj_cpu.violated_constraints.count(cstr_idx)) { + if (new_cost < -cstr_tolerance && !fj_cpu.violated_constraints.contains(cstr_idx)) { fj_cpu.violated_constraints.insert(cstr_idx); - cuopt_assert(fj_cpu.satisfied_constraints.count(cstr_idx) == 1, ""); - fj_cpu.satisfied_constraints.erase(cstr_idx); - } else if (!(new_cost < -cstr_tolerance) && fj_cpu.violated_constraints.count(cstr_idx)) { - cuopt_assert(fj_cpu.satisfied_constraints.count(cstr_idx) == 0, ""); - fj_cpu.violated_constraints.erase(cstr_idx); + cuopt_assert(fj_cpu.satisfied_constraints.contains(cstr_idx), ""); + fj_cpu.satisfied_constraints.remove(cstr_idx); + } else if (!(new_cost < -cstr_tolerance) && fj_cpu.violated_constraints.contains(cstr_idx)) { + cuopt_assert(!fj_cpu.satisfied_constraints.contains(cstr_idx), ""); + fj_cpu.violated_constraints.remove(cstr_idx); fj_cpu.satisfied_constraints.insert(cstr_idx); } @@ -1282,6 +1293,27 @@ static thrust::tuple find_mtm_move( return thrust::make_tuple(best_move, best_score); } +template +static void sample_with_replacement(const host_contiguous_set_t& pool, + i_t sample_size, + uint64_t seed, + std::vector& out) +{ + cuopt_assert(sample_size > 0, "invalid sample size"); + out.clear(); + const i_t pool_size = pool.size(); + if (pool_size == 0) { return; } + if (pool_size <= sample_size) { + out.assign(pool.begin(), pool.end()); + return; + } + out.reserve(sample_size); + cuopt::pcgenerator_t rng(seed); + for (i_t i = 0; i < sample_size; ++i) { + out.push_back(pool.contents[rng.next_u32() % (uint32_t)pool_size]); + } +} + template static thrust::tuple find_mtm_move_viol( fj_cpu_climber_t& fj_cpu, i_t sample_size = 100, bool localmin = false) @@ -1290,12 +1322,10 @@ static thrust::tuple find_mtm_move_viol( CPUFJ_NVTX_RANGE("CPUFJ::find_mtm_move_viol"); std::vector sampled_cstrs; - sampled_cstrs.reserve(sample_size); - std::sample(fj_cpu.violated_constraints.begin(), - fj_cpu.violated_constraints.end(), - std::back_inserter(sampled_cstrs), - sample_size, - fj_cpu.rng); + sample_with_replacement(fj_cpu.violated_constraints, + sample_size, + fj_cpu.settings.seed + fj_cpu.iterations, + sampled_cstrs); return find_mtm_move(fj_cpu, sampled_cstrs, localmin); } @@ -1308,12 +1338,10 @@ static thrust::tuple find_mtm_move_sat( CPUFJ_NVTX_RANGE("CPUFJ::find_mtm_move_sat"); std::vector sampled_cstrs; - sampled_cstrs.reserve(sample_size); - std::sample(fj_cpu.satisfied_constraints.begin(), - fj_cpu.satisfied_constraints.end(), - std::back_inserter(sampled_cstrs), - sample_size, - fj_cpu.rng); + sample_with_replacement(fj_cpu.satisfied_constraints, + sample_size, + fj_cpu.settings.seed + fj_cpu.iterations, + sampled_cstrs); return find_mtm_move(fj_cpu, sampled_cstrs); } @@ -1569,6 +1597,60 @@ static void init_fj_cpu(fj_cpu_climber_t& fj_cpu, problem.tolerances); } +template +static void init_fj_cpu_from_template(fj_cpu_climber_t& fj_cpu, + const fj_cpu_climber_t& tmpl, + problem_t& problem, + const std::vector& left_weights, + const std::vector& right_weights, + f_t objective_weight) +{ + cuopt_assert(tmpl.h_offsets.size() == static_cast(problem.n_constraints + 1), + "template built on a different problem"); + cuopt_assert(tmpl.h_reverse_offsets.size() == static_cast(problem.n_variables + 1), + "template built on a different problem"); + cuopt_assert(tmpl.h_coefficients.size() == static_cast(problem.nnz), + "template built on a different problem"); + + fj_cpu.view = typename fj_t::climber_data_t::view_t{}; + fj_cpu.view.pb = problem.view(); + fj_cpu.pb_ptr = &problem; + + fj_cpu.h_reverse_coefficients = tmpl.h_reverse_coefficients; + fj_cpu.h_reverse_constraints = tmpl.h_reverse_constraints; + fj_cpu.h_reverse_offsets = tmpl.h_reverse_offsets; + fj_cpu.h_coefficients = tmpl.h_coefficients; + fj_cpu.h_offsets = tmpl.h_offsets; + fj_cpu.h_variables = tmpl.h_variables; + fj_cpu.h_obj_coeffs = tmpl.h_obj_coeffs; + fj_cpu.h_var_bounds = tmpl.h_var_bounds; + fj_cpu.h_cstr_lb = tmpl.h_cstr_lb; + fj_cpu.h_cstr_ub = tmpl.h_cstr_ub; + fj_cpu.h_var_types = tmpl.h_var_types; + fj_cpu.h_is_binary_variable = tmpl.h_is_binary_variable; + fj_cpu.h_binary_indices = tmpl.h_binary_indices; + + fj_cpu.h_cstr_left_weights = left_weights; + fj_cpu.h_cstr_right_weights = right_weights; + fj_cpu.max_weight = 1.0; + fj_cpu.h_objective_weight = objective_weight; + fj_cpu.h_assignment = tmpl.h_assignment; + fj_cpu.h_best_assignment = tmpl.h_assignment; + fj_cpu.h_tabu_nodec_until.resize(problem.n_variables, 0); + fj_cpu.h_tabu_noinc_until.resize(problem.n_variables, 0); + fj_cpu.h_tabu_lastdec.resize(problem.n_variables, 0); + fj_cpu.h_tabu_lastinc.resize(problem.n_variables, 0); + fj_cpu.iterations = 0; + + finalize_fj_cpu_host_initialization_from_template(fj_cpu, + tmpl, + problem.n_variables, + problem.n_constraints, + problem.n_integer_vars, + problem.nnz, + problem.tolerances); +} + template static void set_host_data_view( fj_cpu_climber_t& fj_cpu, @@ -1612,7 +1694,7 @@ static void set_host_data_view( } template -void finalize_fj_cpu_host_initialization( +static void wire_fj_cpu_host_views( fj_cpu_climber_t& fj_cpu, i_t n_variables, i_t n_constraints, @@ -1620,8 +1702,6 @@ void finalize_fj_cpu_host_initialization( i_t nnz, const typename mip_solver_settings_t::tolerances_t& tolerances) { - raft::common::nvtx::range scope("finalize_fj_cpu_host_initialization"); - cuopt_assert(n_variables >= 0, "invalid variable count"); cuopt_assert(n_constraints >= 0, "invalid constraint count"); cuopt_assert(fj_cpu.h_offsets.size() == static_cast(n_constraints + 1), @@ -1655,6 +1735,30 @@ void finalize_fj_cpu_host_initialization( fj_cpu.view.best_objective = &fj_cpu.h_best_objective; fj_cpu.view.settings = &fj_cpu.settings; + fj_cpu.h_best_objective = +std::numeric_limits::infinity(); + + // nnz count + fj_cpu.cached_mtm_moves.resize(fj_cpu.h_coefficients.size(), + std::make_pair(0, fj_staged_score_t::zero())); + + fj_cpu.flip_move_computed.resize(n_variables, false); + fj_cpu.var_bitmap.resize(n_variables, false); + fj_cpu.iter_mtm_vars.reserve(n_variables); +} + +template +void finalize_fj_cpu_host_initialization( + fj_cpu_climber_t& fj_cpu, + i_t n_variables, + i_t n_constraints, + i_t n_integer_vars, + i_t nnz, + const typename mip_solver_settings_t::tolerances_t& tolerances) +{ + raft::common::nvtx::range scope("finalize_fj_cpu_host_initialization"); + + wire_fj_cpu_host_views(fj_cpu, n_variables, n_constraints, n_integer_vars, nnz, tolerances); + fj_cpu.h_objective_vars.resize(n_variables); auto end = std::copy_if( thrust::counting_iterator(0), @@ -1665,12 +1769,6 @@ void finalize_fj_cpu_host_initialization( fj_cpu.view.objective_vars = raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); - fj_cpu.h_best_objective = +std::numeric_limits::infinity(); - - // nnz count - fj_cpu.cached_mtm_moves.resize(fj_cpu.h_coefficients.size(), - std::make_pair(0, fj_staged_score_t::zero())); - fj_cpu.cached_cstr_bounds.resize(fj_cpu.h_reverse_coefficients.size()); for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); @@ -1698,12 +1796,61 @@ void finalize_fj_cpu_host_initialization( fj_cpu.var_bitmap.resize(n_variables, false); fj_cpu.iter_mtm_vars.reserve(n_variables); + // Must precede recompute_lhs, which is what first populates them. + fj_cpu.violated_constraints.resize(n_constraints); + fj_cpu.satisfied_constraints.resize(n_constraints); + recompute_lhs(fj_cpu); // Precompute static problem features for regression model precompute_problem_features(fj_cpu); } +template +static void finalize_fj_cpu_host_initialization_from_template( + fj_cpu_climber_t& fj_cpu, + const fj_cpu_climber_t& tmpl, + i_t n_variables, + i_t n_constraints, + i_t n_integer_vars, + i_t nnz, + const typename mip_solver_settings_t::tolerances_t& tolerances) +{ + raft::common::nvtx::range scope("finalize_fj_cpu_host_initialization_from_template"); + + cuopt_assert(tmpl.h_lhs.size() == static_cast(n_constraints), "template lhs mismatch"); + cuopt_assert(tmpl.violated_constraints.max_size() == n_constraints, + "template violated set mismatch"); + cuopt_assert(tmpl.satisfied_constraints.max_size() == n_constraints, + "template satisfied set mismatch"); + cuopt_assert(tmpl.cached_cstr_bounds.size() == fj_cpu.h_reverse_coefficients.size(), + "template cached bounds mismatch"); + + fj_cpu.h_objective_vars = tmpl.h_objective_vars; + fj_cpu.cached_cstr_bounds = tmpl.cached_cstr_bounds; + + fj_cpu.h_lhs = tmpl.h_lhs; + fj_cpu.h_lhs_sumcomp = tmpl.h_lhs_sumcomp; + fj_cpu.violated_constraints = tmpl.violated_constraints; + fj_cpu.satisfied_constraints = tmpl.satisfied_constraints; + fj_cpu.total_violations = tmpl.total_violations; + fj_cpu.h_incumbent_objective = tmpl.h_incumbent_objective; + + fj_cpu.n_binary_vars = tmpl.n_binary_vars; + fj_cpu.n_integer_vars = tmpl.n_integer_vars; + fj_cpu.avg_var_degree = tmpl.avg_var_degree; + fj_cpu.max_var_degree = tmpl.max_var_degree; + fj_cpu.var_degree_cv = tmpl.var_degree_cv; + fj_cpu.avg_cstr_degree = tmpl.avg_cstr_degree; + fj_cpu.max_cstr_degree = tmpl.max_cstr_degree; + fj_cpu.cstr_degree_cv = tmpl.cstr_degree_cv; + fj_cpu.problem_density = tmpl.problem_density; + + wire_fj_cpu_host_views(fj_cpu, n_variables, n_constraints, n_integer_vars, nnz, tolerances); + fj_cpu.view.objective_vars = + raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); +} + template static std::unique_ptr> init_fj_cpu_from_host_lp( const lp_problem_t& problem, @@ -1830,7 +1977,7 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) // Check that each violated constraint is actually violated and not present in // satisfied_constraints for (const auto& cstr_idx : fj_cpu.violated_constraints) { - cuopt_assert(fj_cpu.satisfied_constraints.count(cstr_idx) == 0, + cuopt_assert(!fj_cpu.satisfied_constraints.contains(cstr_idx), "Violated constraint also in satisfied_constraints"); f_t lhs = fj_cpu.h_lhs[cstr_idx]; f_t tol = fj_cpu.view.get_corrected_tolerance(cstr_idx); @@ -1841,7 +1988,7 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) // Check that each satisfied constraint is actually satisfied and not present in // violated_constraints for (const auto& cstr_idx : fj_cpu.satisfied_constraints) { - cuopt_assert(fj_cpu.violated_constraints.count(cstr_idx) == 0, + cuopt_assert(!fj_cpu.violated_constraints.contains(cstr_idx), "Satisfied constraint also in violated_constraints"); f_t lhs = fj_cpu.h_lhs[cstr_idx]; f_t tol = fj_cpu.view.get_corrected_tolerance(cstr_idx); @@ -1851,8 +1998,8 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) // Check that each constraint is in exactly one of violated_constraints or satisfied_constraints for (i_t cstr_idx = 0; cstr_idx < fj_cpu.view.pb.n_constraints; ++cstr_idx) { - bool in_viol = fj_cpu.violated_constraints.count(cstr_idx) > 0; - bool in_sat = fj_cpu.satisfied_constraints.count(cstr_idx) > 0; + bool in_viol = fj_cpu.violated_constraints.contains(cstr_idx); + bool in_sat = fj_cpu.satisfied_constraints.contains(cstr_idx); cuopt_assert( in_viol != in_sat, "Constraint must be in exactly one of violated_constraints or satisfied_constraints"); @@ -2270,6 +2417,25 @@ std::unique_ptr> init_fj_cpu_standalone( return fj_cpu; } +template +std::unique_ptr> init_fj_cpu_standalone_from_template( + problem_t& problem, + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings) +{ + raft::common::nvtx::range scope("init_fj_cpu_standalone_from_template"); + + auto fj_cpu = std::make_unique>(preemption_flag); + + std::vector default_weights(problem.n_constraints, 1.0); + init_fj_cpu_from_template(*fj_cpu, tmpl, problem, default_weights, default_weights, 0.0); + fj_cpu->settings = settings; + fj_cpu->settings.seed = cuopt::seed_generator::get_seed(); + + return fj_cpu; +} + template void fj_cpu_worker_t::fj_cpu_deleter_t::operator()(fj_cpu_climber_t* ptr) const { @@ -2347,6 +2513,11 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_standalone_from_template( + problem_t& problem, + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings); template std::unique_ptr> init_fj_cpu_from_optimization_problem( const optimization_problem_t& problem, const typename mip_solver_settings_t::tolerances_t& tolerances, @@ -2372,6 +2543,11 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_standalone_from_template( + problem_t& problem, + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings); template std::unique_ptr> init_fj_cpu_from_optimization_problem( const optimization_problem_t& problem, const typename mip_solver_settings_t::tolerances_t& tolerances, diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 89db669dd0..302c1a4e13 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -8,6 +8,7 @@ #pragma once #include +#include #include #include #include @@ -25,6 +26,65 @@ namespace cuopt::mathematical_optimization::mip { template class probing_cache_t; +template +struct host_contiguous_set_t { + void resize(i_t max_size) + { + cuopt_assert(max_size >= 0, "invalid max size"); + contents.clear(); + contents.reserve(max_size); + index_map.assign(max_size, -1); + is_member.assign(max_size, 0); + } + + void clear() + { + for (i_t val : contents) { + index_map[val] = -1; + is_member[val] = 0; + } + contents.clear(); + } + + void insert(i_t val) + { + cuopt_assert(val >= 0 && val < max_size(), "Value is out of bounds"); + cuopt_assert(!contains(val), "Value already exists"); + index_map[val] = contents.size(); + is_member[val] = 1; + contents.push_back(val); + } + + void remove(i_t val) + { + cuopt_assert(val >= 0 && val < max_size(), "Value is out of bounds"); + cuopt_assert(contains(val), "Value not found"); + const i_t idx = index_map[val]; + const i_t last_val = contents.back(); + contents[idx] = last_val; + index_map[last_val] = idx; + contents.pop_back(); + index_map[val] = -1; + is_member[val] = 0; + } + + bool contains(i_t val) const + { + cuopt_assert(val >= 0 && val < max_size(), "Value is out of bounds"); + return is_member[val] != 0; + } + + auto begin() const { return contents.begin(); } + auto end() const { return contents.end(); } + i_t size() const { return contents.size(); } + i_t max_size() const { return index_map.size(); } + bool empty() const { return contents.empty(); } + + std::vector contents; + std::vector index_map; + std::vector is_member; +}; + // NOTE: this seems an easy pick for reflection/xmacros once this is available (C++26?) // Maintaining a single source of truth for all members would be nice template @@ -123,8 +183,8 @@ struct fj_cpu_climber_t { f_t h_best_objective; i_t last_feasible_entrance_iter{0}; i_t iterations; - std::unordered_set violated_constraints; - std::unordered_set satisfied_constraints; + host_contiguous_set_t violated_constraints; + host_contiguous_set_t satisfied_constraints; bool feasible_found{false}; bool trigger_early_lhs_recomputation{false}; f_t total_violations{0}; @@ -232,6 +292,13 @@ std::unique_ptr> init_fj_cpu_standalone( std::atomic& preemption_flag, fj_settings_t settings = fj_settings_t{}); +template +std::unique_ptr> init_fj_cpu_standalone_from_template( + problem_t& problem, + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings = fj_settings_t{}); + template std::unique_ptr> init_fj_cpu_from_optimization_problem( const optimization_problem_t& problem, From 9d46a9f5264aa291e8d795f6f9348b1725813b27 Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 00:01:08 -0700 Subject: [PATCH 21/61] some bug fixes regarding lift moves --- .../linear_programming/cuopt/run_cpufj.cu | 3 +-- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 17 ++++++++++++ .../feasibility_jump/fj_cpu_binary.cu | 27 ++++++++++++------- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu index d071cc0648..d8144132d3 100644 --- a/benchmarks/linear_programming/cuopt/run_cpufj.cu +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -129,8 +129,7 @@ int main(int argc, char** argv) mip::clamp_within_var_bounds(solution.assignment, &problem, &handle); handle.sync_stream(); - // Built serially: only the first climber host-copies the problem off the stream, the rest clone - // its host data. + // Built serially: the first climber host-copies the problem, the rest clone it. std::vector> preemption_flags(n_climbers); std::vector>> climbers(n_climbers); for (int k = 0; k < n_climbers; ++k) { diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 9adce2a3f7..c86aead066 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1415,6 +1415,23 @@ static thrust::tuple find_lift_move( delta = round(1.0 - 2 * val); // flip move wouldn't improve if (delta * obj_coeff >= 0) continue; + + auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); + bool breaks_a_row = false; + for (i_t j = offset_begin; j < offset_end; ++j) { + auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[j].get(); + const i_t cstr_idx = fj_cpu.h_reverse_constraints[j]; + const f_t cstr_coeff = fj_cpu.h_reverse_coefficients[j]; + const f_t lhs = fj_cpu.h_lhs[cstr_idx]; + const f_t sumcomp = fj_cpu.h_lhs_sumcomp[cstr_idx]; + const f_t new_lhs = lhs + (cstr_coeff * delta - sumcomp); + if (fj_cpu.view.excess_score(cstr_idx, new_lhs, c_lb, c_ub) < + -fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub)) { + breaks_a_row = true; + break; + } + } + if (breaks_a_row) continue; } else { f_t lfd_lb = get_lower(fj_cpu.h_var_bounds[var_idx].get()) - val; f_t lfd_ub = get_upper(fj_cpu.h_var_bounds[var_idx].get()) - val; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 03685a4e83..6e2e3e0215 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -578,6 +578,7 @@ struct fj_bin_engine_t { std::vector sample_buf; // move-selection row sample, reused to keep the loop allocation-free int32_t objective_weight{0}; + int32_t seed_objective_weight{0}; double incumbent_objective{0}; double best_objective{std::numeric_limits::infinity()}; int32_t max_weight{1}; @@ -1055,12 +1056,16 @@ struct fj_bin_engine_t { std::pair find_lift_move() const { + cuopt_assert(violated_list.empty(), "lift moves require a feasible incumbent"); + int32_t best_v = -1; int64_t best_s = 0; for (int32_t v : pb.objective_vars) { const int8_t delta = (int8_t)(1 - 2 * assign[v]); if ((double)delta * pb.objective[v] >= 0) continue; if (tabu_blocked(v, false)) continue; + // Base field is zero iff the flip breaks no row; K/2 splits it while |bonus| < 2^31. + if (var_score[v] <= -(fj_bin_score_k / 2)) continue; const int64_t s = (int64_t)(-std::llround(pb.objective[v] * delta)) * fj_bin_score_k; if (s > best_s) { best_s = s; @@ -1090,7 +1095,7 @@ struct fj_bin_engine_t { for (int32_t v = 0; v < pb.n_variables; ++v) assign_i32[v] = assign[v]; for (int32_t r = 0; r < pb.n_constraints; ++r) row_weight[r] = pb.initial_weight[r]; max_weight = fj_bin_ddfw_init; - objective_weight = 0; + objective_weight = seed_objective_weight; tabu.clear(iters); recompute_slack(); last_restart_iter = iters; @@ -1141,14 +1146,18 @@ struct fj_bin_engine_t { violated_list.clear(); var_bitmap.assign(n, 0); - argmax_tile = fj_bin_argmax_tile(); - objective_weight = 0; - max_weight = fj_bin_ddfw_init; - incumbent_objective = 0; - best_objective = std::numeric_limits::infinity(); - feasible_found = false; - iters = 0; - last_restart_iter = 0; + const int32_t seeded_weight = (int32_t)std::lround(climber.h_objective_weight); + cuopt_assert(seeded_weight >= 0, "objective weight should be positive or zero"); + + argmax_tile = fj_bin_argmax_tile(); + objective_weight = seeded_weight > 0 ? seeded_weight : 0; + seed_objective_weight = objective_weight; + max_weight = fj_bin_ddfw_init; + incumbent_objective = 0; + best_objective = std::numeric_limits::infinity(); + feasible_found = false; + iters = 0; + last_restart_iter = 0; recompute_slack(); } From 996c832ceb979092648b3a06842e74486a1369b4 Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 04:00:47 -0700 Subject: [PATCH 22/61] hiverge inspired improvements --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 78 ++++++++++++++++ .../feasibility_jump/fj_cpu.cuh | 14 +++ .../feasibility_jump/fj_cpu_binary.cu | 89 +++++++++++++++++++ 3 files changed, 181 insertions(+) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index c86aead066..41d456d245 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1520,6 +1520,12 @@ template static void perturb(fj_cpu_climber_t& fj_cpu) { CPUFJ_NVTX_RANGE("CPUFJ::perturb"); + if (fj_cpu.feasible_found) { + cuopt_assert(fj_cpu.h_assignment.size() == fj_cpu.h_best_assignment.size(), + "incumbent_assignment span would be invalidated"); + fj_cpu.h_assignment = fj_cpu.h_best_assignment; + } + // select N variables, assign them a random value between their bounds std::vector sampled_vars; std::sample(fj_cpu.h_objective_vars.begin(), @@ -1548,6 +1554,67 @@ static void perturb(fj_cpu_climber_t& fj_cpu) recompute_lhs(fj_cpu); } +template +static void reset_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) +{ + fj_cpu.h_best_infeasible_assignment.clear(); + fj_cpu.best_infeasible_severity = std::numeric_limits::infinity(); + fj_cpu.checkpoint_severity = std::numeric_limits::infinity(); + fj_cpu.iters_since_infeasible_improve = 0; +} + +template +static void restart_from_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) +{ + cuopt_assert(fj_cpu.h_assignment.size() == fj_cpu.h_best_infeasible_assignment.size(), + "incumbent_assignment span would be invalidated"); + fj_cpu.h_assignment = fj_cpu.h_best_infeasible_assignment; + recompute_lhs(fj_cpu); + for (size_t i = 0; i < fj_cpu.cached_mtm_moves.size(); ++i) + fj_cpu.cached_mtm_moves[i].first = 0; +} + +template +static void track_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) +{ + CPUFJ_NVTX_RANGE("CPUFJ::track_infeasible_checkpoint"); + if (fj_cpu.violated_constraints.empty()) { + reset_infeasible_checkpoint(fj_cpu); + return; + } + + const f_t severity = -fj_cpu.total_violations; + cuopt_assert(severity >= 0, "violation severity should be positive or zero"); + + if (severity < fj_cpu.best_infeasible_severity) { + fj_cpu.best_infeasible_severity = severity; + fj_cpu.iters_since_infeasible_improve = 0; + fj_cpu.restores_since_improvement = 0; + if (severity < fj_cpu.checkpoint_severity * fj_cpu.infeasible_checkpoint_refresh_ratio) { + fj_cpu.h_best_infeasible_assignment = fj_cpu.h_assignment; + fj_cpu.checkpoint_severity = severity; + ++fj_cpu.n_checkpoint_snapshots; + } + return; + } + + if (fj_cpu.restores_since_improvement >= fj_cpu.infeasible_restart_max_streak) return; + if (++fj_cpu.iters_since_infeasible_improve < fj_cpu.infeasible_restart_window) return; + if (severity <= fj_cpu.best_infeasible_severity * fj_cpu.infeasible_restart_degrade_ratio) return; + if (fj_cpu.h_best_infeasible_assignment.empty()) return; + + cuopt_assert(fj_cpu.checkpoint_severity >= fj_cpu.best_infeasible_severity, + "checkpoint cannot beat the best severity seen"); + + restart_from_infeasible_checkpoint(fj_cpu); + + ++fj_cpu.n_checkpoint_restores; + ++fj_cpu.restores_since_improvement; + if (fj_cpu.restores_since_improvement > fj_cpu.max_restores_since_improvement) + fj_cpu.max_restores_since_improvement = fj_cpu.restores_since_improvement; + fj_cpu.iters_since_infeasible_improve = 0; +} + template static void init_fj_cpu(fj_cpu_climber_t& fj_cpu, solution_t& solution, @@ -2073,6 +2140,11 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w fj_cpu->last_feature_log_time = loop_start; fj_cpu->prev_best_objective = fj_cpu->h_best_objective; fj_cpu->iterations_since_best = 0; + reset_infeasible_checkpoint(*fj_cpu); + fj_cpu->n_checkpoint_restores = 0; + fj_cpu->n_checkpoint_snapshots = 0; + fj_cpu->restores_since_improvement = 0; + fj_cpu->max_restores_since_improvement = 0; while (!fj_cpu->halted && !fj_cpu->preemption_flag.load()) { // Check if 5 seconds have passed @@ -2142,6 +2214,7 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w } else { // Local Min update_weights(*fj_cpu); + track_infeasible_checkpoint(*fj_cpu); if (should_perturb) { perturb(*fj_cpu); for (size_t i = 0; i < fj_cpu->cached_mtm_moves.size(); i++) @@ -2229,6 +2302,11 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w CUOPT_LOG_TRACE("%sCPUFJ Average time per iteration: %.8fms", fj_cpu->log_prefix.c_str(), avg_time_per_iter * 1000.0); + CUOPT_LOG_DEBUG("%sCPUFJ checkpoint: %lld restores, %lld snapshots, max streak %d", + fj_cpu->log_prefix.c_str(), + (long long)fj_cpu->n_checkpoint_restores, + (long long)fj_cpu->n_checkpoint_snapshots, + fj_cpu->max_restores_since_improvement); #if CPUFJ_TIMING_TRACE // Print final timing statistics diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 302c1a4e13..37505682c9 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -125,6 +125,7 @@ struct fj_cpu_climber_t { ADD_INSTRUMENTED(h_cstr_right_weights), ADD_INSTRUMENTED(h_assignment), ADD_INSTRUMENTED(h_best_assignment), + ADD_INSTRUMENTED(h_best_infeasible_assignment), ADD_INSTRUMENTED(cached_cstr_bounds), ADD_INSTRUMENTED(iter_mtm_vars)}; @@ -222,10 +223,23 @@ struct fj_cpu_climber_t { std::vector> two_opt_partners; std::vector> two_opt_row_deltas; + ins_vector h_best_infeasible_assignment; + f_t best_infeasible_severity{std::numeric_limits::infinity()}; + f_t checkpoint_severity{std::numeric_limits::infinity()}; + i_t iters_since_infeasible_improve{0}; + i_t restores_since_improvement{0}; + i_t max_restores_since_improvement{0}; + int64_t n_checkpoint_restores{0}; + int64_t n_checkpoint_snapshots{0}; + i_t mtm_viol_samples{25}; i_t mtm_sat_samples{15}; i_t nnz_samples{50000}; i_t perturb_interval{100}; + i_t infeasible_restart_window{300}; + i_t infeasible_restart_max_streak{20}; + f_t infeasible_restart_degrade_ratio{1.15}; + f_t infeasible_checkpoint_refresh_ratio{0.99}; i_t log_interval{1000}; i_t diversity_callback_interval{3000}; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 6e2e3e0215..c234c69186 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -559,6 +559,12 @@ struct fj_bin_engine_t { std::vector seed_assign; // restart target std::vector assign_i32; // gather mirror for the SIMD patch (Batch B) + std::vector best_infeasible_assign; + int64_t best_infeasible_severity{std::numeric_limits::max()}; + int64_t checkpoint_severity{std::numeric_limits::max()}; + int32_t iters_since_infeasible_improve{0}; + int32_t restores_since_improvement{0}; + std::vector var_score; // live feasibility score of flipping each variable std::vector nnz_score_delta; // per CSR nnz: last score delta of variables[k] in its row @@ -594,6 +600,10 @@ struct fj_bin_engine_t { int64_t nnz_patched{0}; int64_t rows_walked{0}; + int64_t n_checkpoint_restores{0}; + int64_t n_checkpoint_snapshots{0}; + int32_t max_restores_since_improvement{0}; + // Tile width for the argmax sweep, in variables. Set at init from fj_bin_argmax_tile(). int32_t argmax_tile{fj_bin_argmax_tile_target}; @@ -604,6 +614,10 @@ struct fj_bin_engine_t { int32_t perturb_interval{100}; int32_t mtm_viol_samples{25}; int32_t mtm_sat_samples{15}; + int32_t infeasible_restart_window{300}; + int32_t infeasible_restart_max_streak{20}; + double infeasible_restart_degrade_ratio{1.15}; + double infeasible_checkpoint_refresh_ratio{0.99}; double breakthrough_margin{1e-4}; int32_t max_aggregate_base{0}; @@ -948,6 +962,61 @@ struct fj_bin_engine_t { reweight_constraint(best_donor, row_weight[best_donor] - fj_bin_ddfw_transfer); } if (violated_list.empty()) objective_weight += 1; + track_infeasible_checkpoint(); + } + + void reset_infeasible_checkpoint() + { + best_infeasible_assign.clear(); + best_infeasible_severity = std::numeric_limits::max(); + checkpoint_severity = std::numeric_limits::max(); + iters_since_infeasible_improve = 0; + } + + void track_infeasible_checkpoint() + { + if (violated_list.empty()) { + reset_infeasible_checkpoint(); + return; + } + + int64_t severity = 0; + for (int32_t r : violated_list) { + cuopt_assert(row_slack[r] < 0, "row in violated_list is not violated"); + severity -= (int64_t)row_slack[r]; + } + + if (severity < best_infeasible_severity) { + best_infeasible_severity = severity; + iters_since_infeasible_improve = 0; + restores_since_improvement = 0; + if ((double)severity < (double)checkpoint_severity * infeasible_checkpoint_refresh_ratio) { + best_infeasible_assign = assign; + checkpoint_severity = severity; + ++n_checkpoint_snapshots; + } + return; + } + + if (restores_since_improvement >= infeasible_restart_max_streak) return; + if (++iters_since_infeasible_improve < infeasible_restart_window) return; + if ((double)severity <= (double)best_infeasible_severity * infeasible_restart_degrade_ratio) + return; + if (best_infeasible_assign.empty()) return; + + cuopt_assert(checkpoint_severity >= best_infeasible_severity, + "checkpoint cannot beat the best severity seen"); + + assign = best_infeasible_assign; + for (int32_t v = 0; v < pb.n_variables; ++v) + assign_i32[v] = assign[v]; + recompute_slack(); + + ++n_checkpoint_restores; + ++restores_since_improvement; + if (restores_since_improvement > max_restores_since_improvement) + max_restores_since_improvement = restores_since_improvement; + iters_since_infeasible_improve = 0; } // Global argmax over every variable, affordable because var_score is maintained live. While the @@ -1096,6 +1165,7 @@ struct fj_bin_engine_t { for (int32_t r = 0; r < pb.n_constraints; ++r) row_weight[r] = pb.initial_weight[r]; max_weight = fj_bin_ddfw_init; objective_weight = seed_objective_weight; + reset_infeasible_checkpoint(); tabu.clear(iters); recompute_slack(); last_restart_iter = iters; @@ -1113,6 +1183,18 @@ struct fj_bin_engine_t { perturb_interval = climber.perturb_interval; mtm_viol_samples = climber.mtm_viol_samples; mtm_sat_samples = climber.mtm_sat_samples; + + infeasible_restart_window = climber.infeasible_restart_window; + infeasible_restart_max_streak = climber.infeasible_restart_max_streak; + infeasible_restart_degrade_ratio = (double)climber.infeasible_restart_degrade_ratio; + infeasible_checkpoint_refresh_ratio = (double)climber.infeasible_checkpoint_refresh_ratio; + cuopt_assert(infeasible_restart_window > 0, "invalid infeasible restart window"); + cuopt_assert(infeasible_restart_max_streak > 0, "invalid infeasible restart streak cap"); + cuopt_assert(infeasible_restart_degrade_ratio >= 1.0, "degrade ratio should be at least one"); + cuopt_assert( + infeasible_checkpoint_refresh_ratio > 0.0 && infeasible_checkpoint_refresh_ratio <= 1.0, + "checkpoint refresh ratio should be in (0, 1]"); + if (tabu_tenure_max <= tabu_tenure_min) tabu_tenure_max = tabu_tenure_min + 1; // The tabu ring is indexed by iteration modulo its size, so a slot is reused after ring_size @@ -1132,6 +1214,7 @@ struct fj_bin_engine_t { } seed_assign = assign; best_assign = assign; + reset_infeasible_checkpoint(); assign_i32.assign(n, 0); for (int32_t v = 0; v < n; ++v) assign_i32[v] = assign[v]; @@ -1244,6 +1327,12 @@ struct fj_bin_engine_t { coefficient_bits(), (long long)nnz_patched, (long long)rows_walked); + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] checkpoint: %lld restores, %lld snapshots, max streak %d", + climber.log_prefix.c_str(), + coefficient_bits(), + (long long)n_checkpoint_restores, + (long long)n_checkpoint_snapshots, + max_restores_since_improvement); } }; From 075ded0c5263405a4544f182ee3792c839a6170e Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 04:44:09 -0700 Subject: [PATCH 23/61] ddfw improvements --- .../feasibility_jump/fj_cpu_binary.cu | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index c234c69186..536254bb31 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -199,9 +199,13 @@ constexpr int64_t fj_bin_scale_cap = std::numeric_limits::max(); // reason to promote them alongside the other FJ knobs. constexpr int32_t fj_bin_ddfw_init = 10; // initial weight, also the donation floor constexpr int32_t fj_bin_ddfw_transfer = 1; -constexpr int32_t fj_bin_ddfw_donor_samples = 1; +constexpr int32_t fj_bin_ddfw_donor_samples = 4; constexpr int32_t fj_bin_restart_period = 5000000; +// Escalation threshold and step, in infeasible local minima without a severity improvement. +constexpr int32_t fj_bin_ddfw_escalate_after = 2000; +constexpr int32_t fj_bin_ddfw_escalate_max = 100; + // prefetch distance // TODO: check if it actually matters at all for performance constexpr int32_t fj_bin_pf_dist = 8; @@ -940,13 +944,27 @@ struct fj_bin_engine_t { // DDFW: every violated row gains weight taken from a satisfied neighbour above the donation // floor, so total weight is roughly conserved and differentiation stays local to the hard region. + // Unit transfers stop moving the landscape on a long stall, so the amount grows with the stall. + int32_t ddfw_transfer() const + { + if (iters_since_infeasible_improve <= fj_bin_ddfw_escalate_after) return fj_bin_ddfw_transfer; + const int32_t over = iters_since_infeasible_improve - fj_bin_ddfw_escalate_after; + const int32_t steps = over / fj_bin_ddfw_escalate_after + 1; + const int32_t scale = steps < fj_bin_ddfw_escalate_max ? steps : fj_bin_ddfw_escalate_max; + return fj_bin_ddfw_transfer * scale; + } + void update_weights() { + const int32_t transfer = ddfw_transfer(); + // Donors must stay above the floor, or weights go negative and every base score inverts. + const int32_t donor_floor = fj_bin_ddfw_init + transfer - 1; + for (int32_t cf : violated_list) { - reweight_constraint(cf, row_weight[cf] + fj_bin_ddfw_transfer); + reweight_constraint(cf, row_weight[cf] + transfer); const int32_t vo = pb.offsets[cf], ve = pb.offsets[cf + 1]; if (ve <= vo) continue; - int32_t best_donor = -1, best_w = fj_bin_ddfw_init; + int32_t best_donor = -1, best_w = donor_floor; for (int32_t s = 0; s < fj_bin_ddfw_donor_samples; ++s) { const int32_t v = pb.variables[vo + (int32_t)(rng.next_u32() % (uint32_t)(ve - vo))]; const int32_t no = pb.reverse_offsets[v], ne = pb.reverse_offsets[v + 1]; @@ -958,8 +976,11 @@ struct fj_bin_engine_t { best_donor = d; } } - if (best_donor >= 0) - reweight_constraint(best_donor, row_weight[best_donor] - fj_bin_ddfw_transfer); + if (best_donor >= 0) { + const int32_t donated = row_weight[best_donor] - transfer; + cuopt_assert(donated >= fj_bin_ddfw_init, "donation broke the weight floor"); + reweight_constraint(best_donor, donated); + } } if (violated_list.empty()) objective_weight += 1; track_infeasible_checkpoint(); From cdc805b7b49623b8c682bdc32ea79f85e4f3e21b Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 06:16:08 -0700 Subject: [PATCH 24/61] changes for the hiverge harness --- .../linear_programming/cuopt/run_cpufj.cu | 60 ++++++++++++------- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 47 +++++++++++++++ .../feasibility_jump/fj_cpu.cuh | 12 ++++ cpp/src/mip_heuristics/solution/solution.cu | 26 ++++++++ cpp/src/mip_heuristics/solution/solution.cuh | 9 +++ 5 files changed, 131 insertions(+), 23 deletions(-) diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu index d8144132d3..781c28236c 100644 --- a/benchmarks/linear_programming/cuopt/run_cpufj.cu +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -116,40 +116,54 @@ int main(int argc, char** argv) cuopt::mathematical_optimization::mps_data_model_to_optimization_problem( &handle, mps_data_model); mip::problem_t problem(op_problem); + + // Anonymise the instance before anything under evolution can see it. + // + // problem_t exposes var_names, row_names and objective_name as public members, and + // the FJ code receives problem_t&. For a fixed benchmark set those strings are an + // exact fingerprint -- row_names[0] alone identifies most MIPLIB instances -- so a + // candidate could branch on identity and return a memorised objective. Reading the + // MODEL is intended and useful: coefficients, bounds, variable types, sparsity and + // row structure are all untouched here, so recognising set-packing rows, knapsack + // substructure or GUB constraints still works exactly as before. Only the labels go. + // + // Each string is cleared in place rather than the vectors being emptied, so size() + // and indexing stay valid and any code that walks names by variable index still + // works -- it just gets empty strings. + // + // This file is outside target_code and is sha256-gated by evaluate.py's FROZEN_FILES, + // so a candidate cannot restore the names. Do not move this below the solve. + for (auto& name : problem.var_names) name.clear(); + for (auto& name : problem.row_names) name.clear(); + problem.objective_name.clear(); + std::printf("instance: %s n_vars=%d n_cstrs=%d nnz=%d\n", path.c_str(), problem.n_variables, problem.n_constraints, problem.nnz); - // Zero start, clamped into the variable bounds. Shared by every climber; diversity comes from - // the per-climber seed and sampling parameters below. + // FROZEN -- defines t=0 for the benchmark. Everything above it (the MPS parse, + // problem construction under problem/, and the name anonymisation) is outside + // target_code; everything below it is editable. A marker any later would leave + // editable code ahead of the clock, which is somewhere to do unmeasured work; any + // earlier would charge the budget for a parse and a CUDA context no candidate can + // influence. + CUOPT_LOG_INFO("CPUFJ solve window start"); + + // Shared by every climber. Built by build_start_assignment, which is editable -- + // this driver is not. mip::solution_t solution(problem); - thrust::fill(handle.get_thrust_policy(), solution.assignment.begin(), solution.assignment.end(), f_t{0}); - mip::clamp_within_var_bounds(solution.assignment, &problem, &handle); - handle.sync_stream(); + mip::build_start_assignment(problem, solution, &handle); - // Built serially: the first climber host-copies the problem, the rest clone it. std::vector> preemption_flags(n_climbers); std::vector>> climbers(n_climbers); + // Composition and per-climber parameters come from build_climber_portfolio, which + // is editable. The log prefix is assigned here and not there, so every climber + // stays identifiable in the log whatever the portfolio does. + mip::build_climber_portfolio(problem, solution, preemption_flags, climbers, base_seed); for (int k = 0; k < n_climbers; ++k) { - preemption_flags[k].store(false); - mip::fj_settings_t settings; - settings.seed = (int)(base_seed + k); - if (k == 0) { - climbers[k] = mip::init_fj_cpu_standalone(problem, solution, preemption_flags[k], settings); - } else { - climbers[k] = mip::init_fj_cpu_standalone_from_template( - problem, *climbers[0], preemption_flags[k], settings); - } - - // Portfolio diversification, decorrelated from the value RNG. - std::mt19937 rng(base_seed + 7919u * k); - climbers[k]->mtm_viol_samples = std::uniform_int_distribution(15, 50)(rng); - climbers[k]->mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); - climbers[k]->nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); - climbers[k]->perturb_interval = std::uniform_int_distribution(50, 500)(rng); - climbers[k]->log_prefix = "[climber " + std::to_string(k) + "] "; + climbers[k]->log_prefix = "[climber " + std::to_string(k) + "] "; } const std::vector cpus = allowed_cpus(); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 41d456d245..5c48cab1e3 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -2657,4 +2657,51 @@ template void finalize_fj_cpu_host_initialization( const typename mip_solver_settings_t::tolerances_t& tolerances); #endif +// Portfolio construction for the standalone benchmark. Host logic, but it lives +// in a .cu because fj_cpu.cuh pulls in raft/util/cuda_dev_essentials.cuh through +// solution.cuh, which does not compile under the host compiler. Kept out of the +// header regardless: editing this file rebuilds one translation unit rather than +// the fifteen that including headers pull in. +template +void build_climber_portfolio(problem_t& problem, + solution_t& solution, + std::vector>& preemption_flags, + std::vector>>& climbers, + int64_t base_seed) +{ + const int n_climbers = static_cast(climbers.size()); + for (int k = 0; k < n_climbers; ++k) { + preemption_flags[k].store(false); + fj_settings_t settings; + settings.seed = (int)(base_seed + k); + // Built serially: the first climber host-copies the problem, the rest clone it. + if (k == 0) { + climbers[k] = init_fj_cpu_standalone(problem, solution, preemption_flags[k], settings); + } else { + climbers[k] = + init_fj_cpu_standalone_from_template(problem, *climbers[0], preemption_flags[k], settings); + } + + // Default: every climber identical apart from its seed and a random draw of the + // four sampling parameters. Diversification, decorrelated from the value RNG. + std::mt19937 rng(base_seed + 7919u * k); + climbers[k]->mtm_viol_samples = std::uniform_int_distribution(15, 50)(rng); + climbers[k]->mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); + climbers[k]->nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); + climbers[k]->perturb_interval = std::uniform_int_distribution(50, 500)(rng); + } +} + +#if MIP_INSTANTIATE_FLOAT +template void build_climber_portfolio( + problem_t&, solution_t&, std::vector>&, + std::vector>>&, int64_t); +#endif + +#if MIP_INSTANTIATE_DOUBLE +template void build_climber_portfolio( + problem_t&, solution_t&, std::vector>&, + std::vector>>&, int64_t); +#endif + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 37505682c9..53de8ba2d4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -313,6 +313,18 @@ std::unique_ptr> init_fj_cpu_standalone_from_template std::atomic& preemption_flag, fj_settings_t settings = fj_settings_t{}); +// Builds the climber portfolio the standalone benchmark races: how many distinct +// behaviours, what parameters each gets, whether they are randomized or +// specialized. Defined in fj_cpu_portfolio.cpp -- host code, compiled by the host +// compiler, so editing it is markedly cheaper than editing this header. Runs +// inside the measured window. +template +void build_climber_portfolio(problem_t& problem, + solution_t& solution, + std::vector>& preemption_flags, + std::vector>>& climbers, + int64_t base_seed); + template std::unique_ptr> init_fj_cpu_from_optimization_problem( const optimization_problem_t& problem, diff --git a/cpp/src/mip_heuristics/solution/solution.cu b/cpp/src/mip_heuristics/solution/solution.cu index 3b00fca7a8..197db0627c 100644 --- a/cpp/src/mip_heuristics/solution/solution.cu +++ b/cpp/src/mip_heuristics/solution/solution.cu @@ -5,6 +5,7 @@ */ /* clang-format on */ +#include #include "feasibility_test.cuh" #include "solution.cuh" #include "solution_kernels.cuh" @@ -652,4 +653,29 @@ template class solution_t; template class solution_t; #endif +template +void build_start_assignment(problem_t& problem, + solution_t& solution, + const raft::handle_t* handle_ptr) +{ + // Default: zero, projected into the variable bounds. Deliberately the simplest + // thing that works -- the seeding strategy is what this hook exists to change. + thrust::fill(handle_ptr->get_thrust_policy(), + solution.assignment.begin(), + solution.assignment.end(), + f_t{0}); + clamp_within_var_bounds(solution.assignment, &problem, handle_ptr); + handle_ptr->sync_stream(); +} + +#if MIP_INSTANTIATE_FLOAT +template void build_start_assignment( + problem_t&, solution_t&, const raft::handle_t*); +#endif + +#if MIP_INSTANTIATE_DOUBLE +template void build_start_assignment( + problem_t&, solution_t&, const raft::handle_t*); +#endif + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/solution/solution.cuh b/cpp/src/mip_heuristics/solution/solution.cuh index f243937d3e..4839ba1fb8 100644 --- a/cpp/src/mip_heuristics/solution/solution.cuh +++ b/cpp/src/mip_heuristics/solution/solution.cuh @@ -153,4 +153,13 @@ class solution_t { void test_variable_bounds(bool check_integer = true, i_t* is_feasible = nullptr); }; +// Builds the start assignment every climber is derived from. Defined in +// solution.cu, so editing it recompiles one translation unit rather than every +// file that includes this header. Runs inside the measured window: a better start +// has to be worth what it costs to build. +template +void build_start_assignment(problem_t& problem, + solution_t& solution, + const raft::handle_t* handle_ptr); + } // namespace cuopt::mathematical_optimization::mip From 25e4479b413f18a8ed8e7d9bbf9afcf1f21bf91f Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 07:04:48 -0700 Subject: [PATCH 25/61] replace operator overloaded arithmetic with explicit highway functions, + some fusing --- .../fj_cpu_binary_kernels.cpp | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index 915a98f9d4..f140041ac6 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -98,7 +98,8 @@ int32_t WalkRowsImpl(int32_t* HWY_RESTRICT row_slack, const V cmax = hn::PromoteTo(d, hn::LoadU(dc, incident_row_cmax + ii)); const V os = hn::MaskedGatherIndex(active, d, row_slack, rows); - const V ns = hn::Sub(os, hn::Mul(skv, vdelta)); + // os - skv * vdelta + const V ns = hn::NegMulAdd(skv, vdelta, os); // Only the satisfied side. deep_viol is the caller's business: it fires on 0.02% of visits but // guards the widest rows in the matrix, so it belongs where the row length is already known. @@ -178,9 +179,8 @@ static HWY_INLINE void PatchRowBody(D d, int32_t os_new, int32_t skip_var) { - const hn::Rebind dc; // same lane count, narrower lanes + const hn::Rebind dc; // same lane count, narrower lanes const hn::Repartition dw; // half the lanes, twice as wide: the packed score - const hn::Half dh; // int32 half, the source of each promotion using V = hn::Vec; using VW = hn::Vec; const size_t N = hn::Lanes(d); @@ -201,9 +201,10 @@ static HWY_INLINE void PatchRowBody(D d, const V vos = hn::Set(d, os_new); const V vw = hn::Set(d, weight), vw2 = hn::Set(d, weight / 2); - // The row's own slack is uniform across lanes, so its flags are scalars. + // The row's own slack is uniform across lanes, so its flags are scalars. Broadcast negated to + // match the new-state flags below, which come from VecFromMask and are 0 or -1. const int32_t osat = os_new >= 0, ost = os_new > 0; - const V vosat = hn::Set(d, osat), vost = hn::Set(d, ost); + const V vneg_osat = hn::Set(d, -osat), vneg_ost = hn::Set(d, -ost); const V v_not_osat = hn::Set(d, 1 - osat); // The loads always run unmasked and read into the per-nnz padding; when the remainder is masked, @@ -222,19 +223,27 @@ static HWY_INLINE void PatchRowBody(D d, // which cannot store-to-load forward, and that cost 959 interlocks per iteration against 72. // The score update escapes this because it already needs the spill for its read-modify-write. const V a01 = hn::MaskedGatherIndex(active, d, assign_i32, v); - const V flip = vone - hn::ShiftLeft<1>(a01); + const V flip = hn::Sub(vone, hn::ShiftLeft<1>(a01)); const V coef = hn::PromoteTo(d, hn::LoadU(dc, coefficients + k)); - const V ns = vos - coef * flip; + // vos - coef * flip + const V ns = hn::NegMulAdd(coef, flip, vos); - const V nsat = hn::IfThenElseZero(hn::Ge(ns, vzero), vone); - const V nst = hn::IfThenElseZero(hn::Gt(ns, vzero), vone); + // -(ns >= 0) + const V nsat_neg = hn::VecFromMask(d, hn::Ge(ns, vzero)); + // -(ns > 0) + const V nst_neg = hn::VecFromMask(d, hn::Gt(ns, vzero)); + // (ns > vos) - (ns < vos) const V improving = - hn::IfThenElseZero(hn::Gt(ns, vos), vone) - hn::IfThenElseZero(hn::Lt(ns, vos), vone); + hn::Sub(hn::VecFromMask(d, hn::Lt(ns, vos)), hn::VecFromMask(d, hn::Gt(ns, vos))); - const V both_violated = v_not_osat * (vone - nsat); - const V base = vw * (nsat - vosat) + both_violated * improving * vw2; - const V bonus = vw * (nst - vost); + // (1 - osat) * (1 - nsat) + const V both_violated = hn::Mul(v_not_osat, hn::Add(vone, nsat_neg)); + // vw * (nsat - osat) + both_violated * improving * vw2 + const V base = + hn::MulAdd(vw, hn::Sub(vneg_osat, nsat_neg), hn::Mul(hn::Mul(both_violated, improving), vw2)); + // vw * (nst - ost) + const V bonus = hn::Mul(vw, hn::Sub(vneg_ost, nst_neg)); // The score is int64, so packing it costs two vectors where the fields took one. Both fields // are per-row here and fit int32, so they are computed at full lane count above and widened @@ -242,23 +251,23 @@ static HWY_INLINE void PatchRowBody(D d, // difference and the store back. What reaches the scalar loop is one add per nonzero, which is // what it was before the score widened -- that loop is 38% of all cycles, so work belongs // anywhere but there. - const VW base_lo = hn::PromoteTo(dw, hn::LowerHalf(dh, base)); - const VW base_hi = hn::PromoteTo(dw, hn::UpperHalf(dh, base)); - const VW bonus_lo = hn::PromoteTo(dw, hn::LowerHalf(dh, bonus)); - const VW bonus_hi = hn::PromoteTo(dw, hn::UpperHalf(dh, bonus)); + const VW base_lo = hn::PromoteLowerTo(dw, base); + const VW base_hi = hn::PromoteUpperTo(dw, base); + const VW bonus_lo = hn::PromoteLowerTo(dw, bonus); + const VW bonus_hi = hn::PromoteUpperTo(dw, bonus); - const VW packed_lo = hn::ShiftLeft(base_lo) + bonus_lo; - const VW packed_hi = hn::ShiftLeft(base_hi) + bonus_hi; + const VW packed_lo = hn::Add(hn::ShiftLeft(base_lo), bonus_lo); + const VW packed_hi = hn::Add(hn::ShiftLeft(base_hi), bonus_hi); - const VW delta_lo = packed_lo - hn::LoadU(dw, nnz_score_delta + k); - const VW delta_hi = packed_hi - hn::LoadU(dw, nnz_score_delta + k + NW); + const VW delta_lo = hn::Sub(packed_lo, hn::LoadU(dw, nnz_score_delta + k)); + const VW delta_hi = hn::Sub(packed_hi, hn::LoadU(dw, nnz_score_delta + k + NW)); // The store mask is rebuilt at int64 width rather than narrowed from `active`: the same two // conditions, on the promoted indices. FirstN is applied on every target because where the // remainder is peeled the body never runs short, so it is all-true there anyway. const size_t rem = (size_t)(ke - k); - const VW v_lo = hn::PromoteTo(dw, hn::LowerHalf(dh, v)); - const VW v_hi = hn::PromoteTo(dw, hn::UpperHalf(dh, v)); + const VW v_lo = hn::PromoteLowerTo(dw, v); + const VW v_hi = hn::PromoteUpperTo(dw, v); const VW vskip_w = hn::Set(dw, skip_var); const auto act_lo = hn::And(hn::Ne(v_lo, vskip_w), hn::FirstN(dw, rem)); const auto act_hi = hn::And(hn::Ne(v_hi, vskip_w), hn::FirstN(dw, rem > NW ? rem - NW : 0)); @@ -285,8 +294,8 @@ static HWY_INLINE void PatchRowBody(D d, // indices, in the two halves the pack already produced. const VW cur_lo = hn::MaskedGatherIndex(act_lo, dw, var_score, v_lo); const VW cur_hi = hn::MaskedGatherIndex(act_hi, dw, var_score, v_hi); - hn::MaskedScatterIndex(cur_lo + delta_lo, act_lo, dw, var_score, v_lo); - hn::MaskedScatterIndex(cur_hi + delta_hi, act_hi, dw, var_score, v_hi); + hn::MaskedScatterIndex(hn::Add(cur_lo, delta_lo), act_lo, dw, var_score, v_lo); + hn::MaskedScatterIndex(hn::Add(cur_hi, delta_hi), act_hi, dw, var_score, v_hi); #endif } From 71d07a612b4969de7571cf1391bbb6523d5ddb66 Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 08:21:03 -0700 Subject: [PATCH 26/61] save user_callbacks in run_mip --- .../linear_programming/cuopt/run_mip.cpp | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/benchmarks/linear_programming/cuopt/run_mip.cpp b/benchmarks/linear_programming/cuopt/run_mip.cpp index 98cd9a56d2..207c553dcb 100644 --- a/benchmarks/linear_programming/cuopt/run_mip.cpp +++ b/benchmarks/linear_programming/cuopt/run_mip.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -136,6 +137,52 @@ std::vector> read_solution_from_dir(const std::string file_p return initial_solutions; } +struct incumbent_record_t { + double objective; + double work_timestamp; + double wall_time; +}; + +class incumbent_tracker_t : public cuopt::internals::get_solution_callback_t { + public: + explicit incumbent_tracker_t(std::chrono::high_resolution_clock::time_point start_time) + : start_time_(start_time) + { + } + + void get_solution(void* /*data*/, + void* cost, + void* /*solution_bound*/, + void* /*user_data*/) override + { + const auto now = std::chrono::high_resolution_clock::now(); + records_.push_back({*static_cast(cost), + 0.0, + std::chrono::duration(now - start_time_).count()}); + } + + void write_csv(const std::string& path) const + { + std::ofstream file(path); + if (!file.is_open()) { + std::cerr << "Error opening incumbent file " << path << std::endl; + return; + } + file << "index,objective,work_timestamp,wall_time_s\n"; + for (size_t i = 0; i < records_.size(); ++i) { + file << i << "," << std::setprecision(15) << records_[i].objective << "," + << records_[i].work_timestamp << "," << std::setprecision(6) << records_[i].wall_time + << "\n"; + } + } + + size_t size() const { return records_.size(); } + + private: + std::chrono::high_resolution_clock::time_point start_time_; + std::vector records_; +}; + int run_single_file(std::string file_path, int device, int batch_id, @@ -218,6 +265,8 @@ int run_single_file(std::string file_path, cuopt::mathematical_optimization::benchmark_info_t benchmark_info; settings.benchmark_info_ptr = &benchmark_info; auto start_run_solver = std::chrono::high_resolution_clock::now(); + incumbent_tracker_t incumbent_tracker(start_run_solver); + settings.set_mip_callback(&incumbent_tracker); auto solution = cuopt::mathematical_optimization::solve_mip(&handle_, mps_data_model, settings); CUOPT_LOG_INFO( "first obj: %f last improvement of best feasible: %f last improvement after recombination: %f", @@ -291,6 +340,13 @@ int run_single_file(std::string file_path, << "\n"; write_to_output_file(out_dir, base_filename, device, n_gpus, batch_id, ss.str()); CUOPT_LOG_INFO("Results written to the file %s", base_filename.c_str()); + if (out_dir != "") { + std::string csv_path = + out_dir + "/" + base_filename.substr(0, base_filename.find(".mps")) + "_incumbents.csv"; + incumbent_tracker.write_csv(csv_path); + CUOPT_LOG_INFO( + "Incumbent trace (%zu entries) written to %s", incumbent_tracker.size(), csv_path.c_str()); + } return sol_found; } From 27bae2f163c8e51f083e14111e764337a9f80df6 Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 09:08:17 -0700 Subject: [PATCH 27/61] let B&B publih solutions via the user callbacks as well to reduce latency --- cpp/src/branch_and_bound/branch_and_bound.cpp | 16 +- cpp/src/branch_and_bound/branch_and_bound.hpp | 8 +- .../mip_heuristics/diversity/population.cu | 50 +------ .../mip_heuristics/diversity/population.cuh | 3 - .../mip_heuristics/problem/presolve_data.cu | 15 +- .../mip_heuristics/problem/presolve_data.cuh | 4 +- cpp/src/mip_heuristics/problem/problem.cu | 5 +- cpp/src/mip_heuristics/problem/problem.cuh | 7 +- .../mip_heuristics/solution_publication.cuh | 141 ++++++++++++++++++ cpp/src/mip_heuristics/solver.cu | 36 ++--- cpp/src/mip_heuristics/solver_context.cuh | 3 + 11 files changed, 201 insertions(+), 87 deletions(-) create mode 100644 cpp/src/mip_heuristics/solution_publication.cuh diff --git a/cpp/src/branch_and_bound/branch_and_bound.cpp b/cpp/src/branch_and_bound/branch_and_bound.cpp index 29174148b7..c532e37918 100644 --- a/cpp/src/branch_and_bound/branch_and_bound.cpp +++ b/cpp/src/branch_and_bound/branch_and_bound.cpp @@ -524,7 +524,8 @@ void branch_and_bound_t::update_user_bound(f_t lower_bound) template bool branch_and_bound_t::set_solution_from_heuristics(const std::vector& solution, - heuristics_origin_t origin) + heuristics_origin_t origin, + f_t* solver_objective) { mutex_original_lp_.lock(); if (solution.size() != original_problem_.num_cols) { @@ -537,6 +538,7 @@ bool branch_and_bound_t::set_solution_from_heuristics(const std::vecto f_t obj = compute_objective(original_lp_, crushed_solution); mutex_original_lp_.unlock(); + if (solver_objective != nullptr) { *solver_objective = obj; } bool is_feasible = false; bool attempt_repair = false; bool success = false; @@ -687,10 +689,18 @@ void branch_and_bound_t::set_solution_from_submip( log_prefix, compute_user_objective(lp, obj)); - bool success = set_solution_from_heuristics(user_sol, heuristics_origin_t::SUBMIP); + // `obj` is in the sub-MIP's own space (fixed variables, own presolve offset), so it cannot be + // handed to solution_callback alongside a user-space assignment. + f_t original_lp_objective = std::numeric_limits::quiet_NaN(); + bool success = + set_solution_from_heuristics(user_sol, heuristics_origin_t::SUBMIP, &original_lp_objective); if (success) { submip_stats.save_success(fixrate); - if (settings_.solution_callback != nullptr) { settings_.solution_callback(user_sol, obj); } + cuopt_assert(std::isfinite(original_lp_objective), + "SubMIP incumbent objective must be finite when accepted"); + if (settings_.solution_callback != nullptr) { + settings_.solution_callback(user_sol, original_lp_objective); + } } } diff --git a/cpp/src/branch_and_bound/branch_and_bound.hpp b/cpp/src/branch_and_bound/branch_and_bound.hpp index 7cf5ed3680..5b95195bb5 100644 --- a/cpp/src/branch_and_bound/branch_and_bound.hpp +++ b/cpp/src/branch_and_bound/branch_and_bound.hpp @@ -132,8 +132,12 @@ class branch_and_bound_t { } } - // Set a solution based on the user problem during the course of the solve - bool set_solution_from_heuristics(const std::vector& solution, heuristics_origin_t origin); + // Set a solution based on the user problem during the course of the solve. + // When non-null, `solver_objective` receives the objective of `solution` in original_lp_ space, + // which is the space every solution_callback consumer expects. + bool set_solution_from_heuristics(const std::vector& solution, + heuristics_origin_t origin, + f_t* solver_objective = nullptr); // Apply a solution found by a CPU FJ worker. void set_solution_from_cpu_fj(f_t obj, const std::vector& assignment, double work_units); diff --git a/cpp/src/mip_heuristics/diversity/population.cu b/cpp/src/mip_heuristics/diversity/population.cu index e6fffa97b2..033119915f 100644 --- a/cpp/src/mip_heuristics/diversity/population.cu +++ b/cpp/src/mip_heuristics/diversity/population.cu @@ -264,41 +264,6 @@ bool population_t::is_better_than_best_feasible(solution_t& return obj_better && sol.get_feasible(); } -template -void population_t::invoke_get_solution_callback( - solution_t& sol, internals::get_solution_callback_t* callback) -{ - f_t user_objective = sol.get_user_objective(); - f_t user_bound = context.stats.get_solution_bound(); - solution_t temp_sol(sol); - problem_ptr->post_process_assignment(temp_sol.assignment); - if (problem_ptr->has_papilo_presolve_data()) { - problem_ptr->papilo_uncrush_assignment(temp_sol.assignment); - } - - std::vector user_objective_vec(1); - std::vector user_bound_vec(1); - std::vector user_assignment_vec(temp_sol.assignment.size()); - user_objective_vec[0] = user_objective; - user_bound_vec[0] = user_bound; - raft::copy(user_assignment_vec.data(), - temp_sol.assignment.data(), - temp_sol.assignment.size(), - temp_sol.handle_ptr->get_stream()); - temp_sol.handle_ptr->sync_stream(); - if (mip_solver_settings_accessor::has_semi_continuous_callback_translation( - context.settings)) { - mip::strip_semi_continuous_auxiliaries_from_assignment( - user_assignment_vec, - mip_solver_settings_accessor::get_semi_continuous_original_num_variables( - context.settings)); - } - callback->get_solution(user_assignment_vec.data(), - user_objective_vec.data(), - user_bound_vec.data(), - callback->get_user_data()); -} - template void population_t::run_solution_callbacks(solution_t& sol) { @@ -309,15 +274,14 @@ void population_t::run_solution_callbacks(solution_t& sol) context.settings.benchmark_info_ptr->last_improvement_of_best_feasible = timer.elapsed_time(); } CUOPT_LOG_DEBUG("Population: Found new best solution %g", sol.get_user_objective()); - if (problem_ptr->branch_and_bound_callback != nullptr) { - problem_ptr->branch_and_bound_callback(sol.get_host_assignment(), - heuristics_origin_t::HEURISTICS); - } - for (auto callback : user_callbacks) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - invoke_get_solution_callback(sol, get_sol_callback); + if (problem_ptr->branch_and_bound_callback != nullptr || + context.solution_publication.enabled()) { + auto host_assignment = sol.get_host_assignment(); + if (problem_ptr->branch_and_bound_callback != nullptr) { + problem_ptr->branch_and_bound_callback(host_assignment, heuristics_origin_t::HEURISTICS); } + context.solution_publication.publish_if_better( + problem_ptr, host_assignment, sol.get_objective()); } // Save the best objective here even if callback handling later exits early. // This prevents older solutions from being reported as "new best" in subsequent callbacks. diff --git a/cpp/src/mip_heuristics/diversity/population.cuh b/cpp/src/mip_heuristics/diversity/population.cuh index 593b1ddf1e..5a9db26928 100644 --- a/cpp/src/mip_heuristics/diversity/population.cuh +++ b/cpp/src/mip_heuristics/diversity/population.cuh @@ -160,9 +160,6 @@ class population_t { void diversity_step(i_t max_iterations_without_improvement); - void invoke_get_solution_callback(solution_t& sol, - internals::get_solution_callback_t* callback); - // does some consistency tests bool test_invariant(); diff --git a/cpp/src/mip_heuristics/problem/presolve_data.cu b/cpp/src/mip_heuristics/problem/presolve_data.cu index 3c621bc2cd..ae78c3778c 100644 --- a/cpp/src/mip_heuristics/problem/presolve_data.cu +++ b/cpp/src/mip_heuristics/problem/presolve_data.cu @@ -257,8 +257,8 @@ void presolve_data_t::set_papilo_presolve_data( } template -void presolve_data_t::papilo_uncrush_assignment( - problem_t& problem, rmm::device_uvector& assignment) const +void presolve_data_t::papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const { if (papilo_presolve_ptr == nullptr) { CUOPT_LOG_INFO("Papilo presolve data not set, skipping uncrushing assignment"); @@ -266,15 +266,12 @@ void presolve_data_t::papilo_uncrush_assignment( } cuopt_assert(assignment.size() == papilo_reduced_to_original_map.size(), "Papilo uncrush assignment size mismatch"); - auto h_assignment = cuopt::host_copy(assignment, problem.handle_ptr->get_stream()); + auto h_assignment = cuopt::host_copy(assignment, stream); std::vector full_assignment; papilo_presolve_ptr->uncrush_primal_solution(h_assignment, full_assignment); - assignment.resize(full_assignment.size(), problem.handle_ptr->get_stream()); - raft::copy(assignment.data(), - full_assignment.data(), - full_assignment.size(), - problem.handle_ptr->get_stream()); - problem.handle_ptr->sync_stream(); + assignment.resize(full_assignment.size(), stream); + raft::copy(assignment.data(), full_assignment.data(), full_assignment.size(), stream); + stream.synchronize(); } #if MIP_INSTANTIATE_FLOAT || PDLP_INSTANTIATE_FLOAT diff --git a/cpp/src/mip_heuristics/problem/presolve_data.cuh b/cpp/src/mip_heuristics/problem/presolve_data.cuh index 713bb24c0d..65b492a7e6 100644 --- a/cpp/src/mip_heuristics/problem/presolve_data.cuh +++ b/cpp/src/mip_heuristics/problem/presolve_data.cuh @@ -122,8 +122,8 @@ class presolve_data_t { i_t original_num_variables); bool has_papilo_presolve_data() const { return papilo_presolve_ptr != nullptr; } i_t get_papilo_original_num_variables() const { return papilo_original_num_variables; } - void papilo_uncrush_assignment(problem_t& problem, - rmm::device_uvector& assignment) const; + void papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const; presolve_data_t(presolve_data_t&&) = default; presolve_data_t& operator=(presolve_data_t&&) = default; diff --git a/cpp/src/mip_heuristics/problem/problem.cu b/cpp/src/mip_heuristics/problem/problem.cu index 17d55b7cc2..b84206e08f 100644 --- a/cpp/src/mip_heuristics/problem/problem.cu +++ b/cpp/src/mip_heuristics/problem/problem.cu @@ -2192,9 +2192,10 @@ void problem_t::set_papilo_presolve_data( } template -void problem_t::papilo_uncrush_assignment(rmm::device_uvector& assignment) const +void problem_t::papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const { - presolve_data.papilo_uncrush_assignment(const_cast(*this), assignment); + presolve_data.papilo_uncrush_assignment(assignment, stream); } template diff --git a/cpp/src/mip_heuristics/problem/problem.cuh b/cpp/src/mip_heuristics/problem/problem.cuh index 3ea3973d1d..fd38117b50 100644 --- a/cpp/src/mip_heuristics/problem/problem.cuh +++ b/cpp/src/mip_heuristics/problem/problem.cuh @@ -119,7 +119,12 @@ class problem_t { { return presolve_data.get_papilo_original_num_variables(); } - void papilo_uncrush_assignment(rmm::device_uvector& assignment) const; + void papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const; + void papilo_uncrush_assignment(rmm::device_uvector& assignment) const + { + papilo_uncrush_assignment(assignment, handle_ptr->get_stream()); + } void compute_transpose_of_problem(); f_t get_user_obj_from_solver_obj(f_t solver_obj) const; f_t get_solver_obj_from_user_obj(f_t user_obj) const; diff --git a/cpp/src/mip_heuristics/solution_publication.cuh b/cpp/src/mip_heuristics/solution_publication.cuh new file mode 100644 index 0000000000..0e5c1d92ea --- /dev/null +++ b/cpp/src/mip_heuristics/solution_publication.cuh @@ -0,0 +1,141 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace cuopt::mathematical_optimization::mip { + +// Single point at which MIP incumbents are reported to the user get-solution callbacks. +// The heuristic thread (through the population) and the branch-and-bound thread both publish +// here, so the guard on the last published objective is shared and every incumbent is reported +// once, at the moment it is found rather than when the heuristic thread next drains its queue. +template +class solution_publication_t { + public: + solution_publication_t(const mip_solver_settings_t& settings, + const solver_stats_t& stats) + : settings_(settings), stats_(stats) + { + if (has_get_solution_callback()) { + RAFT_CUDA_TRY(cudaGetDevice(&device_id_)); + handle_ = std::make_unique(); + } + } + + // Whether any get-solution callback is registered. Callers can use this to skip assembling + // the host assignment that publish_if_better would otherwise discard. + bool enabled() const { return handle_ != nullptr; } + + // `assignment` and `solver_objective` are in problem_ptr's solver space, which is always + // oriented as a minimization. Returns whether the incumbent was published. + // + // Post-processing runs on a private stream, so this is safe to call from the branch-and-bound + // thread while the heuristic thread owns problem_ptr->handle_ptr's stream. + bool publish_if_better(problem_t* problem_ptr, + const std::vector& assignment, + f_t solver_objective) + { + if (handle_ == nullptr) { return false; } + cuopt_assert(problem_ptr != nullptr, "Publication problem pointer must not be null"); + cuopt_assert(std::isfinite(solver_objective), "Published objective must be finite"); + + std::lock_guard lock(mutex_); + if (!(solver_objective < best_published_objective_)) { return false; } + best_published_objective_ = solver_objective; + + const auto user_assignment = build_user_assignment(problem_ptr, assignment); + const f_t user_objective = problem_ptr->get_user_obj_from_solver_obj(solver_objective); + const f_t user_bound = stats_.get_solution_bound(); + CUOPT_LOG_DEBUG("Publishing incumbent: objective %g, %lu variables", + user_objective, + user_assignment.size()); + + for (auto callback : settings_.get_mip_callbacks()) { + if (callback == nullptr || + callback->get_type() != internals::base_solution_callback_type::GET_SOLUTION) { + continue; + } + // Each callback gets its own copies: the interface hands out mutable pointers. + std::vector callback_assignment(user_assignment); + std::vector callback_objective(1, user_objective); + std::vector callback_bound(1, user_bound); + auto get_sol_callback = static_cast(callback); + get_sol_callback->get_solution(callback_assignment.data(), + callback_objective.data(), + callback_bound.data(), + get_sol_callback->get_user_data()); + } + return true; + } + + private: + // Lifts a solver-space assignment into the space the callbacks were set up for. + std::vector build_user_assignment(problem_t* problem_ptr, + const std::vector& assignment) + { + // The B&B thread may never have selected a device of its own. + RAFT_CUDA_TRY(cudaSetDevice(device_id_)); + auto stream = handle_->get_stream(); + rmm::device_uvector d_assignment(assignment.size(), stream); + raft::copy(d_assignment.data(), assignment.data(), assignment.size(), stream); + // post_process_assignment writes through problem_ptr->presolve_data.fixed_var_assignment, + // which both publishing threads share: the caller's lock is what keeps them apart. + problem_ptr->post_process_assignment(d_assignment, true, stream); + if (problem_ptr->has_papilo_presolve_data()) { + problem_ptr->papilo_uncrush_assignment(d_assignment, stream); + } + auto user_assignment = cuopt::host_copy(d_assignment, stream); + if (mip_solver_settings_accessor::has_semi_continuous_callback_translation( + settings_)) { + strip_semi_continuous_auxiliaries_from_assignment( + user_assignment, + mip_solver_settings_accessor::get_semi_continuous_original_num_variables( + settings_)); + } + return user_assignment; + } + + bool has_get_solution_callback() const + { + for (auto callback : settings_.get_mip_callbacks()) { + if (callback != nullptr && + callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { + return true; + } + } + return false; + } + + const mip_solver_settings_t& settings_; + const solver_stats_t& stats_; + int device_id_{0}; + // Null when no get-solution callback is registered, which also disables publication. + std::unique_ptr handle_; + std::mutex mutex_; + f_t best_published_objective_{std::numeric_limits::max()}; +}; + +} // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/solver.cu b/cpp/src/mip_heuristics/solver.cu index f8eac0c4d8..f0a5a3c5aa 100644 --- a/cpp/src/mip_heuristics/solver.cu +++ b/cpp/src/mip_heuristics/solver.cu @@ -69,6 +69,10 @@ struct branch_and_bound_solution_helper_t { void solution_callback(std::vector& solution, f_t objective) { + if (dm->context.settings.determinism_mode == CUOPT_MODE_OPPORTUNISTIC) { + dm->context.solution_publication.publish_if_better( + dm->context.problem_ptr, solution, objective); + } dm->population.add_external_solution(solution, objective, solution_origin_t::BRANCH_AND_BOUND); } @@ -197,12 +201,8 @@ solution_t mip_solver_t::run_solver() if (context.problem_ptr->empty) { CUOPT_LOG_INFO("Problem fully reduced in presolve"); sol.set_problem_fully_reduced(); - for (auto callback : context.settings.get_mip_callbacks()) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - dm.population.invoke_get_solution_callback(sol, get_sol_callback); - } - } + context.solution_publication.publish_if_better( + context.problem_ptr, sol.get_host_assignment(), sol.get_objective()); context.problem_ptr->post_process_solution(sol); return sol; } @@ -237,12 +237,8 @@ solution_t mip_solver_t::run_solver() if (run_presolve && context.problem_ptr->empty) { CUOPT_LOG_INFO("Problem full reduced in presolve"); sol.set_problem_fully_reduced(); - for (auto callback : context.settings.get_mip_callbacks()) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - dm.population.invoke_get_solution_callback(sol, get_sol_callback); - } - } + context.solution_publication.publish_if_better( + context.problem_ptr, sol.get_host_assignment(), sol.get_objective()); context.problem_ptr->post_process_solution(sol); return sol; } @@ -273,12 +269,8 @@ solution_t mip_solver_t::run_solver() sol.set_problem_fully_reduced(); } if (opt_sol.get_termination_status() == pdlp_termination_status_t::Optimal) { - for (auto callback : context.settings.get_mip_callbacks()) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - dm.population.invoke_get_solution_callback(sol, get_sol_callback); - } - } + context.solution_publication.publish_if_better( + context.problem_ptr, sol.get_host_assignment(), sol.get_objective()); } context.problem_ptr->post_process_solution(sol); return sol; @@ -445,10 +437,10 @@ solution_t mip_solver_t::run_solver() branch_and_bound->set_concurrent_lp_root_solve(true); context.problem_ptr->branch_and_bound_callback = - std::bind(&mip::branch_and_bound_t::set_solution_from_heuristics, - branch_and_bound.get(), - std::placeholders::_1, - std::placeholders::_2); + [bb = branch_and_bound.get()](const std::vector& solution, + heuristics_origin_t origin) { + return bb->set_solution_from_heuristics(solution, origin); + }; } else if (context.settings.determinism_mode == CUOPT_MODE_DETERMINISTIC) { branch_and_bound->set_concurrent_lp_root_solve(false); // TODO once deterministic GPU heuristics are integrated diff --git a/cpp/src/mip_heuristics/solver_context.cuh b/cpp/src/mip_heuristics/solver_context.cuh index f98386cbaf..344d4e8d86 100644 --- a/cpp/src/mip_heuristics/solver_context.cuh +++ b/cpp/src/mip_heuristics/solver_context.cuh @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -58,6 +59,8 @@ struct mip_solver_context_t { std::atomic preempt_heuristic_solver_ = false; const mip_solver_settings_t settings; solver_stats_t stats; + // Every incumbent reported to the user goes through here, from whichever thread found it. + solution_publication_t solution_publication{settings, stats}; // Work limit context for tracking work units in deterministic mode (shared across all timers in // GPU heuristic loop) work_limit_context_t gpu_heur_loop{"GPUHeur"}; From 8b26bcdf9401b10f95d7e5c4340f7b7d7a300d3c Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 09:19:33 -0700 Subject: [PATCH 28/61] publish cpufj scratch solutions early as well --- cpp/src/mip_heuristics/local_search/local_search.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/src/mip_heuristics/local_search/local_search.cu b/cpp/src/mip_heuristics/local_search/local_search.cu index 23edf555cd..c1d80fcda7 100644 --- a/cpp/src/mip_heuristics/local_search/local_search.cu +++ b/cpp/src/mip_heuristics/local_search/local_search.cu @@ -79,6 +79,7 @@ void local_search_t::start_cpufj_scratch_threads(population_timprovement_callback = [this, &population, problem_ptr = context.problem_ptr]( f_t obj, const std::vector& h_vec, double /*work_units*/) { + context.solution_publication.publish_if_better(problem_ptr, h_vec, obj); population.add_external_solution(h_vec, obj, solution_origin_t::CPUFJ); (void)problem_ptr; if (obj < this->local_search_best_obj) { @@ -127,6 +128,7 @@ void local_search_t::start_cpufj_lptopt_scratch_threads( scratch_cpu_fj_on_lp_opt->log_prefix = "******* scratch on LP optimal: "; scratch_cpu_fj_on_lp_opt->improvement_callback = [this, &population](f_t obj, const std::vector& h_vec, double /*work_units*/) { + context.solution_publication.publish_if_better(context.problem_ptr, h_vec, obj); population.add_external_solution(h_vec, obj, solution_origin_t::CPUFJ); if (obj < this->local_search_best_obj) { CUOPT_LOG_DEBUG("******* New local search best obj %g, best overall %g", From abdad3e265f4ee338d5bcf369c86c9ad3e09548d Mon Sep 17 00:00:00 2001 From: yboucher Date: Thu, 20 Aug 2026 10:11:23 -0700 Subject: [PATCH 29/61] absorb cuda driver startup in run_mip.cpp --- benchmarks/linear_programming/cuopt/run_mip.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/linear_programming/cuopt/run_mip.cpp b/benchmarks/linear_programming/cuopt/run_mip.cpp index 207c553dcb..131ec15e6d 100644 --- a/benchmarks/linear_programming/cuopt/run_mip.cpp +++ b/benchmarks/linear_programming/cuopt/run_mip.cpp @@ -407,6 +407,8 @@ void return_gpu_to_the_queue(std::unordered_map& pid_gpu_map, int main(int argc, char* argv[]) { + (void)cudaFree(0); + argparse::ArgumentParser program("solve_MIP"); // Define all arguments with appropriate defaults and help messages From 04a25faca75eba230d722328ed121ffa932a4f2e Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 02:53:29 -0700 Subject: [PATCH 30/61] fix multigpu runs --- benchmarks/linear_programming/cuopt/run_mip.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/linear_programming/cuopt/run_mip.cpp b/benchmarks/linear_programming/cuopt/run_mip.cpp index 131ec15e6d..6a9a3303fd 100644 --- a/benchmarks/linear_programming/cuopt/run_mip.cpp +++ b/benchmarks/linear_programming/cuopt/run_mip.cpp @@ -198,6 +198,8 @@ int run_single_file(std::string file_path, double work_limit, bool deterministic) { + (void)cudaFree(0); + const raft::handle_t handle_{}; cuopt::mathematical_optimization::mip_solver_settings_t settings; std::string base_filename = file_path.substr(file_path.find_last_of("/\\") + 1); @@ -407,8 +409,6 @@ void return_gpu_to_the_queue(std::unordered_map& pid_gpu_map, int main(int argc, char* argv[]) { - (void)cudaFree(0); - argparse::ArgumentParser program("solve_MIP"); // Define all arguments with appropriate defaults and help messages From a977cf9cdc6d9b4511141ae5fa496fe84fbfca6d Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 05:28:14 -0700 Subject: [PATCH 31/61] lhs refresh period stretch, lazy mtm cache invalidate, magnitude aware objective term --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 88 +++++++++++++------ .../feasibility_jump/fj_cpu.cuh | 11 +++ .../feasibility_jump/fj_cpu_binary.cu | 23 ++++- 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 5c48cab1e3..7f07df475a 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -38,7 +38,7 @@ #include #include -#define CPUFJ_TIMING_TRACE 0 +#define CPUFJ_TIMING_TRACE 1 // Define CPUFJ_NVTX_RANGES to enable detailed NVTX profiling ranges #ifdef CPUFJ_NVTX_RANGES @@ -639,10 +639,18 @@ static inline std::pair compute_score(fj_cpu_climber_t 0) - base_obj = -fj_cpu.h_objective_weight; + if (fj_cpu.h_objective_weight > 0 && obj_diff != 0) { + // Scaling base by the objective magnitude only means something where there is feasibility + // impact to trade against; at base_feas_sum zero it would only make base distinct across + // candidates, which strands the bonus stage of the staged comparison. + f_t weighted = fj_cpu.h_objective_weight; + if (base_feas_sum != 0) { + cuopt_assert(fj_cpu.obj_magnitude > 0, "objective magnitude unit must be positive"); + weighted *= min((f_t)fj_obj_mult_max, + max((f_t)fj_obj_mult_min, fabs(obj_diff) / fj_cpu.obj_magnitude)); + } + base_obj = obj_diff < 0 ? weighted : -weighted; + } f_t bonus_breakthrough = 0; @@ -939,7 +947,8 @@ static void smooth_weights(fj_cpu_climber_t& fj_cpu) } if (fj_cpu.h_objective_weight > 0 && fj_cpu.h_incumbent_objective >= fj_cpu.h_best_objective) { - fj_cpu.h_objective_weight = max((f_t)0, fj_cpu.h_objective_weight - 1); + fj_cpu.h_objective_weight = + max(fj_cpu.seed_objective_weight, fj_cpu.h_objective_weight - 1); } } @@ -989,11 +998,7 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) } // Invalidate related cached move scores - auto [relvar_offset_begin, relvar_offset_end] = - range_for_constraint(fj_cpu, cstr_idx); - for (auto i = relvar_offset_begin; i < relvar_offset_end; i++) { - fj_cpu.cached_mtm_moves[i].first = 0; - } + fj_cpu.h_cstr_version[cstr_idx]++; } if (fj_cpu.violated_constraints.empty()) { fj_cpu.h_objective_weight += 1; } @@ -1072,11 +1077,7 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, cuopt_assert(isfinite(fj_cpu.h_lhs[cstr_idx]), "assignment should be finite"); // Invalidate related cached move scores - auto [relvar_offset_begin, relvar_offset_end] = - range_for_constraint(fj_cpu, cstr_idx); - for (auto i = relvar_offset_begin; i < relvar_offset_end; i++) { - fj_cpu.cached_mtm_moves[i].first = 0; - } + fj_cpu.h_cstr_version[cstr_idx]++; } if (previous_viol > 0 && fj_cpu.violated_constraints.empty()) { @@ -1171,7 +1172,11 @@ static thrust::tuple find_mtm_move( auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); for (auto i = offset_begin; i < offset_end; i++) { // early cached check - if (auto& cached_move = fj_cpu.cached_mtm_moves[i]; cached_move.first != 0) { + cuopt_assert(fj_cpu.cached_mtm_moves_version[i] <= fj_cpu.h_cstr_version[cstr_idx], + "cached move newer than its constraint"); + if (auto& cached_move = fj_cpu.cached_mtm_moves[i]; + cached_move.first != 0 && + fj_cpu.cached_mtm_moves_version[i] == fj_cpu.h_cstr_version[cstr_idx]) { if (best_score < cached_move.second) { auto var_idx = fj_cpu.h_variables[i]; if (check_variable_within_bounds( @@ -1241,8 +1246,9 @@ static thrust::tuple find_mtm_move( cuopt_assert(move.var_idx < fj_cpu.h_assignment.size(), "move.var_idx is out of bounds"); cuopt_assert(move.var_idx >= 0, "move.var_idx is not positive"); - auto [score, infeasibility] = compute_score(fj_cpu, var_idx, delta); - fj_cpu.cached_mtm_moves[i] = std::make_pair(delta, score); + auto [score, infeasibility] = compute_score(fj_cpu, var_idx, delta); + fj_cpu.cached_mtm_moves[i] = std::make_pair(delta, score); + fj_cpu.cached_mtm_moves_version[i] = fj_cpu.h_cstr_version[cstr_idx]; fj_cpu.miss_count++; // reject this move if it would increase the target variable to a numerically unstable value if (fj_cpu.view.move_numerically_stable( @@ -1389,6 +1395,7 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) fj_cpu.h_assignment.begin(), fj_cpu.h_assignment.end(), fj_cpu.h_obj_coeffs.begin(), 0.); } + template static thrust::tuple find_lift_move( fj_cpu_climber_t& fj_cpu) @@ -1563,6 +1570,13 @@ static void reset_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) fj_cpu.iters_since_infeasible_improve = 0; } +template +static void invalidate_mtm_cache(fj_cpu_climber_t& fj_cpu) +{ + for (size_t c = 0; c < fj_cpu.h_cstr_version.size(); ++c) + fj_cpu.h_cstr_version[c]++; +} + template static void restart_from_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) { @@ -1570,8 +1584,7 @@ static void restart_from_infeasible_checkpoint(fj_cpu_climber_t& fj_cp "incumbent_assignment span would be invalidated"); fj_cpu.h_assignment = fj_cpu.h_best_infeasible_assignment; recompute_lhs(fj_cpu); - for (size_t i = 0; i < fj_cpu.cached_mtm_moves.size(); ++i) - fj_cpu.cached_mtm_moves[i].first = 0; + invalidate_mtm_cache(fj_cpu); } template @@ -1824,6 +1837,8 @@ static void wire_fj_cpu_host_views( // nnz count fj_cpu.cached_mtm_moves.resize(fj_cpu.h_coefficients.size(), std::make_pair(0, fj_staged_score_t::zero())); + fj_cpu.cached_mtm_moves_version.assign(fj_cpu.h_coefficients.size(), -1); + fj_cpu.h_cstr_version.assign(n_constraints, 0); fj_cpu.flip_move_computed.resize(n_variables, false); fj_cpu.var_bitmap.resize(n_variables, false); @@ -1853,6 +1868,15 @@ void finalize_fj_cpu_host_initialization( fj_cpu.view.objective_vars = raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); + f_t abs_obj_sum = 0; + for (auto var_idx : fj_cpu.h_objective_vars) { + const f_t coeff = fj_cpu.h_obj_coeffs[var_idx]; + abs_obj_sum += coeff < 0 ? -coeff : coeff; + } + fj_cpu.obj_magnitude = abs_obj_sum > 0 ? abs_obj_sum / fj_cpu.h_objective_vars.size() : f_t{1}; + cuopt_assert(isfinite(fj_cpu.obj_magnitude) && fj_cpu.obj_magnitude > 0, + "objective magnitude unit must be finite and positive"); + fj_cpu.cached_cstr_bounds.resize(fj_cpu.h_reverse_coefficients.size()); for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); @@ -1912,6 +1936,7 @@ static void finalize_fj_cpu_host_initialization_from_template( fj_cpu.h_objective_vars = tmpl.h_objective_vars; fj_cpu.cached_cstr_bounds = tmpl.cached_cstr_bounds; + fj_cpu.obj_magnitude = tmpl.obj_magnitude; fj_cpu.h_lhs = tmpl.h_lhs; fj_cpu.h_lhs_sumcomp = tmpl.h_lhs_sumcomp; @@ -2092,6 +2117,8 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) cuopt_assert(fj_cpu.h_cstr_right_weights[cstr_idx] >= 0, "Weights should be positive or zero"); } cuopt_assert(fj_cpu.h_objective_weight >= 0, "Objective weight should be positive or zero"); + cuopt_assert(fj_cpu.seed_objective_weight >= 0, + "Objective weight floor should be positive or zero"); } template @@ -2123,6 +2150,9 @@ std::unique_ptr> fj_t::create_cpu_climber( return fj_cpu; // move } +constexpr int32_t fj_nnz_per_refresh_stretch = 100000; +constexpr int32_t fj_max_refresh_stretch = 8; + template void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double work_unit_limit) { @@ -2146,6 +2176,14 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w fj_cpu->restores_since_improvement = 0; fj_cpu->max_restores_since_improvement = 0; + // The recompute is O(nnz), so a fixed period costs a growing share of the budget. + cuopt_assert(fj_cpu->settings.parameters.lhs_refresh_period > 0, + "lhs_refresh_period should be positive"); + const i_t nnz_stretch = std::min( + (i_t)fj_cpu->h_coefficients.size() / fj_nnz_per_refresh_stretch, fj_max_refresh_stretch); + const i_t refresh_period = fj_cpu->settings.parameters.lhs_refresh_period * (1 + nnz_stretch); + cuopt_assert(refresh_period > 0, "refresh period overflowed"); + while (!fj_cpu->halted && !fj_cpu->preemption_flag.load()) { // Check if 5 seconds have passed auto now = std::chrono::high_resolution_clock::now(); @@ -2167,10 +2205,7 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w // periodically recompute the LHS and violation scores // to correct any accumulated numerical errors - cuopt_assert(fj_cpu->settings.parameters.lhs_refresh_period > 0, - "lhs_refresh_period should be positive"); - if (fj_cpu->iterations % fj_cpu->settings.parameters.lhs_refresh_period == 0 || - fj_cpu->trigger_early_lhs_recomputation) { + if (fj_cpu->iterations % refresh_period == 0 || fj_cpu->trigger_early_lhs_recomputation) { recompute_lhs(*fj_cpu); fj_cpu->trigger_early_lhs_recomputation = false; } @@ -2217,8 +2252,7 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w track_infeasible_checkpoint(*fj_cpu); if (should_perturb) { perturb(*fj_cpu); - for (size_t i = 0; i < fj_cpu->cached_mtm_moves.size(); i++) - fj_cpu->cached_mtm_moves[i].first = 0; + invalidate_mtm_cache(*fj_cpu); } two_opt_move_t two_opt_move; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 53de8ba2d4..6b4d0f0f76 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -85,6 +85,9 @@ struct host_contiguous_set_t { std::vector is_member; }; +constexpr double fj_obj_mult_min = 0.25; +constexpr double fj_obj_mult_max = 4.0; + // NOTE: this seems an easy pick for reflection/xmacros once this is available (C++26?) // Maintaining a single source of truth for all members would be nice template @@ -180,6 +183,10 @@ struct fj_cpu_climber_t { ins_vector h_assignment; ins_vector h_best_assignment; f_t h_objective_weight; + // Lower bound h_objective_weight decays to, so a lane seeded with objective pressure keeps it. + f_t seed_objective_weight{0}; + // Mean absolute nonzero objective coefficient; the unit of the objective score term. + f_t obj_magnitude{1}; f_t h_incumbent_objective; f_t h_best_objective; i_t last_feasible_entrance_iter{0}; @@ -210,6 +217,10 @@ struct fj_cpu_climber_t { // CSR nnz offset -> (delta, score) std::vector> cached_mtm_moves; + // Entry i is live only while cached_mtm_moves_version[i] == h_cstr_version of i's row. + std::vector cached_mtm_moves_version; + std::vector h_cstr_version; + // CSC (transposed!) nnz-offset-indexed constraint bounds (lb, ub) // std::pair better compile down to 16 bytes!! GCC do your job! ins_vector> cached_cstr_bounds; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 536254bb31..7ba19e6a52 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -205,7 +205,6 @@ constexpr int32_t fj_bin_restart_period = 5000000; // Escalation threshold and step, in infeasible local minima without a severity improvement. constexpr int32_t fj_bin_ddfw_escalate_after = 2000; constexpr int32_t fj_bin_ddfw_escalate_max = 100; - // prefetch distance // TODO: check if it actually matters at all for performance constexpr int32_t fj_bin_pf_dist = 8; @@ -589,6 +588,8 @@ struct fj_bin_engine_t { int32_t objective_weight{0}; int32_t seed_objective_weight{0}; + // Mean absolute nonzero objective coefficient; the unit of the objective score term. + double obj_magnitude{1.0}; double incumbent_objective{0}; double best_objective{std::numeric_limits::infinity()}; int32_t max_weight{1}; @@ -768,8 +769,18 @@ struct fj_bin_engine_t { int64_t objective_terms(int32_t v, int8_t delta) const { const double obj_diff = pb.objective[v] * delta; - const int32_t base = obj_diff < 0 ? objective_weight : (obj_diff > 0 ? -objective_weight : 0); - int32_t bonus = 0; + int32_t base = 0; + if (obj_diff != 0) { + cuopt_assert(obj_magnitude > 0, "objective magnitude unit must be positive"); + const double rel = std::fabs(obj_diff) / obj_magnitude; + const double mult = + rel < fj_obj_mult_min ? fj_obj_mult_min : (rel > fj_obj_mult_max ? fj_obj_mult_max : rel); + const double raw = objective_weight * mult; + cuopt_assert(fj_bin_in_int32(raw), "scaled objective weight out of int32 range"); + const int32_t scaled = (int32_t)std::lround(raw); + base = obj_diff < 0 ? scaled : -scaled; + } + int32_t bonus = 0; const bool old_better = incumbent_objective < best_objective; const bool new_better = incumbent_objective + obj_diff < best_objective; if (!old_better && new_better) { @@ -1253,6 +1264,12 @@ struct fj_bin_engine_t { const int32_t seeded_weight = (int32_t)std::lround(climber.h_objective_weight); cuopt_assert(seeded_weight >= 0, "objective weight should be positive or zero"); + double abs_obj_sum = 0; + for (int32_t v : pb.objective_vars) abs_obj_sum += std::fabs(pb.objective[v]); + obj_magnitude = abs_obj_sum > 0 ? abs_obj_sum / (double)pb.objective_vars.size() : 1.0; + cuopt_assert(std::isfinite(obj_magnitude) && obj_magnitude > 0, + "objective magnitude unit must be finite and positive"); + argmax_tile = fj_bin_argmax_tile(); objective_weight = seeded_weight > 0 ? seeded_weight : 0; seed_objective_weight = objective_weight; From b4ed10494500655059e565d16e896e10e01e88de Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 06:36:53 -0700 Subject: [PATCH 32/61] objective weight tweaks --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 26 +++++++++- .../feasibility_jump/fj_cpu_binary.cu | 48 +++++++++++++++---- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 7f07df475a..4b36bb3aaa 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -38,7 +38,7 @@ #include #include -#define CPUFJ_TIMING_TRACE 1 +#define CPUFJ_TIMING_TRACE 0 // Define CPUFJ_NVTX_RANGES to enable detailed NVTX profiling ranges #ifdef CPUFJ_NVTX_RANGES @@ -1004,6 +1004,12 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) if (fj_cpu.violated_constraints.empty()) { fj_cpu.h_objective_weight += 1; } } +// Applied to the objective weight when a new incumbent lands: the bump it gains, and the ceiling +// it is held at so smooth_weights and update_weights cannot walk it out of scale with the +// feasibility term. +constexpr double fj_obj_weight_incumbent_bump = 4.0; +constexpr double fj_obj_weight_incumbent_cap = 64.0; + template static void apply_move(fj_cpu_climber_t& fj_cpu, i_t var_idx, @@ -1109,6 +1115,13 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.h_incumbent_objective, fj_cpu.h_assignment, current_work_units); } fj_cpu.feasible_found = true; + // Counteract the smooth_weights decay for a lane that is actively improving, and hold the + // weight at a scale where base_feas_sum still registers against it. + if (fj_cpu.h_objective_weight > 0) { + fj_cpu.h_objective_weight = + min((f_t)fj_obj_weight_incumbent_cap, + fj_cpu.h_objective_weight + (f_t)fj_obj_weight_incumbent_bump); + } } } @@ -2704,6 +2717,14 @@ void build_climber_portfolio(problem_t& problem, int64_t base_seed) { const int n_climbers = static_cast(climbers.size()); + + // Objective pressure across the portfolio, indexed by lane. Lanes 0 and 3 stay pure feasibility + // seekers until they cross, since the objective term only enters the score once the weight is + // positive; their nonzero floor then keeps a pull on the objective afterwards rather than letting + // smooth_weights decay it back to nothing. + const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; + const f_t obj_weight_floor[4] = {1, 4, 32, 1}; + for (int k = 0; k < n_climbers; ++k) { preemption_flags[k].store(false); fj_settings_t settings; @@ -2723,6 +2744,9 @@ void build_climber_portfolio(problem_t& problem, climbers[k]->mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); climbers[k]->nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); climbers[k]->perturb_interval = std::uniform_int_distribution(50, 500)(rng); + + climbers[k]->h_objective_weight = obj_weight_ladder[k % 4]; + //climbers[k]->seed_objective_weight = obj_weight_floor[k % 4]; } } diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 7ba19e6a52..65ccbf1d4a 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -205,6 +205,10 @@ constexpr int32_t fj_bin_restart_period = 5000000; // Escalation threshold and step, in infeasible local minima without a severity improvement. constexpr int32_t fj_bin_ddfw_escalate_after = 2000; constexpr int32_t fj_bin_ddfw_escalate_max = 100; + +// The same, in feasible local minima without a best-objective improvement. +constexpr int32_t fj_bin_obj_stall_after = 50; +constexpr int32_t fj_bin_obj_escalate_max = 10; // prefetch distance // TODO: check if it actually matters at all for performance constexpr int32_t fj_bin_pf_dist = 8; @@ -588,6 +592,9 @@ struct fj_bin_engine_t { int32_t objective_weight{0}; int32_t seed_objective_weight{0}; + // Feasible local minima since best_objective last moved, and the value it was last seen at. + int32_t iterations_at_same_objective{0}; + double last_best_objective{std::numeric_limits::infinity()}; // Mean absolute nonzero objective coefficient; the unit of the objective score term. double obj_magnitude{1.0}; double incumbent_objective{0}; @@ -993,10 +1000,29 @@ struct fj_bin_engine_t { reweight_constraint(best_donor, donated); } } - if (violated_list.empty()) objective_weight += 1; + if (violated_list.empty()) { + if (best_objective < last_best_objective) { + iterations_at_same_objective = 0; + last_best_objective = best_objective; + } else { + ++iterations_at_same_objective; + } + objective_weight += objective_weight_increment(); + } track_infeasible_checkpoint(); } + // Stall-escalation for the objective weight, the feasible-region counterpart of ddfw_transfer: + // a lane that keeps reaching local minima without moving its best objective needs more + // objective pressure than one that is still improving. + int32_t objective_weight_increment() const + { + if (iterations_at_same_objective <= fj_bin_obj_stall_after) return 1; + const int32_t steps = + 1 + (iterations_at_same_objective - fj_bin_obj_stall_after) / fj_bin_obj_stall_after; + return steps < fj_bin_obj_escalate_max ? steps : fj_bin_obj_escalate_max; + } + void reset_infeasible_checkpoint() { best_infeasible_assign.clear(); @@ -1270,15 +1296,17 @@ struct fj_bin_engine_t { cuopt_assert(std::isfinite(obj_magnitude) && obj_magnitude > 0, "objective magnitude unit must be finite and positive"); - argmax_tile = fj_bin_argmax_tile(); - objective_weight = seeded_weight > 0 ? seeded_weight : 0; - seed_objective_weight = objective_weight; - max_weight = fj_bin_ddfw_init; - incumbent_objective = 0; - best_objective = std::numeric_limits::infinity(); - feasible_found = false; - iters = 0; - last_restart_iter = 0; + argmax_tile = fj_bin_argmax_tile(); + objective_weight = seeded_weight > 0 ? seeded_weight : 0; + seed_objective_weight = objective_weight; + max_weight = fj_bin_ddfw_init; + incumbent_objective = 0; + best_objective = std::numeric_limits::infinity(); + last_best_objective = std::numeric_limits::infinity(); + iterations_at_same_objective = 0; + feasible_found = false; + iters = 0; + last_restart_iter = 0; recompute_slack(); } From 64b5bfde5b2b8713113ad35130758b4cf58d6c50 Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 07:41:10 -0700 Subject: [PATCH 33/61] perf optimization --- .../feasibility_jump/feasibility_jump.cuh | 9 ++-- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 43 ++++++++++--------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh index ac1da031e3..437dfa3a1b 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh @@ -529,12 +529,14 @@ class fj_t { HDI f_t lower_excess_score(i_t cstr, f_t lhs, f_t c_lb) const { - return raft::min(lhs - c_lb, (f_t)0); + const f_t excess = lhs - c_lb; + return excess < (f_t)0 ? excess : (f_t)0; } HDI f_t upper_excess_score(i_t cstr, f_t lhs, f_t c_ub) const { - return raft::min(c_ub - lhs, (f_t)0); + const f_t excess = c_ub - lhs; + return excess < (f_t)0 ? excess : (f_t)0; } // Computes the constraint's contribution to the feasibility score: @@ -564,7 +566,8 @@ class fj_t { { f_t cstr_tolerance = get_cstr_tolerance( c_lb, c_ub, pb.tolerances.absolute_tolerance, pb.tolerances.relative_tolerance); - return max((f_t)1e-12, cstr_tolerance - MACHINE_EPSILON); + const f_t corrected = cstr_tolerance - MACHINE_EPSILON; + return corrected > (f_t)1e-12 ? corrected : (f_t)1e-12; } HDI f_t get_corrected_tolerance(i_t cstr) const { diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 4b36bb3aaa..ac1a9aafba 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -38,7 +38,7 @@ #include #include -#define CPUFJ_TIMING_TRACE 0 +#define CPUFJ_TIMING_TRACE 1 // Define CPUFJ_NVTX_RANGES to enable detailed NVTX profiling ranges #ifdef CPUFJ_NVTX_RANGES @@ -163,6 +163,13 @@ std::pair feas_score_constraint(const typename fj_t::climber f_t bounds[2] = {c_lb, c_ub}; cuopt_assert(isfinite(c_lb) || isfinite(c_ub), "no range"); + + // Independent of bound_idx, so paid once rather than on both passes of an equality row. + const f_t moved_lhs = current_lhs + cstr_coeff * delta; + const f_t cstr_tolerance = fj.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + const bool old_viol = fj.excess_score(cstr_idx, current_lhs, c_lb, c_ub) < -cstr_tolerance; + const bool new_viol = fj.excess_score(cstr_idx, moved_lhs, c_lb, c_ub) < -cstr_tolerance; + for (i_t bound_idx = 0; bound_idx < 2; ++bound_idx) { if (!isfinite(bounds[bound_idx])) continue; @@ -177,7 +184,7 @@ std::pair feas_score_constraint(const typename fj_t::climber f_t sign = bound_idx == 0 ? -1 : 1; f_t rhs = bounds[bound_idx] * sign; f_t old_lhs = current_lhs * sign; - f_t new_lhs = (current_lhs + cstr_coeff * delta) * sign; + f_t new_lhs = moved_lhs * sign; f_t old_slack = rhs - old_lhs; f_t new_slack = rhs - new_lhs; @@ -187,12 +194,6 @@ std::pair feas_score_constraint(const typename fj_t::climber cuopt_assert(isfinite(new_lhs), ""); cuopt_assert(isfinite(old_slack) && isfinite(new_slack), ""); - f_t cstr_tolerance = fj.get_corrected_tolerance(cstr_idx, c_lb, c_ub); - - bool old_viol = fj.excess_score(cstr_idx, current_lhs, c_lb, c_ub) < -cstr_tolerance; - bool new_viol = - fj.excess_score(cstr_idx, current_lhs + cstr_coeff * delta, c_lb, c_ub) < -cstr_tolerance; - bool old_sat = old_lhs < rhs + cstr_tolerance; bool new_sat = new_lhs < rhs + cstr_tolerance; @@ -288,43 +289,43 @@ static void print_timing_stats(fj_cpu_climber_t& fj_cpu) auto [apply_avg, apply_total] = compute_avg_and_total(fj_cpu.apply_move_times); auto [weights_avg, weights_total] = compute_avg_and_total(fj_cpu.update_weights_times); auto [compute_score_avg, compute_score_total] = compute_avg_and_total(fj_cpu.compute_score_times); - CUOPT_LOG_TRACE("=== Timing Statistics (Iteration %d) ===", fj_cpu.iterations); - CUOPT_LOG_TRACE("find_lift_move: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("=== Timing Statistics (Iteration %d) ===", fj_cpu.iterations); + CUOPT_LOG_DEBUG("find_lift_move: avg=%.6f ms, total=%.6f ms, calls=%zu", lift_avg * 1000.0, lift_total * 1000.0, fj_cpu.find_lift_move_times.size()); - CUOPT_LOG_TRACE("find_mtm_move_viol: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("find_mtm_move_viol: avg=%.6f ms, total=%.6f ms, calls=%zu", viol_avg * 1000.0, viol_total * 1000.0, fj_cpu.find_mtm_move_viol_times.size()); - CUOPT_LOG_TRACE("find_mtm_move_sat: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("find_mtm_move_sat: avg=%.6f ms, total=%.6f ms, calls=%zu", sat_avg * 1000.0, sat_total * 1000.0, fj_cpu.find_mtm_move_sat_times.size()); - CUOPT_LOG_TRACE("apply_move: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("apply_move: avg=%.6f ms, total=%.6f ms, calls=%zu", apply_avg * 1000.0, apply_total * 1000.0, fj_cpu.apply_move_times.size()); - CUOPT_LOG_TRACE("update_weights: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("update_weights: avg=%.6f ms, total=%.6f ms, calls=%zu", weights_avg * 1000.0, weights_total * 1000.0, fj_cpu.update_weights_times.size()); - CUOPT_LOG_TRACE("compute_score: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("compute_score: avg=%.6f ms, total=%.6f ms, calls=%zu", compute_score_avg * 1000.0, compute_score_total * 1000.0, fj_cpu.compute_score_times.size()); - CUOPT_LOG_TRACE("cache hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("cache hit percentage: %.2f%%", (double)fj_cpu.hit_count / (fj_cpu.hit_count + fj_cpu.miss_count) * 100.0); - CUOPT_LOG_TRACE("bin candidate move hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("bin candidate move hit percentage: %.2f%%", (double)fj_cpu.candidate_move_hits[0] / (fj_cpu.candidate_move_hits[0] + fj_cpu.candidate_move_misses[0]) * 100.0); - CUOPT_LOG_TRACE("int candidate move hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("int candidate move hit percentage: %.2f%%", (double)fj_cpu.candidate_move_hits[1] / (fj_cpu.candidate_move_hits[1] + fj_cpu.candidate_move_misses[1]) * 100.0); - CUOPT_LOG_TRACE("cont candidate move hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("cont candidate move hit percentage: %.2f%%", (double)fj_cpu.candidate_move_hits[2] / (fj_cpu.candidate_move_hits[2] + fj_cpu.candidate_move_misses[2]) * 100.0); - CUOPT_LOG_TRACE("========================================"); + CUOPT_LOG_DEBUG("========================================"); } template @@ -2357,7 +2358,7 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w #if CPUFJ_TIMING_TRACE // Print final timing statistics - CUOPT_LOG_TRACE("=== Final Timing Statistics ==="); + CUOPT_LOG_DEBUG("=== Final Timing Statistics ==="); print_timing_stats(*fj_cpu); #endif } From 9c9488989d4a595ea2b854fa0ff9a6bbcd722204 Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 08:14:57 -0700 Subject: [PATCH 34/61] more optimization --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 44 ++++++++++++------- .../cuopt-developer/references/conventions.md | 18 ++++++++ 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index ac1a9aafba..d05c721c00 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -38,7 +38,7 @@ #include #include -#define CPUFJ_TIMING_TRACE 1 +#define CPUFJ_TIMING_TRACE 0 // Define CPUFJ_NVTX_RANGES to enable detailed NVTX profiling ranges #ifdef CPUFJ_NVTX_RANGES @@ -164,7 +164,7 @@ std::pair feas_score_constraint(const typename fj_t::climber f_t bounds[2] = {c_lb, c_ub}; cuopt_assert(isfinite(c_lb) || isfinite(c_ub), "no range"); - // Independent of bound_idx, so paid once rather than on both passes of an equality row. + // Independent of bound_idx. const f_t moved_lhs = current_lhs + cstr_coeff * delta; const f_t cstr_tolerance = fj.get_corrected_tolerance(cstr_idx, c_lb, c_ub); const bool old_viol = fj.excess_score(cstr_idx, current_lhs, c_lb, c_ub) < -cstr_tolerance; @@ -617,23 +617,37 @@ static inline std::pair compute_score(fj_cpu_climber_t(fj_cpu, var_idx); fj_cpu.nnz_processed_window += (offset_end - offset_begin); + const size_t nnz_read = (size_t)(offset_end - offset_begin); + fj_cpu.h_reverse_constraints.byte_loads += nnz_read * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_read * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.h_cstr_left_weights.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.h_cstr_right_weights.byte_loads += nnz_read * sizeof(f_t); + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + const f_t* const weight_l = fj_cpu.view.cstr_left_weights.data(); + const f_t* const weight_r = fj_cpu.view.cstr_right_weights.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + for (i_t i = offset_begin; i < offset_end; i++) { - auto cstr_idx = fj_cpu.h_reverse_constraints[i]; - auto cstr_coeff = fj_cpu.h_reverse_coefficients[i]; - auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[i].get(); + const i_t cstr_idx = rev_cstr[i]; + const f_t cstr_coeff = rev_coeff[i]; + const auto [c_lb, c_ub] = cstr_bounds[i]; cuopt_assert(c_lb <= c_ub, "invalid bounds"); - auto [cstr_base_feas, cstr_bonus_robust] = - feas_score_constraint(fj_cpu.view, - delta, - cstr_idx, - cstr_coeff, - c_lb, - c_ub, - fj_cpu.h_lhs[cstr_idx], - fj_cpu.h_cstr_left_weights[cstr_idx], - fj_cpu.h_cstr_right_weights[cstr_idx]); + auto [cstr_base_feas, cstr_bonus_robust] = feas_score_constraint(fj_cpu.view, + delta, + cstr_idx, + cstr_coeff, + c_lb, + c_ub, + row_lhs[cstr_idx], + weight_l[cstr_idx], + weight_r[cstr_idx]); base_feas_sum += cstr_base_feas; bonus_robust_sum += cstr_bonus_robust; diff --git a/skills/cuopt-developer/references/conventions.md b/skills/cuopt-developer/references/conventions.md index 1bef2bbe3a..74fac403bf 100644 --- a/skills/cuopt-developer/references/conventions.md +++ b/skills/cuopt-developer/references/conventions.md @@ -197,6 +197,24 @@ rmm::device_uvector data(100, stream); Read existing code in `cpp/src/` for real examples of RMM allocation, stream-ordering, RAFT utilities, and kernel launch patterns. +### Bypassing `ins_vector`: credit the bytes back to the wrapper + +The instrumented accessors record a load per element read, and the counter lives in the +wrapper while the data lives in the vector's buffer. The compiler cannot prove those do not +alias, so the counter round-trips through memory every iteration and serializes the loop. +That cost is measurable in the innermost scoring loops. + +Two instrumentation-free paths to the same buffers already exist: `data()` on the wrapper +returns the raw pointer without recording, and the spans published on `fj_cpu.view` alias the +same allocations. + +When you take either path, add the skipped bytes back into the wrapper you bypassed — +`byte_loads` and `byte_stores` are public `mutable size_t` on +`memory_instrumentation_base_t`, so one `+= n * sizeof(element)` above the loop replaces N +per-element records. Do not route them into a separate counter: the byte totals feed the +deterministic work-unit proxy, and crediting the wrapper keeps both `collect()` and +`collect_per_wrapper()` correct and leaves the work-unit calibration untouched. + ## Test Impact Check **Before any behavioral change, ask:** From 08a858aaf67d4878b96efb06607b03a5a951adbd Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 09:11:34 -0700 Subject: [PATCH 35/61] weight_escalation_delta --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index d05c721c00..379ce7da59 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -655,9 +655,7 @@ static inline std::pair compute_score(fj_cpu_climber_t 0 && obj_diff != 0) { - // Scaling base by the objective magnitude only means something where there is feasibility - // impact to trade against; at base_feas_sum zero it would only make base distinct across - // candidates, which strands the bonus stage of the staged comparison. + // Scaling base is only meaningful where there is feasibility impact to trade against. f_t weighted = fj_cpu.h_objective_weight; if (base_feas_sum != 0) { cuopt_assert(fj_cpu.obj_magnitude > 0, "objective magnitude unit must be positive"); @@ -967,6 +965,19 @@ static void smooth_weights(fj_cpu_climber_t& fj_cpu) } } +// Escalation threshold and step for the violated-row bump, in local minima without a severity gain. +constexpr int32_t fj_weight_escalate_after = 2000; +constexpr int32_t fj_weight_escalate_max = 100; + +template +static i_t weight_escalation_delta(const fj_cpu_climber_t& fj_cpu) +{ + const i_t stall = fj_cpu.iters_since_infeasible_improve; + if (stall <= fj_weight_escalate_after) return 1; + const i_t steps = (stall - fj_weight_escalate_after) / fj_weight_escalate_after + 1; + return steps < fj_weight_escalate_max ? steps : fj_weight_escalate_max; +} + template static void update_weights(fj_cpu_climber_t& fj_cpu) { @@ -981,6 +992,8 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) return; } + const i_t escalated_delta = weight_escalation_delta(fj_cpu); + for (auto cstr_idx : fj_cpu.violated_constraints) { f_t curr_incumbent_lhs = fj_cpu.h_lhs[cstr_idx]; f_t curr_lower_excess = @@ -998,7 +1011,7 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) cuopt_assert(curr_excess_score < 0, "constraint not violated"); - i_t int_delta = 1.0; + i_t int_delta = escalated_delta; f_t delta = int_delta; f_t new_weight = old_weight + delta; @@ -1019,9 +1032,7 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) if (fj_cpu.violated_constraints.empty()) { fj_cpu.h_objective_weight += 1; } } -// Applied to the objective weight when a new incumbent lands: the bump it gains, and the ceiling -// it is held at so smooth_weights and update_weights cannot walk it out of scale with the -// feasibility term. +// Bump and ceiling applied to the objective weight when a new incumbent lands. constexpr double fj_obj_weight_incumbent_bump = 4.0; constexpr double fj_obj_weight_incumbent_cap = 64.0; From ff86b7438c2d937487c45ff1e1cd424490f908ae Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 09:54:17 -0700 Subject: [PATCH 36/61] infeasible region kicks --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 68 +++++++++++++++---- .../feasibility_jump/fj_cpu_binary.cu | 53 ++++++++++++++- 2 files changed, 105 insertions(+), 16 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 379ce7da59..de159fcd98 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1562,6 +1562,27 @@ static thrust::tuple find_lift_move( return thrust::make_tuple(best_move, best_score); } +// Draws a uniform in-bounds value, rounded and re-clamped for integer variables. +template +static void randomize_variable(fj_cpu_climber_t& fj_cpu, + i_t var_idx, + raft::random::PCGenerator& rng) +{ + f_t lb = std::max(get_lower(fj_cpu.h_var_bounds[var_idx].get()), -1e7); + f_t ub = std::min(get_upper(fj_cpu.h_var_bounds[var_idx].get()), 1e7); + f_t val = lb + (ub - lb) * rng.next_double(); + if (is_integer_var(fj_cpu, var_idx)) { + lb = std::ceil(lb); + ub = std::floor(ub); + val = std::round(val); + val = std::min(std::max(val, lb), ub); + } + + cuopt_assert((check_variable_within_bounds(fj_cpu, var_idx, val)), + "value is out of bounds"); + fj_cpu.h_assignment[var_idx] = val; +} + template static void perturb(fj_cpu_climber_t& fj_cpu) { @@ -1581,21 +1602,8 @@ static void perturb(fj_cpu_climber_t& fj_cpu) fj_cpu.rng); raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); - for (auto var_idx : sampled_vars) { - f_t lb = std::max(get_lower(fj_cpu.h_var_bounds[var_idx].get()), -1e7); - f_t ub = std::min(get_upper(fj_cpu.h_var_bounds[var_idx].get()), 1e7); - f_t val = lb + (ub - lb) * rng.next_double(); - if (is_integer_var(fj_cpu, var_idx)) { - lb = std::ceil(lb); - ub = std::floor(ub); - val = std::round(val); - val = std::min(std::max(val, lb), ub); - } - - cuopt_assert((check_variable_within_bounds(fj_cpu, var_idx, val)), - "value is out of bounds"); - fj_cpu.h_assignment[var_idx] = val; - } + for (auto var_idx : sampled_vars) + randomize_variable(fj_cpu, var_idx, rng); recompute_lhs(fj_cpu); } @@ -1626,6 +1634,11 @@ static void restart_from_infeasible_checkpoint(fj_cpu_climber_t& fj_cp invalidate_mtm_cache(fj_cpu); } +// Nonzeros per extra restart window, the cap on that, and how many windows a lane waits. +constexpr int32_t fj_restart_window_nnz_scale = 100000; +constexpr int32_t fj_restart_window_scale_max = 4; +constexpr int32_t fj_restart_window_multiple = 4; + template static void track_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) { @@ -1650,6 +1663,31 @@ static void track_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) return; } + // A lane that has never crossed and has exhausted its restores abandons the basin outright. + if (!fj_cpu.feasible_found) { + const i_t nnz_scale = + 1 + (i_t)fj_cpu.h_coefficients.size() / fj_restart_window_nnz_scale; + const i_t capped = nnz_scale < fj_restart_window_scale_max ? nnz_scale + : fj_restart_window_scale_max; + if (fj_cpu.iters_since_infeasible_improve >= + fj_restart_window_multiple * fj_cpu.infeasible_restart_window * capped && + fj_cpu.restores_since_improvement >= fj_cpu.infeasible_restart_max_streak) { + raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); + for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) + randomize_variable(fj_cpu, var_idx, rng); + + recompute_lhs(fj_cpu); + invalidate_mtm_cache(fj_cpu); + reset_infeasible_checkpoint(fj_cpu); + fj_cpu.restores_since_improvement = 0; + + CUOPT_LOG_DEBUG("%sCPUFJ randomized restart at iteration %d", + fj_cpu.log_prefix.c_str(), + fj_cpu.iterations); + return; + } + } + if (fj_cpu.restores_since_improvement >= fj_cpu.infeasible_restart_max_streak) return; if (++fj_cpu.iters_since_infeasible_improve < fj_cpu.infeasible_restart_window) return; if (severity <= fj_cpu.best_infeasible_severity * fj_cpu.infeasible_restart_degrade_ratio) return; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 65ccbf1d4a..023d433b5d 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -209,6 +209,13 @@ constexpr int32_t fj_bin_ddfw_escalate_max = 100; // The same, in feasible local minima without a best-objective improvement. constexpr int32_t fj_bin_obj_stall_after = 50; constexpr int32_t fj_bin_obj_escalate_max = 10; + +// Infeasible-region kick: stall, cooldown, post-restart quiet window, rows drawn, flips per row. +constexpr int32_t fj_bin_kick_after = 200; +constexpr int32_t fj_bin_kick_cooldown = 200; +constexpr int32_t fj_bin_kick_restart_guard = 50; +constexpr int32_t fj_bin_kick_rows = 3; +constexpr int32_t fj_bin_kick_vars_per_row = 2; // prefetch distance // TODO: check if it actually matters at all for performance constexpr int32_t fj_bin_pf_dist = 8; @@ -605,6 +612,7 @@ struct fj_bin_engine_t { int32_t iters{0}; int32_t last_feasible_entrance_iter{0}; int32_t last_restart_iter{0}; + int32_t last_kick_iter{0}; int64_t nnz_touched{0}; // Denominator for the ops-per-nnz roofline: nonzeros the row kernel actually processes, and the @@ -1202,6 +1210,39 @@ struct fj_bin_engine_t { return {best_v, best_s}; } + // Flips a few variables drawn from violated rows, to leave a basin the weights cannot escape. + void infeasible_region_kick() + { + const int32_t n_viol = (int32_t)violated_list.size(); + cuopt_assert(n_viol > 0, "kick requires a violated row"); + + int32_t flipped[fj_bin_kick_rows * fj_bin_kick_vars_per_row]; + int32_t n_flipped = 0; + + for (int32_t i = 0; i < fj_bin_kick_rows; ++i) { + const int32_t r = violated_list[rng.next_u32() % (uint32_t)n_viol]; + const int32_t row_begin = pb.offsets[r]; + const int32_t row_end = pb.offsets[r + 1]; + if (row_begin >= row_end) continue; + + for (int32_t j = 0; j < fj_bin_kick_vars_per_row; ++j) { + const int32_t k = row_begin + (int32_t)(rng.next_u32() % (uint32_t)(row_end - row_begin)); + const int32_t v = pb.variables[k]; + + bool already = false; + for (int32_t f = 0; f < n_flipped && !already; ++f) + already = flipped[f] == v; + if (already) continue; + + cuopt_assert(n_flipped < fj_bin_kick_rows * fj_bin_kick_vars_per_row, "flip list overflow"); + flipped[n_flipped++] = v; + assign[v] = (int8_t)(1 - assign[v]); + assign_i32[v] = assign[v]; + } + } + recompute_slack(); + } + void perturb() { if (pb.objective_vars.empty()) return; @@ -1307,6 +1348,7 @@ struct fj_bin_engine_t { feasible_found = false; iters = 0; last_restart_iter = 0; + last_kick_iter = 0; recompute_slack(); } @@ -1341,7 +1383,16 @@ struct fj_bin_engine_t { apply_move(move_var, (int8_t)(1 - 2 * assign[move_var]), climber); } else { update_weights(); - if (perturb_now) perturb(); + const bool kick_ready = !violated_list.empty() && + iters_since_infeasible_improve >= fj_bin_kick_after && + iters - last_kick_iter >= fj_bin_kick_cooldown && + iters - last_restart_iter >= fj_bin_kick_restart_guard; + if (kick_ready) { + infeasible_region_kick(); + last_kick_iter = iters; + } else if (perturb_now) { + perturb(); + } std::tie(move_var, score) = find_move_violated(1, true); const int32_t v = move_var >= 0 ? move_var : 0; apply_move(v, (int8_t)(1 - 2 * assign[v]), climber); From 4ff4650fc56589f6cb9d52909b7461c40dd200cc Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 10:14:14 -0700 Subject: [PATCH 37/61] 2opt lift move thing --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 139 +++++++++++++++++- .../feasibility_jump/fj_cpu_binary.cu | 104 ++++++++++++- 2 files changed, 236 insertions(+), 7 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index de159fcd98..7658583c54 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1435,6 +1435,125 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) } +// Candidate draws per 2-opt lift search. +constexpr int32_t fj_2opt_candidates = 32; + +// True when flipping both variables leaves every row they touch satisfied. Both reverse ranges are +// row-ascending, so a merge handles rows containing both variables with their joint delta. +template +static bool paired_flip_keeps_feasible( + fj_cpu_climber_t& fj_cpu, i_t var1, f_t delta1, i_t var2, f_t delta2) +{ + const auto range1 = reverse_range_for_var(fj_cpu, var1); + const auto range2 = reverse_range_for_var(fj_cpu, var2); + i_t i = range1.first, ie = range1.second; + i_t j = range2.first, je = range2.second; + + while (i < ie || j < je) { + const i_t r1 = i < ie ? (i_t)fj_cpu.h_reverse_constraints[i] : std::numeric_limits::max(); + const i_t r2 = j < je ? (i_t)fj_cpu.h_reverse_constraints[j] : std::numeric_limits::max(); + const i_t r = r1 < r2 ? r1 : r2; + + f_t change = 0; + f_t c_lb = 0; + f_t c_ub = 0; + if (r1 == r) { + auto [lb, ub] = fj_cpu.cached_cstr_bounds[i].get(); + c_lb = lb; + c_ub = ub; + change += (f_t)fj_cpu.h_reverse_coefficients[i] * delta1; + ++i; + } + if (r2 == r) { + auto [lb, ub] = fj_cpu.cached_cstr_bounds[j].get(); + c_lb = lb; + c_ub = ub; + change += (f_t)fj_cpu.h_reverse_coefficients[j] * delta2; + ++j; + } + + const f_t new_lhs = fj_cpu.h_lhs[r] + (change - fj_cpu.h_lhs_sumcomp[r]); + if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < + -fj_cpu.view.get_corrected_tolerance(r, c_lb, c_ub)) + return false; + } + return true; +} + +template +static thrust::tuple find_lift_2opt_move( + fj_cpu_climber_t& fj_cpu) +{ + timing_raii_t timer(fj_cpu.find_lift_move_times); + CPUFJ_NVTX_RANGE("CPUFJ::find_lift_2opt_move"); + cuopt_assert(fj_cpu.violated_constraints.empty(), "lift moves require a feasible incumbent"); + + fj_move_t best_first = fj_move_t{-1, 0}; + fj_move_t best_second = fj_move_t{-1, 0}; + fj_staged_score_t best_score = fj_staged_score_t::zero(); + + const i_t n_obj = (i_t)fj_cpu.h_objective_vars.size(); + if (n_obj == 0) return thrust::make_tuple(best_first, best_second, best_score); + + raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); + const i_t n_draws = n_obj < fj_2opt_candidates ? n_obj : fj_2opt_candidates; + + for (i_t t = 0; t < n_draws; ++t) { + const i_t var1 = fj_cpu.h_objective_vars[rng.next_u32() % (uint32_t)n_obj]; + if (!fj_cpu.h_is_binary_variable[var1]) continue; + + const f_t coeff1 = fj_cpu.h_obj_coeffs[var1]; + const f_t val1 = fj_cpu.h_assignment[var1]; + const f_t delta1 = round(1.0 - 2 * val1); + if (delta1 * coeff1 >= 0) continue; + if (tabu_check(fj_cpu, var1, delta1)) continue; + + // Breaking nothing is the single-flip lift's job; breaking several rows cannot be repaired by + // one companion. + const auto range1 = reverse_range_for_var(fj_cpu, var1); + i_t broken = -1; + bool multiple = false; + for (i_t k = range1.first; k < range1.second && !multiple; ++k) { + auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[k].get(); + const i_t r = fj_cpu.h_reverse_constraints[k]; + const f_t new_lhs = fj_cpu.h_lhs[r] + ((f_t)fj_cpu.h_reverse_coefficients[k] * delta1 - + fj_cpu.h_lhs_sumcomp[r]); + if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < + -fj_cpu.view.get_corrected_tolerance(r, c_lb, c_ub)) { + if (broken >= 0) + multiple = true; + else + broken = r; + } + } + if (multiple || broken < 0) continue; + + const auto row = range_for_constraint(fj_cpu, broken); + for (i_t k = row.first; k < row.second; ++k) { + const i_t var2 = fj_cpu.h_variables[k]; + if (var2 == var1) continue; + if (!fj_cpu.h_is_binary_variable[var2]) continue; + + const f_t coeff2 = fj_cpu.h_obj_coeffs[var2]; + const f_t val2 = fj_cpu.h_assignment[var2]; + const f_t delta2 = round(1.0 - 2 * val2); + const f_t combined = delta1 * coeff1 + delta2 * coeff2; + if (combined >= 0) continue; + if (tabu_check(fj_cpu, var2, delta2)) continue; + if (!paired_flip_keeps_feasible(fj_cpu, var1, delta1, var2, delta2)) continue; + + fj_staged_score_t score = fj_staged_score_t::zero(); + score.base = round(-combined); + if (best_score < score) { + best_score = score; + best_first = fj_move_t{var1, delta1}; + best_second = fj_move_t{var2, delta2}; + } + } + } + return thrust::make_tuple(best_first, best_second, best_score); +} + template static thrust::tuple find_lift_move( fj_cpu_climber_t& fj_cpu) @@ -2294,9 +2413,23 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w bool is_mtm_sat = false; // Perform lift moves + fj_move_t lift_companion = fj_move_t{-1, 0}; if (fj_cpu->violated_constraints.empty()) { thrust::tie(move, score) = find_lift_move(*fj_cpu); - if (score > fj_staged_score_t::zero()) is_lift = true; + if (score > fj_staged_score_t::zero()) { + is_lift = true; + } else { + // Pairs are only reachable once no single improving flip preserves feasibility. + fj_move_t first, second; + fj_staged_score_t pair_score; + thrust::tie(first, second, pair_score) = find_lift_2opt_move(*fj_cpu); + if (pair_score > fj_staged_score_t::zero()) { + move = first; + lift_companion = second; + score = pair_score; + is_lift = true; + } + } } // Regular MTM if (!(score > fj_staged_score_t::zero())) { @@ -2319,6 +2452,10 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w if (score > fj_staged_score_t::zero() && !should_perturb) { apply_move(*fj_cpu, move.var_idx, move.value, false); + if (lift_companion.var_idx >= 0) { + apply_move(*fj_cpu, lift_companion.var_idx, lift_companion.value, false); + fj_cpu->n_lift_moves_window++; + } // Track move types if (is_lift) fj_cpu->n_lift_moves_window++; if (is_mtm_viol) fj_cpu->n_mtm_viol_moves_window++; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 023d433b5d..cdbe16b8bd 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -216,6 +216,9 @@ constexpr int32_t fj_bin_kick_cooldown = 200; constexpr int32_t fj_bin_kick_restart_guard = 50; constexpr int32_t fj_bin_kick_rows = 3; constexpr int32_t fj_bin_kick_vars_per_row = 2; + +// Candidate draws per 2-opt lift search. +constexpr int32_t fj_bin_2opt_candidates = 64; // prefetch distance // TODO: check if it actually matters at all for performance constexpr int32_t fj_bin_pf_dist = 8; @@ -1189,6 +1192,82 @@ struct fj_bin_engine_t { return find_move_in_rows(sample_buf, false); } + // True when flipping both variables leaves every row they touch satisfied. Both reverse ranges + // are row-ascending, so shared rows are handled jointly by merging them. + bool paired_flip_keeps_feasible( + int32_t var1, int8_t delta1, int32_t var2, int8_t delta2) const + { + int32_t i = pb.reverse_offsets[var1], ie = pb.reverse_offsets[var1 + 1]; + int32_t j = pb.reverse_offsets[var2], je = pb.reverse_offsets[var2 + 1]; + + while (i < ie || j < je) { + const int32_t r1 = i < ie ? pb.reverse_constraints[i] : INT32_MAX; + const int32_t r2 = j < je ? pb.reverse_constraints[j] : INT32_MAX; + const int32_t r = r1 < r2 ? r1 : r2; + + int32_t change = 0; + if (r1 == r) change += (int32_t)pb.reverse_coefficients[i++] * delta1; + if (r2 == r) change += (int32_t)pb.reverse_coefficients[j++] * delta2; + if (row_slack[r] - change < 0) return false; + } + return true; + } + + std::pair, int64_t> find_lift_2opt_move() + { + cuopt_assert(violated_list.empty(), "lift moves require a feasible incumbent"); + + std::pair best_pair = {-1, -1}; + int64_t best_s = 0; + if (pb.objective_vars.empty()) return {best_pair, best_s}; + + const uint32_t n_obj = (uint32_t)pb.objective_vars.size(); + const int32_t n_draws = n_obj < (uint32_t)fj_bin_2opt_candidates ? (int32_t)n_obj + : fj_bin_2opt_candidates; + + for (int32_t t = 0; t < n_draws; ++t) { + const int32_t var1 = pb.objective_vars[rng.next_u32() % n_obj]; + const int8_t delta1 = (int8_t)(1 - 2 * assign[var1]); + if ((double)delta1 * pb.objective[var1] >= 0) continue; + if (tabu_blocked(var1, false)) continue; + + // Only pairs are useful here: a flip breaking nothing is already the single-flip lift's job, + // and one breaking several rows cannot be repaired by a single companion. + int32_t broken = -1; + bool multiple = false; + for (int32_t i = pb.reverse_offsets[var1]; i < pb.reverse_offsets[var1 + 1] && !multiple; + ++i) { + const int32_t r = pb.reverse_constraints[i]; + if (row_slack[r] - (int32_t)pb.reverse_coefficients[i] * delta1 < 0) { + if (broken >= 0) + multiple = true; + else + broken = r; + } + } + if (multiple || broken < 0) continue; + + for (int32_t k = pb.offsets[broken]; k < pb.offsets[broken + 1]; ++k) { + const int32_t var2 = pb.variables[k]; + if (var2 == var1) continue; + + const int8_t delta2 = (int8_t)(1 - 2 * assign[var2]); + const double combined = (double)delta1 * pb.objective[var1] + + (double)delta2 * pb.objective[var2]; + if (combined >= 0) continue; + if (tabu_blocked(var2, false)) continue; + if (!paired_flip_keeps_feasible(var1, delta1, var2, delta2)) continue; + + const int64_t s = (int64_t)(-std::llround(combined)) * fj_bin_score_k; + if (s > best_s) { + best_s = s; + best_pair = {var1, var2}; + } + } + } + return {best_pair, best_s}; + } + std::pair find_lift_move() const { cuopt_assert(violated_list.empty(), "lift moves require a feasible incumbent"); @@ -1367,11 +1446,21 @@ struct fj_bin_engine_t { if (iters - last_restart_iter >= fj_bin_restart_period) do_restart(); tabu.maybe_rebase(iters); - int32_t move_var = -1; - int64_t score = fj_bin_score_invalid; - if (violated_list.empty()) std::tie(move_var, score) = find_lift_move(); - if (score <= 0) std::tie(move_var, score) = find_move_global(false); - if (feasible_found && score <= 0) std::tie(move_var, score) = find_move_satisfied(mtm_sat_samples); + int32_t move_var = -1; + int64_t score = fj_bin_score_invalid; + std::pair pair2 = {-1, -1}; + if (violated_list.empty()) { + std::tie(move_var, score) = find_lift_move(); + // Pairs are only reachable once no single improving flip preserves feasibility. + if (score <= 0) { + int64_t pair_score; + std::tie(pair2, pair_score) = find_lift_2opt_move(); + if (pair_score > 0) score = pair_score; + } + } + if (pair2.first < 0 && score <= 0) std::tie(move_var, score) = find_move_global(false); + if (pair2.first < 0 && feasible_found && score <= 0) + std::tie(move_var, score) = find_move_satisfied(mtm_sat_samples); bool perturb_now = false; if (violated_list.empty() && iters - last_feasible_entrance_iter > perturb_interval) { @@ -1379,7 +1468,10 @@ struct fj_bin_engine_t { last_feasible_entrance_iter = iters; } - if (score > 0 && move_var >= 0 && !perturb_now) { + if (pair2.first >= 0 && !perturb_now) { + apply_move(pair2.first, (int8_t)(1 - 2 * assign[pair2.first]), climber); + apply_move(pair2.second, (int8_t)(1 - 2 * assign[pair2.second]), climber); + } else if (score > 0 && move_var >= 0 && !perturb_now) { apply_move(move_var, (int8_t)(1 - 2 * assign[move_var]), climber); } else { update_weights(); From 01b44ae6438a0aa8e2bd959ec1bd92eeee354047 Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 10:48:00 -0700 Subject: [PATCH 38/61] portfolio lane seeding --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 252 +++++++++++++++++- 1 file changed, 238 insertions(+), 14 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 7658583c54..9c698e47ea 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -2754,8 +2754,9 @@ std::unique_ptr> init_fj_cpu_standalone( // Early CPUFJ runs while presolve is still probing, so there are no implications to hand it const probing_cache_t* no_implications = nullptr; init_fj_cpu(*fj_cpu, solution, default_weights, default_weights, 0.0, no_implications); - fj_cpu->settings = settings; - fj_cpu->settings.seed = cuopt::seed_generator::get_seed(); + // settings.seed is caller-drawn: seed_generator steps a non-atomic global and this may run + // concurrently across lanes. + fj_cpu->settings = settings; return fj_cpu; } @@ -2773,8 +2774,8 @@ std::unique_ptr> init_fj_cpu_standalone_from_template std::vector default_weights(problem.n_constraints, 1.0); init_fj_cpu_from_template(*fj_cpu, tmpl, problem, default_weights, default_weights, 0.0); - fj_cpu->settings = settings; - fj_cpu->settings.seed = cuopt::seed_generator::get_seed(); + // See init_fj_cpu_standalone: the seed is caller-drawn, not taken from the global generator. + fj_cpu->settings = settings; return fj_cpu; } @@ -2905,6 +2906,201 @@ template void finalize_fj_cpu_host_initialization( const typename mip_solver_settings_t::tolerances_t& tolerances); #endif +// Above this the O(nnz) seed passes eat a meaningful slice of a short budget, so they are skipped. +constexpr int64_t fj_seed_nnz_limit = 8'000'000; + +// Jumps each two-sided variable to whichever bound has fewer rows locking it in that direction. +template +static void apply_lock_weighted_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const i_t n_variables = fj_cpu.view.pb.n_variables; + for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { + const f_t lb = get_lower(fj_cpu.h_var_bounds[var_idx].get()); + const f_t ub = get_upper(fj_cpu.h_var_bounds[var_idx].get()); + if (!isfinite(lb) || !isfinite(ub) || lb >= ub) continue; + + i_t lock_up = 0; + i_t lock_down = 0; + const auto range = reverse_range_for_var(fj_cpu, var_idx); + for (i_t i = range.first; i < range.second; ++i) { + const f_t coeff = fj_cpu.h_reverse_coefficients[i]; + const i_t cstr_idx = fj_cpu.h_reverse_constraints[i]; + const bool has_lb = isfinite((f_t)fj_cpu.h_cstr_lb[cstr_idx]); + const bool has_ub = isfinite((f_t)fj_cpu.h_cstr_ub[cstr_idx]); + if (coeff > 0) { + lock_up += has_ub; + lock_down += has_lb; + } else if (coeff < 0) { + lock_up += has_lb; + lock_down += has_ub; + } + } + + f_t new_val = lock_up <= lock_down ? ub : lb; + if (is_integer_var(fj_cpu, var_idx)) new_val = std::round(new_val); + fj_cpu.h_assignment[var_idx] = new_val; + } + + recompute_lhs(fj_cpu); + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// Jumps each bounded objective variable to the bound that minimises its own objective term. +template +static void apply_objective_corner_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const i_t n_variables = fj_cpu.view.pb.n_variables; + for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { + const f_t coeff = fj_cpu.h_obj_coeffs[var_idx]; + if (coeff == 0) continue; + + const f_t lb = get_lower(fj_cpu.h_var_bounds[var_idx].get()); + const f_t ub = get_upper(fj_cpu.h_var_bounds[var_idx].get()); + if (!isfinite(lb) || !isfinite(ub) || lb >= ub) continue; + + f_t new_val = coeff > 0 ? lb : ub; + if (is_integer_var(fj_cpu, var_idx)) new_val = std::round(new_val); + fj_cpu.h_assignment[var_idx] = new_val; + } + + recompute_lhs(fj_cpu); + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// A single-variable integer step on a row, with the magnitude of its effect on the row sum. +template +struct row_repair_move_t { + f_t effect; + i_t var; + f_t coeff; + f_t new_val; +}; + +// Collects the unit integer steps that push this row's sum in `direction`, largest effect first. +template +static void collect_row_repair_moves(fj_cpu_climber_t& fj_cpu, + i_t row_begin, + i_t row_end, + f_t direction, + f_t tol, + std::vector>& out) +{ + out.clear(); + for (i_t i = row_begin; i < row_end; ++i) { + const i_t var = fj_cpu.h_variables[i]; + if (!is_integer_var(fj_cpu, var)) continue; + + const f_t coeff = fj_cpu.h_coefficients[i]; + const f_t val = fj_cpu.h_assignment[var]; + const f_t lb = get_lower(fj_cpu.h_var_bounds[var].get()); + const f_t ub = get_upper(fj_cpu.h_var_bounds[var].get()); + const bool is_bin = fj_cpu.h_is_binary_variable[var] != 0; + + // Raising the variable shifts the sum by `direction * coeff`; lowering it by the negation. + const f_t raise = direction * coeff; + if (raise > 0 && val < ub - tol) { + const f_t new_val = is_bin ? (f_t)1 : std::floor(val) + 1; + if (new_val > val && new_val <= ub + tol) out.push_back({raise, var, coeff, new_val}); + } else if (raise < 0 && val > lb + tol) { + const f_t new_val = is_bin ? (f_t)0 : std::ceil(val) - 1; + if (new_val < val && new_val >= lb - tol) out.push_back({-raise, var, coeff, new_val}); + } + } + std::sort(out.begin(), out.end(), [](const row_repair_move_t& a, + const row_repair_move_t& b) { + return a.effect > b.effect; + }); +} + +// Time-boxed greedy row repair. Deliberately myopic, so it reverts unless it strictly reduces the +// violated-row count against the incoming anchor. +template +static void apply_greedy_covering_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + recompute_lhs(fj_cpu); + const i_t baseline_violated = fj_cpu.violated_constraints.size(); + const auto anchor_assignment = fj_cpu.h_assignment; + + const i_t n_constraints = fj_cpu.view.pb.n_constraints; + std::vector row_order(n_constraints); + for (i_t i = 0; i < n_constraints; ++i) + row_order[i] = i; + std::sort(row_order.begin(), row_order.end(), [&](i_t a, i_t b) { + return (fj_cpu.h_offsets[a + 1] - fj_cpu.h_offsets[a]) < + (fj_cpu.h_offsets[b + 1] - fj_cpu.h_offsets[b]); + }); + + const auto started = std::chrono::steady_clock::now(); + const double time_budget_s = 0.4; + const f_t tol = 1e-6; + const i_t max_passes = 2; + std::vector> candidates; + bool out_of_time = false; + + for (i_t pass = 0; pass < max_passes && !out_of_time; ++pass) { + for (i_t k = 0; k < n_constraints; ++k) { + if ((k & 0xFFF) == 0 && + std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + time_budget_s) { + out_of_time = true; + break; + } + const i_t cstr_idx = row_order[k]; + const i_t row_begin = fj_cpu.h_offsets[cstr_idx]; + const i_t row_end = fj_cpu.h_offsets[cstr_idx + 1]; + if (row_begin == row_end) continue; + + const f_t lb = fj_cpu.h_cstr_lb[cstr_idx]; + const f_t ub = fj_cpu.h_cstr_ub[cstr_idx]; + const bool has_lb = isfinite(lb); + const bool has_ub = isfinite(ub); + if (!has_lb && !has_ub) continue; + + f_t sum = 0; + for (i_t i = row_begin; i < row_end; ++i) + sum += (f_t)fj_cpu.h_coefficients[i] * (f_t)fj_cpu.h_assignment[fj_cpu.h_variables[i]]; + + // Equality rows are driven to their bound; one-sided rows only to the side they violate. + const bool is_equality = has_lb && has_ub && std::abs(lb - ub) < tol; + f_t direction = 0; + f_t target = 0; + if (is_equality && std::abs(sum - lb) > tol) { + direction = sum < lb ? (f_t)1 : (f_t)-1; + target = lb; + } else if (has_lb && sum < lb - tol) { + direction = 1; + target = lb; + } else if (has_ub && sum > ub + tol) { + direction = -1; + target = ub; + } else { + continue; + } + + collect_row_repair_moves(fj_cpu, row_begin, row_end, direction, tol, candidates); + for (const auto& m : candidates) { + if (direction > 0 ? sum >= target - tol : sum <= target + tol) break; + const f_t delta = m.new_val - (f_t)fj_cpu.h_assignment[m.var]; + sum += m.coeff * delta; + fj_cpu.h_assignment[m.var] = m.new_val; + } + } + } + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline_violated) { + fj_cpu.h_assignment = anchor_assignment; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + // Portfolio construction for the standalone benchmark. Host logic, but it lives // in a .cu because fj_cpu.cuh pulls in raft/util/cuda_dev_essentials.cuh through // solution.cuh, which does not compile under the host compiler. Kept out of the @@ -2926,17 +3122,24 @@ void build_climber_portfolio(problem_t& problem, const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; const f_t obj_weight_floor[4] = {1, 4, 32, 1}; - for (int k = 0; k < n_climbers; ++k) { + for (int k = 0; k < n_climbers; ++k) preemption_flags[k].store(false); - fj_settings_t settings; - settings.seed = (int)(base_seed + k); - // Built serially: the first climber host-copies the problem, the rest clone it. - if (k == 0) { - climbers[k] = init_fj_cpu_standalone(problem, solution, preemption_flags[k], settings); - } else { - climbers[k] = - init_fj_cpu_standalone_from_template(problem, *climbers[0], preemption_flags[k], settings); - } + + // cuopt::seed_generator::get_seed() steps a non-atomic global, so every lane's seed is drawn here + // in lane order before any concurrent construction below. + std::vector lane_seed(n_climbers); + for (int k = 0; k < n_climbers; ++k) + lane_seed[k] = cuopt::seed_generator::get_seed(); + + // Per-lane work that must run identically whether the lane was built serially or in parallel. + auto finish_lane = [&](int k) { + // Fewer-lock corner, objective-favorable corner, greedy row repair; lane 0 keeps the anchor. + if (k % 4 == 1) + apply_lock_weighted_seed(*climbers[k]); + else if (k % 4 == 2) + apply_objective_corner_seed(*climbers[k]); + else if (k % 4 == 3) + apply_greedy_covering_seed(*climbers[k]); // Default: every climber identical apart from its seed and a random draw of the // four sampling parameters. Diversification, decorrelated from the value RNG. @@ -2948,6 +3151,27 @@ void build_climber_portfolio(problem_t& problem, climbers[k]->h_objective_weight = obj_weight_ladder[k % 4]; //climbers[k]->seed_objective_weight = obj_weight_floor[k % 4]; + }; + + // Lane 0 is a genuine dependency: it host-copies the problem and every other lane clones it. + { + fj_settings_t settings; + settings.seed = (int)lane_seed[0]; + climbers[0] = init_fj_cpu_standalone(problem, solution, preemption_flags[0], settings); + finish_lane(0); + } + + // The remaining lanes depend only on lane 0's finished, read-only template, and the O(nnz) clone + // and seed passes are otherwise paid serially on one thread while the other pinned CPUs idle. +#ifdef _OPENMP +#pragma omp parallel for num_threads(std::max(1, n_climbers - 1)) schedule(static) +#endif + for (int k = 1; k < n_climbers; ++k) { + fj_settings_t settings; + settings.seed = (int)lane_seed[k]; + climbers[k] = + init_fj_cpu_standalone_from_template(problem, *climbers[0], preemption_flags[k], settings); + finish_lane(k); } } From af02f91a6f29ebaf5e48fc97ecc62e8d5b2702d6 Mon Sep 17 00:00:00 2001 From: yboucher Date: Fri, 21 Aug 2026 11:16:28 -0700 Subject: [PATCH 39/61] early cpu portfolio --- .../feasibility_jump/early_cpufj.cu | 82 ++++++++--- .../feasibility_jump/early_cpufj.cuh | 6 +- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 133 ++++++++++-------- .../feasibility_jump/fj_cpu.cuh | 10 +- cpp/src/mip_heuristics/mip_constants.hpp | 4 + 5 files changed, 149 insertions(+), 86 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu index e4db66ed1f..47a3711fb5 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu @@ -8,6 +8,10 @@ #include "early_cpufj.cuh" #include +#include + +#include +#include namespace cuopt::mathematical_optimization::mip { @@ -32,41 +36,77 @@ template void early_cpufj_t::start() { // 1: presolve, 1: early GPU FJ, 1: early CPU FJ - if (fj_cpu_ || omp_get_num_threads() < CUOPT_MIP_EARLY_CPUFJ_REQUIRED_THREAD_COUNT) { return; } + if (!climbers_.empty() || omp_get_num_threads() < CUOPT_MIP_EARLY_CPUFJ_REQUIRED_THREAD_COUNT) { + return; + } this->preemption_flag_.store(false); this->start_time_ = std::chrono::steady_clock::now(); - fj_cpu_ = - init_fj_cpu_from_optimization_problem(*this->problem_ptr_, tolerances_, preemption_flag_); - - fj_cpu_->log_prefix = "[Early CPUFJ] "; - - fj_cpu_->improvement_callback = [this](f_t solver_obj, - const std::vector& assignment, - double) { this->try_update_best(solver_obj, assignment); }; - - CUOPT_LOG_DEBUG("Launching early CPUFJ task"); -#pragma omp task shared(fj_cpu_) priority(CUOPT_DEFAULT_TASK_PRIORITY) \ - depend(out : *fj_cpu_) default(none) - cpufj_solve(fj_cpu_.get()); + // Tasks are not preempted, so a lane posted beyond the team size would sit in the queue for the + // whole of presolve without running an iteration. + const int n_lanes = std::min(CUOPT_MIP_EARLY_CPUFJ_MAX_CLIMBERS, omp_get_num_threads()); + const int64_t base_seed = cuopt::seed_generator::get_seed(); + climbers_.resize(n_lanes); + + auto report_incumbent = [this](f_t solver_obj, const std::vector& assignment, double) { + std::lock_guard guard(incumbent_mutex_); + this->try_update_best(solver_obj, assignment); + }; + + // Lane 0 builds the host problem representation and every other lane copies it. All of it + // finishes before the first task is posted, so no lane reads a template another lane is running + // on. seed_generator steps a non-atomic global, which is why the draws stay on this thread. + for (int k = 0; k < n_lanes; ++k) { + if (k == 0) { + climbers_[0] = + init_fj_cpu_from_optimization_problem(*this->problem_ptr_, tolerances_, preemption_flag_); + } else { + fj_settings_t settings; + settings.seed = (int)cuopt::seed_generator::get_seed(); + climbers_[k] = init_fj_cpu_clone(*climbers_[0], preemption_flag_, settings); + } + apply_lane_diversification(*climbers_[k], k, base_seed); + climbers_[k]->log_prefix = "[Early CPUFJ " + std::to_string(k) + "] "; + climbers_[k]->improvement_callback = report_incumbent; + } + + CUOPT_LOG_DEBUG("Launching %d early CPUFJ tasks", n_lanes); + for (int k = 0; k < n_lanes; ++k) { + auto* climber = climbers_[k].get(); +#pragma omp task firstprivate(climber) priority(CUOPT_DEFAULT_TASK_PRIORITY) \ + depend(out : *climber) default(none) + cpufj_solve(climber); + } } template void early_cpufj_t::stop() { - if (!fj_cpu_) { return; } + if (climbers_.empty()) { return; } preemption_flag_.store(true); - fj_cpu_->halted = true; -#pragma omp taskwait depend(in : *fj_cpu_) // Wait for the early CPUFJ task to finish - - CUOPT_LOG_DEBUG("[Early CPUFJ] Stopped after %d iterations, solution_found=%d", - fj_cpu_ ? fj_cpu_->iterations : 0, + // Every lane is told to stop before any wait, otherwise the first wait blocks on a lane that has + // not been asked to exit yet. + for (auto& climber : climbers_) { + climber->halted = true; + } + for (size_t k = 0; k < climbers_.size(); ++k) { +#pragma omp taskwait depend(in : *climbers_[k]) // Wait for each early CPUFJ task to finish + } + + i_t total_iterations = 0; + for (const auto& climber : climbers_) { + total_iterations += climber->iterations; + } + + CUOPT_LOG_DEBUG("[Early CPUFJ] Stopped after %d iterations over %d climbers, solution_found=%d", + total_iterations, + (int)climbers_.size(), this->solution_found_); - fj_cpu_.reset(); + climbers_.clear(); } template diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh index a6cf3057e0..ede2ffe168 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh @@ -12,6 +12,7 @@ #include #include +#include #include namespace cuopt::mathematical_optimization::mip { @@ -37,8 +38,11 @@ class early_cpufj_t : public early_heuristic_t const optimization_problem_t* problem_ptr_; typename mip_solver_settings_t::tolerances_t tolerances_; - std::unique_ptr> fj_cpu_; + std::vector>> climbers_; std::atomic preemption_flag_{false}; + // try_update_best and the incumbent callback behind it are not thread-safe, and every lane + // reports into them from its own task. + std::mutex incumbent_mutex_; }; } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 9c698e47ea..10c0182bb4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1893,21 +1893,28 @@ static void init_fj_cpu(fj_cpu_climber_t& fj_cpu, template static void init_fj_cpu_from_template(fj_cpu_climber_t& fj_cpu, const fj_cpu_climber_t& tmpl, - problem_t& problem, const std::vector& left_weights, const std::vector& right_weights, f_t objective_weight) { - cuopt_assert(tmpl.h_offsets.size() == static_cast(problem.n_constraints + 1), - "template built on a different problem"); - cuopt_assert(tmpl.h_reverse_offsets.size() == static_cast(problem.n_variables + 1), - "template built on a different problem"); - cuopt_assert(tmpl.h_coefficients.size() == static_cast(problem.nnz), - "template built on a different problem"); - - fj_cpu.view = typename fj_t::climber_data_t::view_t{}; - fj_cpu.view.pb = problem.view(); - fj_cpu.pb_ptr = &problem; + const i_t n_variables = (i_t)tmpl.h_reverse_offsets.size() - 1; + const i_t n_constraints = (i_t)tmpl.h_offsets.size() - 1; + const i_t nnz = (i_t)tmpl.h_coefficients.size(); + + cuopt_assert(n_variables == tmpl.view.pb.n_variables, "template variable count mismatch"); + cuopt_assert(n_constraints == tmpl.view.pb.n_constraints, "template constraint count mismatch"); + cuopt_assert(nnz == tmpl.view.pb.nnz, "template nnz mismatch"); + cuopt_assert(left_weights.size() == static_cast(n_constraints), + "left weight size mismatch"); + cuopt_assert(right_weights.size() == static_cast(n_constraints), + "right weight size mismatch"); + + fj_cpu.view = typename fj_t::climber_data_t::view_t{}; + // Every span the host views cover is re-pointed at this climber's own arrays below. The rest of + // the problem view carries over from the template, which is also what makes this usable on + // climbers built without a problem_t at all. + fj_cpu.view.pb = tmpl.view.pb; + fj_cpu.pb_ptr = tmpl.pb_ptr; fj_cpu.h_reverse_coefficients = tmpl.h_reverse_coefficients; fj_cpu.h_reverse_constraints = tmpl.h_reverse_constraints; @@ -1929,19 +1936,19 @@ static void init_fj_cpu_from_template(fj_cpu_climber_t& fj_cpu, fj_cpu.h_objective_weight = objective_weight; fj_cpu.h_assignment = tmpl.h_assignment; fj_cpu.h_best_assignment = tmpl.h_assignment; - fj_cpu.h_tabu_nodec_until.resize(problem.n_variables, 0); - fj_cpu.h_tabu_noinc_until.resize(problem.n_variables, 0); - fj_cpu.h_tabu_lastdec.resize(problem.n_variables, 0); - fj_cpu.h_tabu_lastinc.resize(problem.n_variables, 0); + fj_cpu.h_tabu_nodec_until.resize(n_variables, 0); + fj_cpu.h_tabu_noinc_until.resize(n_variables, 0); + fj_cpu.h_tabu_lastdec.resize(n_variables, 0); + fj_cpu.h_tabu_lastinc.resize(n_variables, 0); fj_cpu.iterations = 0; finalize_fj_cpu_host_initialization_from_template(fj_cpu, tmpl, - problem.n_variables, - problem.n_constraints, - problem.n_integer_vars, - problem.nnz, - problem.tolerances); + n_variables, + n_constraints, + tmpl.n_integer_vars, + nnz, + tmpl.view.pb.tolerances); } template @@ -2762,18 +2769,17 @@ std::unique_ptr> init_fj_cpu_standalone( } template -std::unique_ptr> init_fj_cpu_standalone_from_template( - problem_t& problem, +std::unique_ptr> init_fj_cpu_clone( const fj_cpu_climber_t& tmpl, std::atomic& preemption_flag, fj_settings_t settings) { - raft::common::nvtx::range scope("init_fj_cpu_standalone_from_template"); + raft::common::nvtx::range scope("init_fj_cpu_clone"); auto fj_cpu = std::make_unique>(preemption_flag); - std::vector default_weights(problem.n_constraints, 1.0); - init_fj_cpu_from_template(*fj_cpu, tmpl, problem, default_weights, default_weights, 0.0); + std::vector default_weights(tmpl.view.pb.n_constraints, 1.0); + init_fj_cpu_from_template(*fj_cpu, tmpl, default_weights, default_weights, f_t{0}); // See init_fj_cpu_standalone: the seed is caller-drawn, not taken from the global generator. fj_cpu->settings = settings; @@ -2857,8 +2863,7 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); -template std::unique_ptr> init_fj_cpu_standalone_from_template( - problem_t& problem, +template std::unique_ptr> init_fj_cpu_clone( const fj_cpu_climber_t& tmpl, std::atomic& preemption_flag, fj_settings_t settings); @@ -2887,8 +2892,7 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); -template std::unique_ptr> init_fj_cpu_standalone_from_template( - problem_t& problem, +template std::unique_ptr> init_fj_cpu_clone( const fj_cpu_climber_t& tmpl, std::atomic& preemption_flag, fj_settings_t settings); @@ -3101,6 +3105,39 @@ static void apply_greedy_covering_seed(fj_cpu_climber_t& fj_cpu) fj_cpu.h_best_assignment = fj_cpu.h_assignment; } +// What makes one lane of a CPUFJ portfolio behave differently from another: which corner it starts +// from, how it samples, and how hard it pulls on the objective. Lane 0 keeps the anchor assignment +// so it is the lane every clone is built from. +template +void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, int64_t base_seed) +{ + // Objective pressure across the portfolio, indexed by lane. Lanes 0 and 3 stay pure feasibility + // seekers until they cross, since the objective term only enters the score once the weight is + // positive; their nonzero floor then keeps a pull on the objective afterwards rather than letting + // smooth_weights decay it back to nothing. + const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; + const f_t obj_weight_floor[4] = {1, 4, 32, 1}; + + // Fewer-lock corner, objective-favorable corner, greedy row repair; lane 0 keeps the anchor. + if (lane % 4 == 1) + apply_lock_weighted_seed(climber); + else if (lane % 4 == 2) + apply_objective_corner_seed(climber); + else if (lane % 4 == 3) + apply_greedy_covering_seed(climber); + + // Default: every climber identical apart from its seed and a random draw of the + // four sampling parameters. Diversification, decorrelated from the value RNG. + std::mt19937 rng(base_seed + 7919u * lane); + climber.mtm_viol_samples = std::uniform_int_distribution(15, 50)(rng); + climber.mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); + climber.nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); + climber.perturb_interval = std::uniform_int_distribution(50, 500)(rng); + + climber.h_objective_weight = obj_weight_ladder[lane % 4]; + //climber.seed_objective_weight = obj_weight_floor[lane % 4]; +} + // Portfolio construction for the standalone benchmark. Host logic, but it lives // in a .cu because fj_cpu.cuh pulls in raft/util/cuda_dev_essentials.cuh through // solution.cuh, which does not compile under the host compiler. Kept out of the @@ -3115,13 +3152,6 @@ void build_climber_portfolio(problem_t& problem, { const int n_climbers = static_cast(climbers.size()); - // Objective pressure across the portfolio, indexed by lane. Lanes 0 and 3 stay pure feasibility - // seekers until they cross, since the objective term only enters the score once the weight is - // positive; their nonzero floor then keeps a pull on the objective afterwards rather than letting - // smooth_weights decay it back to nothing. - const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; - const f_t obj_weight_floor[4] = {1, 4, 32, 1}; - for (int k = 0; k < n_climbers; ++k) preemption_flags[k].store(false); @@ -3131,34 +3161,12 @@ void build_climber_portfolio(problem_t& problem, for (int k = 0; k < n_climbers; ++k) lane_seed[k] = cuopt::seed_generator::get_seed(); - // Per-lane work that must run identically whether the lane was built serially or in parallel. - auto finish_lane = [&](int k) { - // Fewer-lock corner, objective-favorable corner, greedy row repair; lane 0 keeps the anchor. - if (k % 4 == 1) - apply_lock_weighted_seed(*climbers[k]); - else if (k % 4 == 2) - apply_objective_corner_seed(*climbers[k]); - else if (k % 4 == 3) - apply_greedy_covering_seed(*climbers[k]); - - // Default: every climber identical apart from its seed and a random draw of the - // four sampling parameters. Diversification, decorrelated from the value RNG. - std::mt19937 rng(base_seed + 7919u * k); - climbers[k]->mtm_viol_samples = std::uniform_int_distribution(15, 50)(rng); - climbers[k]->mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); - climbers[k]->nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); - climbers[k]->perturb_interval = std::uniform_int_distribution(50, 500)(rng); - - climbers[k]->h_objective_weight = obj_weight_ladder[k % 4]; - //climbers[k]->seed_objective_weight = obj_weight_floor[k % 4]; - }; - // Lane 0 is a genuine dependency: it host-copies the problem and every other lane clones it. { fj_settings_t settings; settings.seed = (int)lane_seed[0]; climbers[0] = init_fj_cpu_standalone(problem, solution, preemption_flags[0], settings); - finish_lane(0); + apply_lane_diversification(*climbers[0], 0, base_seed); } // The remaining lanes depend only on lane 0's finished, read-only template, and the O(nnz) clone @@ -3169,19 +3177,20 @@ void build_climber_portfolio(problem_t& problem, for (int k = 1; k < n_climbers; ++k) { fj_settings_t settings; settings.seed = (int)lane_seed[k]; - climbers[k] = - init_fj_cpu_standalone_from_template(problem, *climbers[0], preemption_flags[k], settings); - finish_lane(k); + climbers[k] = init_fj_cpu_clone(*climbers[0], preemption_flags[k], settings); + apply_lane_diversification(*climbers[k], k, base_seed); } } #if MIP_INSTANTIATE_FLOAT +template void apply_lane_diversification(fj_cpu_climber_t&, int, int64_t); template void build_climber_portfolio( problem_t&, solution_t&, std::vector>&, std::vector>>&, int64_t); #endif #if MIP_INSTANTIATE_DOUBLE +template void apply_lane_diversification(fj_cpu_climber_t&, int, int64_t); template void build_climber_portfolio( problem_t&, solution_t&, std::vector>&, std::vector>>&, int64_t); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 6b4d0f0f76..13eb1778f5 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -317,13 +317,19 @@ std::unique_ptr> init_fj_cpu_standalone( std::atomic& preemption_flag, fj_settings_t settings = fj_settings_t{}); +// Copies a climber that has already paid the O(nnz) problem construction. Everything the engine +// reads is host-owned, so this needs neither a problem handle nor any GPU work. template -std::unique_ptr> init_fj_cpu_standalone_from_template( - problem_t& problem, +std::unique_ptr> init_fj_cpu_clone( const fj_cpu_climber_t& tmpl, std::atomic& preemption_flag, fj_settings_t settings = fj_settings_t{}); +// Per-lane behaviour for a CPUFJ portfolio, shared by every caller that races several climbers so +// the composition cannot drift between them. +template +void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, int64_t base_seed); + // Builds the climber portfolio the standalone benchmark races: how many distinct // behaviours, what parameters each gets, whether they are randomized or // specialized. Defined in fj_cpu_portfolio.cpp -- host code, compiled by the host diff --git a/cpp/src/mip_heuristics/mip_constants.hpp b/cpp/src/mip_heuristics/mip_constants.hpp index f3fb68343a..d09fa710c9 100644 --- a/cpp/src/mip_heuristics/mip_constants.hpp +++ b/cpp/src/mip_heuristics/mip_constants.hpp @@ -21,6 +21,10 @@ #define CUOPT_MIP_BATCH_PDLP_REQUIRED_THREAD_COUNT 3 #define CUOPT_MIP_CLIQUE_CUTS_REQUIRED_THREAD_COUNT 3 +/* @brief Upper bound on the early CPUFJ climber portfolio. Every lane holds its own host copy of + * the problem and occupies an OMP task for the whole of presolve. */ +#define CUOPT_MIP_EARLY_CPUFJ_MAX_CLIMBERS 8 + // MIP-only gate: skip the concurrent barrier when fewer threads are available than this // (1 PDLP + 1 dual simplex + 1 barrier). Stand-alone LP always runs all three. #define CUOPT_CONCURRENT_LP_BARRIER_REQUIRED_THREAD_COUNT 3 From 52932910b067ee07ae918212ddc2cd9ccb57cac1 Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 07:42:50 -0700 Subject: [PATCH 40/61] lift move handle fractionals --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 34 +++++++++++-------- .../feasibility_jump/fj_cpu_binary.cu | 30 ++++++++++------ 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 10c0182bb4..6e9fc4e641 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1491,6 +1491,7 @@ static thrust::tuple find_lift_2opt_mov fj_move_t best_first = fj_move_t{-1, 0}; fj_move_t best_second = fj_move_t{-1, 0}; fj_staged_score_t best_score = fj_staged_score_t::zero(); + f_t best_improvement = 0; const i_t n_obj = (i_t)fj_cpu.h_objective_vars.size(); if (n_obj == 0) return thrust::make_tuple(best_first, best_second, best_score); @@ -1542,15 +1543,19 @@ static thrust::tuple find_lift_2opt_mov if (tabu_check(fj_cpu, var2, delta2)) continue; if (!paired_flip_keeps_feasible(fj_cpu, var1, delta1, var2, delta2)) continue; - fj_staged_score_t score = fj_staged_score_t::zero(); - score.base = round(-combined); - if (best_score < score) { - best_score = score; - best_first = fj_move_t{var1, delta1}; - best_second = fj_move_t{var2, delta2}; + // Both lift operators rank on the objective gain in its own units: the score quantization + // used elsewhere counts weights, so rounding a gain below 0.5 into it discards the move. + const f_t improvement = -combined; + if (improvement > best_improvement) { + best_improvement = improvement; + best_score.base = 1; // sign only, never compared against another operator's score + best_first = fj_move_t{var1, delta1}; + best_second = fj_move_t{var2, delta2}; } } } + cuopt_assert((best_first.var_idx < 0) == (best_improvement <= 0), + "pair and score must agree on whether a move was found"); return thrust::make_tuple(best_first, best_second, best_score); } @@ -1563,6 +1568,7 @@ static thrust::tuple find_lift_move( fj_move_t best_move = fj_move_t{-1, 0}; fj_staged_score_t best_score = fj_staged_score_t::zero(); + f_t best_improvement = 0; for (auto var_idx : fj_cpu.h_objective_vars) { cuopt_assert(var_idx < fj_cpu.h_obj_coeffs.size(), "var_idx is out of bounds"); @@ -1666,18 +1672,16 @@ static thrust::tuple find_lift_move( cuopt_assert(delta * obj_coeff < 0, "lift move doesn't improve the objective!"); - // get the score - auto move = fj_move_t{var_idx, delta}; - fj_staged_score_t score = fj_staged_score_t::zero(); - f_t obj_score = -1 * obj_coeff * delta; // negated to turn this into a positive score - score.base = round(obj_score); - - if (best_score < score) { - best_score = score; - best_move = move; + const f_t improvement = -obj_coeff * delta; + if (improvement > best_improvement) { + best_improvement = improvement; + best_score.base = 1; + best_move = fj_move_t{var_idx, delta}; } } + cuopt_assert((best_move.var_idx < 0) == (best_improvement <= 0), + "move and score must agree on whether a move was found"); return thrust::make_tuple(best_move, best_score); } diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index cdbe16b8bd..2af49ef8ff 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -1219,6 +1219,7 @@ struct fj_bin_engine_t { std::pair best_pair = {-1, -1}; int64_t best_s = 0; + double best_improvement = 0; if (pb.objective_vars.empty()) return {best_pair, best_s}; const uint32_t n_obj = (uint32_t)pb.objective_vars.size(); @@ -1258,13 +1259,18 @@ struct fj_bin_engine_t { if (tabu_blocked(var2, false)) continue; if (!paired_flip_keeps_feasible(var1, delta1, var2, delta2)) continue; - const int64_t s = (int64_t)(-std::llround(combined)) * fj_bin_score_k; - if (s > best_s) { - best_s = s; - best_pair = {var1, var2}; + // Both lift operators rank on the objective gain in its own units: the packed score counts + // weights, and this engine requires an integral matrix but not integral objective terms. + const double improvement = -combined; + if (improvement > best_improvement) { + best_improvement = improvement; + best_s = 1; // sign only, never compared against another operator's score + best_pair = {var1, var2}; } } } + cuopt_assert((best_pair.first < 0) == (best_improvement <= 0), + "pair and score must agree on whether a move was found"); return {best_pair, best_s}; } @@ -1272,20 +1278,24 @@ struct fj_bin_engine_t { { cuopt_assert(violated_list.empty(), "lift moves require a feasible incumbent"); - int32_t best_v = -1; - int64_t best_s = 0; + int32_t best_v = -1; + int64_t best_s = 0; + double best_improvement = 0; for (int32_t v : pb.objective_vars) { const int8_t delta = (int8_t)(1 - 2 * assign[v]); if ((double)delta * pb.objective[v] >= 0) continue; if (tabu_blocked(v, false)) continue; // Base field is zero iff the flip breaks no row; K/2 splits it while |bonus| < 2^31. if (var_score[v] <= -(fj_bin_score_k / 2)) continue; - const int64_t s = (int64_t)(-std::llround(pb.objective[v] * delta)) * fj_bin_score_k; - if (s > best_s) { - best_s = s; - best_v = v; + const double improvement = -pb.objective[v] * (double)delta; + if (improvement > best_improvement) { + best_improvement = improvement; + best_s = 1; + best_v = v; } } + cuopt_assert((best_v < 0) == (best_improvement <= 0), + "move and score must agree on whether a move was found"); return {best_v, best_s}; } From a8bb991278bb808e943d2b0a9415aacd7cc0f0f8 Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 08:21:04 -0700 Subject: [PATCH 41/61] vectorized argmax for binary path in feasible region --- .../feasibility_jump/fj_cpu_binary.cu | 91 ++++++++++++++++--- 1 file changed, 76 insertions(+), 15 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 2af49ef8ff..a80240f336 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -585,6 +585,13 @@ struct fj_bin_engine_t { std::vector var_score; // live feasibility score of flipping each variable std::vector nnz_score_delta; // per CSR nnz: last score delta of variables[k] in its row + // Objective half of the move score, held live so a weighted global scan can stay vectorized. + // Its support is pb.objective_vars, so entries outside that set are zero for the whole solve. + std::vector obj_base_score; + std::vector combined_score; + // Objective weight obj_base_score was built for; -1 marks it stale. + int32_t obj_base_weight{-1}; + fj_bin_tabu_t tabu; std::vector is_violated; @@ -782,23 +789,32 @@ struct fj_bin_engine_t { for (int32_t v = 0; v < pb.n_variables; ++v) incumbent_objective += pb.objective[v] * assign[v]; nnz_touched += pb.nnz; rebuild_scores(); + // Every caller of this reached it by replacing the assignment wholesale, so the cached + // per-variable flip directions no longer describe it. + obj_base_weight = -1; + } + + // Base field of the objective term: the weight, signed by the direction of the gain and scaled by + // how large that gain is against the model's typical coefficient. Depends only on the variable's + // own value and the weight, which is what lets a global scan cache it. + int64_t objective_base(int32_t v, int8_t delta) const + { + const double obj_diff = pb.objective[v] * delta; + if (obj_diff == 0) return 0; + cuopt_assert(obj_magnitude > 0, "objective magnitude unit must be positive"); + const double rel = std::fabs(obj_diff) / obj_magnitude; + const double mult = + rel < fj_obj_mult_min ? fj_obj_mult_min : (rel > fj_obj_mult_max ? fj_obj_mult_max : rel); + const double raw = objective_weight * mult; + cuopt_assert(fj_bin_in_int32(raw), "scaled objective weight out of int32 range"); + const int32_t scaled = (int32_t)std::lround(raw); + return (int64_t)(obj_diff < 0 ? scaled : -scaled) * fj_bin_score_k; } int64_t objective_terms(int32_t v, int8_t delta) const { const double obj_diff = pb.objective[v] * delta; - int32_t base = 0; - if (obj_diff != 0) { - cuopt_assert(obj_magnitude > 0, "objective magnitude unit must be positive"); - const double rel = std::fabs(obj_diff) / obj_magnitude; - const double mult = - rel < fj_obj_mult_min ? fj_obj_mult_min : (rel > fj_obj_mult_max ? fj_obj_mult_max : rel); - const double raw = objective_weight * mult; - cuopt_assert(fj_bin_in_int32(raw), "scaled objective weight out of int32 range"); - const int32_t scaled = (int32_t)std::lround(raw); - base = obj_diff < 0 ? scaled : -scaled; - } - int32_t bonus = 0; + int32_t bonus = 0; const bool old_better = incumbent_objective < best_objective; const bool new_better = incumbent_objective + obj_diff < best_objective; if (!old_better && new_better) { @@ -806,7 +822,20 @@ struct fj_bin_engine_t { } else if (old_better && !new_better) { bonus -= objective_weight; } - return (int64_t)base * fj_bin_score_k + bonus; + return objective_base(v, delta) + bonus; + } + + int64_t flip_objective_base(int32_t v) const + { + return objective_base(v, (int8_t)(1 - 2 * assign[v])); + } + + // Only the objective variables are written: the rest of the array is zero from init onwards. + void ensure_objective_base() + { + if (obj_base_weight == objective_weight) return; + for (int32_t v : pb.objective_vars) obj_base_score[v] = flip_objective_base(v); + obj_base_weight = objective_weight; } int64_t full_score(int32_t v, int8_t delta) const @@ -913,6 +942,9 @@ struct fj_bin_engine_t { assign_i32[var] = new_val; var_score[var] = own_score; incumbent_objective += pb.objective[var] * delta; + // Only this variable's flip direction moved, so a live cache needs one entry rewritten. + if (obj_base_weight == objective_weight && pb.objective[var] != 0) + obj_base_score[var] = flip_objective_base(var); if (violated_list.empty() && incumbent_objective < best_objective) { best_objective = incumbent_objective; @@ -1089,8 +1121,8 @@ struct fj_bin_engine_t { } // Global argmax over every variable, affordable because var_score is maintained live. While the - // objective weight is zero the full score is exactly var_score, which is the vectorized sweep's - // precondition; the objective and local-minimum paths fall to the scalar loop. + // objective weight is zero the full score is exactly var_score; above zero the sweep runs over + // var_score plus the cached objective base. Only the local-minimum path falls to the scalar loop. std::pair find_move_global(bool localmin) { if (!localmin && objective_weight == 0) { @@ -1108,6 +1140,31 @@ struct fj_bin_engine_t { return {v, s}; } + if (!localmin) { + // The breakthrough bonus is deliberately absent from the ranking: it depends on + // incumbent_objective, so no per-variable form of it survives a move, and it occupies the low + // field where it can only separate variables already tied on the base. The winner's score is + // then taken from full_score so the caller sees the true value. + ensure_objective_base(); + const int64_t* const obj_p = obj_base_score.data(); + const int64_t* const var_p = var_score.data(); + int64_t* const comb_p = combined_score.data(); + for (int32_t v = 0; v < pb.n_variables; ++v) + comb_p[v] = var_p[v] + obj_p[v]; + + int32_t saved_var[fj_bin_tabu_t::ring_size]; + int64_t saved_score[fj_bin_tabu_t::ring_size]; + const int32_t blocked = tabu.block_tabu(iters, comb_p, saved_var, saved_score); + + int32_t v = -1; + int64_t s = fj_bin_score_invalid; + fj_bin_argmax(comb_p, pb.n_variables, argmax_tile, v, s); + + fj_bin_tabu_t::unblock_tabu(blocked, comb_p, saved_var, saved_score); + if (v >= 0) s = full_score(v, (int8_t)(1 - 2 * assign[v])); + return {v, s}; + } + int32_t best_v = -1; int64_t best_s = fj_bin_score_invalid; for (int32_t v = 0; v < pb.n_variables; ++v) { @@ -1411,6 +1468,10 @@ struct fj_bin_engine_t { var_score.assign(n, 0); nnz_score_delta.assign(pb.nnz + fj_bin_simd_padding, 0); + // Zeroed once: ensure_objective_base only ever rewrites the objective variables. + obj_base_score.assign(n, 0); + combined_score.assign(n, 0); + obj_base_weight = -1; tabu.resize(n); is_violated.assign(m, 0); vpos.assign(m, -1); From 868142c03aaa8d90e16377d0d0b6c14ba5ef549e Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 09:12:20 -0700 Subject: [PATCH 42/61] kahan sum the objective; audit validity in the harness --- .../linear_programming/cuopt/run_cpufj.cu | 105 ++++++++++++++++++ .../mip_heuristics/feasibility_jump/fj_cpu.cu | 63 ++++++----- .../feasibility_jump/fj_cpu.cuh | 3 + 3 files changed, 142 insertions(+), 29 deletions(-) diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu index 781c28236c..1ab13ca28b 100644 --- a/benchmarks/linear_programming/cuopt/run_cpufj.cu +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -5,6 +5,8 @@ */ /* clang-format on */ +#include "miplib2017_bks.hpp" + #include #include #include @@ -19,7 +21,9 @@ #include #include +#include #include +#include #include #include #include @@ -203,6 +207,107 @@ int main(int argc, char** argv) r.iterations, r.seconds > 0 ? r.iterations / r.seconds : 0.0); } + // Runs after the measured window closes, so its cost is off the clock. + // Solver space is always a minimisation, so beating the best known is always a smaller value. + const auto bks_user = cuopt_bench::lookup_miplib_bks(path); + const double bks = bks_user ? (double)problem.get_solver_obj_from_user_obj((f_t)*bks_user) : 0.0; + const double bks_slack = std::max(1e-6, std::fabs(bks) * 1e-9); + + int audited = 0, invalid = 0; + std::printf("\n climber | viol rows worst/tol | bnd viol worst/tol | int viol worst/tol |" + " obj drift rel | vs bks\n"); + std::printf("---------+----------------------+---------------------+---------------------+" + "----------------------+----------\n"); + for (int k = 0; k < n_climbers; ++k) { + auto& c = *climbers[k]; + if (c.feasible_found != results[k].crossed) { + std::printf(" %7d | feasible_found=%d disagrees with a reported incumbent=%d\n", + k, + (int)c.feasible_found, + (int)results[k].crossed); + ++invalid; + continue; + } + if (!c.feasible_found) continue; + ++audited; + + const double int_tol = c.view.pb.tolerances.integrality_tolerance; + + i_t rows_over = 0; + double worst_row_ratio = 0.0; + for (i_t r = 0; r < c.view.pb.n_constraints; ++r) { + __float128 activity = 0; + for (i_t j = c.h_offsets[r]; j < c.h_offsets[r + 1]; ++j) { + const i_t var = c.h_variables[j]; + const double coefficient = c.h_coefficients[j]; + const double value = c.h_best_assignment[var]; + activity += (__float128)coefficient * (__float128)value; + } + + const f_t lb = c.h_cstr_lb[r]; + const f_t ub = c.h_cstr_ub[r]; + const __float128 below = (__float128)lb - activity; + const __float128 above = activity - (__float128)ub; + const double excess = (double)std::max(std::max(below, above), (__float128)0); + if (excess <= 0.0) continue; + + const double tol = c.view.get_corrected_tolerance(r, lb, ub); + const double ratio = tol > 0 ? excess / tol : std::numeric_limits::infinity(); + if (ratio > 1.0) ++rows_over; + worst_row_ratio = std::max(worst_row_ratio, ratio); + } + + i_t bounds_over = 0; + i_t integers_over = 0; + double worst_bound_ratio = 0.0; + double worst_integer_ratio = 0.0; + __float128 objective = 0; + for (i_t v = 0; v < c.view.pb.n_variables; ++v) { + auto bounds = c.h_var_bounds[v].get(); + const double x = (double)c.h_best_assignment[v]; + const double out = std::max( + std::max((double)cuopt::get_lower(bounds) - x, x - (double)cuopt::get_upper(bounds)), 0.0); + if (out > int_tol) ++bounds_over; + worst_bound_ratio = std::max(worst_bound_ratio, int_tol > 0 ? out / int_tol : 0.0); + + if (c.view.pb.is_integer_var(v)) { + const double residual = std::fabs(x - std::round(x)); + if (residual > int_tol) ++integers_over; + worst_integer_ratio = std::max(worst_integer_ratio, int_tol > 0 ? residual / int_tol : 0.0); + } + const double coefficient = c.h_obj_coeffs[v]; + objective += (__float128)coefficient * (__float128)x; + } + + // Differenced before narrowing; the drift is smaller than a double ulp of the sum. + const __float128 difference = objective - (__float128)results[k].best_objective; + const double drift = (double)(difference < 0 ? -difference : difference); + const double exact = (double)objective; + const double scale = std::max(std::fabs(exact), 1.0); + const bool below_bks = bks_user && exact < bks - bks_slack; + const bool bad = rows_over > 0 || bounds_over > 0 || integers_over > 0 || below_bks; + if (bad) ++invalid; + std::printf(" %7d | %9d %10.3g | %8d %10.3g | %8d %10.3g | %12.3g %6.1e | %9.3g%s%s\n", + k, + rows_over, + worst_row_ratio, + bounds_over, + worst_bound_ratio, + integers_over, + worst_integer_ratio, + drift, + drift / scale, + bks_user ? exact - bks : 0.0, + below_bks ? " BELOW BKS" : "", + bad ? " INVALID" : ""); + } + std::printf("AUDIT: %d/%d reporting climbers checked, %d invalid, bks %s\n", + audited, + crossed, + invalid, + bks_user ? std::to_string(*bks_user).c_str() + : (cuopt_bench::is_known_infeasible(path) ? "known infeasible" : "unknown")); + std::printf("\nSUMMARY: %d/%d crossed (%.0f%%) wall=%.1fs total_iters=%.0f agg_iters/s=%.0f\n", crossed, n_climbers, diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 6e9fc4e641..8c044a1870 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1118,36 +1118,39 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, // update the assignment and objective proper fj_cpu.h_assignment[var_idx] = new_val; - fj_cpu.h_incumbent_objective += fj_cpu.h_obj_coeffs[var_idx] * delta; - if (fj_cpu.h_incumbent_objective < fj_cpu.h_best_objective && - fj_cpu.violated_constraints.empty()) { - // recompute the LHS values to cancel out accumulation errors, then check if feasibility remains - recompute_lhs(fj_cpu); - if (fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { - cuopt_assert(fj_cpu.satisfied_constraints.size() == fj_cpu.view.pb.n_constraints, ""); - fj_cpu.h_best_objective = - fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; - fj_cpu.h_best_assignment = fj_cpu.h_assignment; - fj_cpu.iterations_since_best = 0; - // DEBUG, and reporting the stored best rather than the pre-epsilon incumbent, - // so it matches the binary path and the end-of-solve incumbent audit. - CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", - fj_cpu.log_prefix.c_str(), - fj_cpu.h_best_objective); - if (fj_cpu.improvement_callback) { - double current_work_units = fj_cpu.work_units_elapsed.load(std::memory_order_acquire); - fj_cpu.improvement_callback( - fj_cpu.h_incumbent_objective, fj_cpu.h_assignment, current_work_units); - } - fj_cpu.feasible_found = true; - // Counteract the smooth_weights decay for a lane that is actively improving, and hold the - // weight at a scale where base_feas_sum still registers against it. - if (fj_cpu.h_objective_weight > 0) { - fj_cpu.h_objective_weight = - min((f_t)fj_obj_weight_incumbent_cap, - fj_cpu.h_objective_weight + (f_t)fj_obj_weight_incumbent_bump); - } + // Kahan compensated summation, as for h_lhs. The incumbent objective is reported as-is, so it + // cannot carry the drift of a long uncompensated chain of deltas. + const f_t obj_old = fj_cpu.h_incumbent_objective; + const f_t obj_y = fj_cpu.h_obj_coeffs[var_idx] * delta - fj_cpu.h_objective_sumcomp; + const f_t obj_t = obj_old + obj_y; + fj_cpu.h_objective_sumcomp = (obj_t - obj_old) - obj_y; + fj_cpu.h_incumbent_objective = obj_t; + + if (fj_cpu.h_incumbent_objective < fj_cpu.h_best_objective && + fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { + cuopt_assert(fj_cpu.satisfied_constraints.size() == fj_cpu.view.pb.n_constraints, ""); + fj_cpu.h_best_objective = + fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; + fj_cpu.h_best_assignment = fj_cpu.h_assignment; + fj_cpu.iterations_since_best = 0; + // DEBUG, and reporting the stored best rather than the pre-epsilon incumbent, + // so it matches the binary path and the end-of-solve incumbent audit. + CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", + fj_cpu.log_prefix.c_str(), + fj_cpu.h_best_objective); + if (fj_cpu.improvement_callback) { + double current_work_units = fj_cpu.work_units_elapsed.load(std::memory_order_acquire); + fj_cpu.improvement_callback( + fj_cpu.h_incumbent_objective, fj_cpu.h_assignment, current_work_units); + } + fj_cpu.feasible_found = true; + // Counteract the smooth_weights decay for a lane that is actively improving, and hold the + // weight at a scale where base_feas_sum still registers against it. + if (fj_cpu.h_objective_weight > 0) { + fj_cpu.h_objective_weight = + min((f_t)fj_obj_weight_incumbent_cap, + fj_cpu.h_objective_weight + (f_t)fj_obj_weight_incumbent_bump); } } @@ -1432,6 +1435,7 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) // compute incumbent objective fj_cpu.h_incumbent_objective = thrust::inner_product( fj_cpu.h_assignment.begin(), fj_cpu.h_assignment.end(), fj_cpu.h_obj_coeffs.begin(), 0.); + fj_cpu.h_objective_sumcomp = 0; } @@ -2151,6 +2155,7 @@ static void finalize_fj_cpu_host_initialization_from_template( fj_cpu.satisfied_constraints = tmpl.satisfied_constraints; fj_cpu.total_violations = tmpl.total_violations; fj_cpu.h_incumbent_objective = tmpl.h_incumbent_objective; + fj_cpu.h_objective_sumcomp = tmpl.h_objective_sumcomp; fj_cpu.n_binary_vars = tmpl.n_binary_vars; fj_cpu.n_integer_vars = tmpl.n_integer_vars; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 13eb1778f5..0ebe059d1d 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -188,6 +188,9 @@ struct fj_cpu_climber_t { // Mean absolute nonzero objective coefficient; the unit of the objective score term. f_t obj_magnitude{1}; f_t h_incumbent_objective; + // Kahan compensation for h_incumbent_objective, mirroring h_lhs_sumcomp. Reset wherever the + // objective is re-derived from the assignment. + f_t h_objective_sumcomp{0}; f_t h_best_objective; i_t last_feasible_entrance_iter{0}; i_t iterations; From a2ccbc980cf54bb9029d7fbf6e322cda72211221 Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 09:49:01 -0700 Subject: [PATCH 43/61] Binary perturb re-anchors to the incumbent --- cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index a80240f336..b456559192 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -1392,6 +1392,11 @@ struct fj_bin_engine_t { void perturb() { if (pb.objective_vars.empty()) return; + if (feasible_found) { + cuopt_assert((int32_t)best_assign.size() == pb.n_variables, "incumbent size mismatch"); + assign = best_assign; + for (int32_t v = 0; v < pb.n_variables; ++v) assign_i32[v] = assign[v]; + } const uint32_t n = (uint32_t)pb.objective_vars.size(); for (int i = 0; i < 2; ++i) { const int32_t v = pb.objective_vars[rng.next_u32() % n]; From e02bd00b425b45abc382153a0e0a01c7c6ac93a5 Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 11:13:05 -0700 Subject: [PATCH 44/61] apply_exact_k_seed --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 8c044a1870..36c529d12e 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -2922,6 +2922,14 @@ template void finalize_fj_cpu_host_initialization( // Above this the O(nnz) seed passes eat a meaningful slice of a short budget, so they are skipped. constexpr int64_t fj_seed_nnz_limit = 8'000'000; +// Cardinality-row detection: coefficient agreement tolerance and the widest row worth peeling. +constexpr double fj_exact_k_tol = 1e-6; +constexpr int32_t fj_exact_k_max_width = 20000; +constexpr double fj_exact_k_budget_s = 0.5; +// The anchor repair only runs when this fraction of the rows is violated, and gets this long. +constexpr int32_t fj_anchor_repair_violated_share = 5; +constexpr double fj_anchor_repair_budget_s = 0.1; + // Jumps each two-sided variable to whichever bound has fewer rows locking it in that direction. template static void apply_lock_weighted_seed(fj_cpu_climber_t& fj_cpu) @@ -3114,6 +3122,160 @@ static void apply_greedy_covering_seed(fj_cpu_climber_t& fj_cpu) fj_cpu.h_best_assignment = fj_cpu.h_assignment; } +// Constructively satisfies the equality rows that read as sum(x) = k over binaries sharing one +// coefficient: pick k members of each, narrowest rows first so the wide ones inherit the choices, +// and within a row the variables appearing in fewest other such rows. +template +static void apply_exact_k_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const auto started = std::chrono::steady_clock::now(); + auto timed_out = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_exact_k_budget_s; + }; + + struct exact_k_row_t { + i_t k, begin, end; + }; + std::vector rows; + for (i_t row = 0; row < fj_cpu.view.pb.n_constraints; ++row) { + if ((row & 0xFFF) == 0 && timed_out()) return; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + if (!isfinite(lb) || !isfinite(ub) || std::abs(lb - ub) > fj_exact_k_tol) continue; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + if (end - begin < 2 || end - begin > fj_exact_k_max_width) continue; + + const f_t scale = fj_cpu.h_coefficients[begin]; + if (scale <= 0) continue; + bool uniform_binary = true; + for (i_t p = begin; p < end && uniform_binary; ++p) { + const i_t var = fj_cpu.h_variables[p]; + const f_t coeff = fj_cpu.h_coefficients[p]; + const f_t agreement = fj_exact_k_tol * std::max((f_t)1, std::abs(scale)); + uniform_binary = + fj_cpu.h_is_binary_variable[var] && coeff > 0 && std::abs(coeff - scale) <= agreement; + } + if (!uniform_binary) continue; + + const double cardinality = (double)lb / scale; + const i_t k = (i_t)std::lround(cardinality); + if (std::abs(cardinality - k) <= 1e-4 && k >= 0 && k <= end - begin) + rows.push_back({k, begin, end}); + } + if (rows.empty()) return; + + std::sort(rows.begin(), rows.end(), [](const exact_k_row_t& a, const exact_k_row_t& b) { + return a.end - a.begin < b.end - b.begin; + }); + + const i_t n_variables = fj_cpu.view.pb.n_variables; + std::vector degree(n_variables, 0); + for (const auto& row : rows) + for (i_t p = row.begin; p < row.end; ++p) + ++degree[fj_cpu.h_variables[p]]; + + std::vector state(n_variables, -1); + std::vector free_vars; + for (size_t index = 0; index < rows.size(); ++index) { + if ((index & 0xFFF) == 0 && timed_out()) break; + const auto& row = rows[index]; + + i_t selected = 0; + free_vars.clear(); + for (i_t p = row.begin; p < row.end; ++p) { + const i_t var = fj_cpu.h_variables[p]; + selected += state[var] == 1; + if (state[var] < 0) free_vars.push_back(var); + } + const i_t needed = row.k - selected; + if (needed < 0 || (i_t)free_vars.size() < needed) continue; + + std::sort(free_vars.begin(), free_vars.end(), [°ree](i_t a, i_t b) { + return degree[a] < degree[b]; + }); + for (i_t p = 0; p < (i_t)free_vars.size(); ++p) + state[free_vars[p]] = (int8_t)(p < needed); + } + + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + const auto anchor = fj_cpu.h_assignment; + for (i_t var = 0; var < n_variables; ++var) + if (state[var] >= 0) fj_cpu.h_assignment[var] = state[var]; + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// One repair pass over the violated rows of a start that is mostly violated. Row sums are read from +// the lhs computed on entry, so a row does not see the repairs made for earlier rows; the revert +// below is what keeps that myopia from costing anything. +template +static void repair_difficult_anchor(fj_cpu_climber_t& fj_cpu) +{ + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + if (baseline == 0 || baseline <= fj_cpu.view.pb.n_constraints / fj_anchor_repair_violated_share) + return; + + const auto started = std::chrono::steady_clock::now(); + const auto anchor = fj_cpu.h_assignment; + const std::vector violated(fj_cpu.violated_constraints.begin(), + fj_cpu.violated_constraints.end()); + std::vector> candidates; + + for (i_t row : violated) { + if (std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_anchor_repair_budget_s) + break; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + f_t sum = fj_cpu.h_lhs[row]; + f_t target = 0; + f_t direction = 0; + if (sum < lb) { + direction = 1; + target = lb; + } else if (sum > ub) { + direction = -1; + target = ub; + } else { + continue; + } + + collect_row_repair_moves(fj_cpu, + fj_cpu.h_offsets[row], + fj_cpu.h_offsets[row + 1], + direction, + fj_exact_k_tol, + candidates); + for (const auto& move : candidates) { + if (direction > 0 ? sum >= target : sum <= target) break; + const f_t delta = move.new_val - (f_t)fj_cpu.h_assignment[move.var]; + sum += move.coeff * delta; + fj_cpu.h_assignment[move.var] = move.new_val; + } + } + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + // What makes one lane of a CPUFJ portfolio behave differently from another: which corner it starts // from, how it samples, and how hard it pulls on the objective. Lane 0 keeps the anchor assignment // so it is the lane every clone is built from. @@ -3175,6 +3337,9 @@ void build_climber_portfolio(problem_t& problem, fj_settings_t settings; settings.seed = (int)lane_seed[0]; climbers[0] = init_fj_cpu_standalone(problem, solution, preemption_flags[0], settings); + // Runs before the clones are taken, so every lane starts from the repaired anchor. + apply_exact_k_seed(*climbers[0]); + repair_difficult_anchor(*climbers[0]); apply_lane_diversification(*climbers[0], 0, base_seed); } From 0e4453f3a10328b0ccb64c631476e91f90ef241d Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 11:33:47 -0700 Subject: [PATCH 45/61] more seeds --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 114 ++++++++++++++++-- 1 file changed, 107 insertions(+), 7 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 36c529d12e..21351c1736 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -2922,6 +2922,11 @@ template void finalize_fj_cpu_host_initialization( // Above this the O(nnz) seed passes eat a meaningful slice of a short budget, so they are skipped. constexpr int64_t fj_seed_nnz_limit = 8'000'000; +// The aggressive corner pushes harder than the covering seed: more passes, a longer budget, a +// tighter clock, and it gives up as soon as a pass changes nothing. +constexpr int32_t fj_aggressive_passes = 6; +constexpr double fj_aggressive_budget_s = 0.9; + // Cardinality-row detection: coefficient agreement tolerance and the widest row worth peeling. constexpr double fj_exact_k_tol = 1e-6; constexpr int32_t fj_exact_k_max_width = 20000; @@ -3122,6 +3127,97 @@ static void apply_greedy_covering_seed(fj_cpu_climber_t& fj_cpu) fj_cpu.h_best_assignment = fj_cpu.h_assignment; } +// Repeated one-sided row repair in CSR order. Unlike the covering seed it revisits rows until a +// pass changes nothing, so a repair that breaks a row already visited gets another chance. +template +static void apply_aggressive_constraint_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const auto started = std::chrono::steady_clock::now(); + auto timed_out = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_aggressive_budget_s; + }; + + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + const auto anchor = fj_cpu.h_assignment; + const f_t tol = 1e-6; + std::vector> candidates; + + for (i_t pass = 0; pass < fj_aggressive_passes && !timed_out(); ++pass) { + i_t moves = 0; + for (i_t row = 0; row < fj_cpu.view.pb.n_constraints; ++row) { + if ((row & 0xFF) == 0 && timed_out()) break; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + if (begin == end) continue; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + const bool has_lb = isfinite(lb); + const bool has_ub = isfinite(ub); + if (!has_lb && !has_ub) continue; + + f_t sum = 0; + for (i_t p = begin; p < end; ++p) + sum += (f_t)fj_cpu.h_coefficients[p] * (f_t)fj_cpu.h_assignment[fj_cpu.h_variables[p]]; + + f_t direction = 0; + f_t target = 0; + if (has_lb && sum < lb - tol) { + direction = 1; + target = lb; + } else if (has_ub && sum > ub + tol) { + direction = -1; + target = ub; + } else { + continue; + } + + collect_row_repair_moves(fj_cpu, begin, end, direction, tol, candidates); + for (const auto& move : candidates) { + if (direction > 0 ? sum >= target - tol : sum <= target + tol) break; + const f_t delta = move.new_val - (f_t)fj_cpu.h_assignment[move.var]; + sum += move.coeff * delta; + fj_cpu.h_assignment[move.var] = move.new_val; + ++moves; + } + } + if (moves == 0) break; + } + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// Every variable to its lower bound, or its upper where the lower is infinite. +template +static void apply_lower_bound_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) { + auto bounds = fj_cpu.h_var_bounds[var_idx].get(); + const f_t lower = get_lower(bounds); + const f_t upper = get_upper(bounds); + if (!isfinite(lower) && !isfinite(upper)) continue; + + f_t new_val = isfinite(lower) ? lower : upper; + if (is_integer_var(fj_cpu, var_idx)) new_val = std::round(new_val); + fj_cpu.h_assignment[var_idx] = new_val; + } + + recompute_lhs(fj_cpu); + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + // Constructively satisfies the equality rows that read as sum(x) = k over binaries sharing one // coefficient: pick k members of each, narrowest rows first so the wide ones inherit the choices, // and within a row the variables appearing in fewest other such rows. @@ -3289,13 +3385,17 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; const f_t obj_weight_floor[4] = {1, 4, 32, 1}; - // Fewer-lock corner, objective-favorable corner, greedy row repair; lane 0 keeps the anchor. - if (lane % 4 == 1) - apply_lock_weighted_seed(climber); - else if (lane % 4 == 2) - apply_objective_corner_seed(climber); - else if (lane % 4 == 3) - apply_greedy_covering_seed(climber); + // One structural start per lane. Lanes 0 and 4 keep the shared anchor, and lane 5 doubles the + // fewer-lock corner until it has a seed of its own. + switch (lane % 8) { + case 1: apply_lock_weighted_seed(climber); break; + case 2: apply_aggressive_constraint_seed(climber); break; + case 3: apply_greedy_covering_seed(climber); break; + case 5: apply_lock_weighted_seed(climber); break; + case 6: apply_objective_corner_seed(climber); break; + case 7: apply_lower_bound_seed(climber); break; + default: break; + } // Default: every climber identical apart from its seed and a random draw of the // four sampling parameters. Diversification, decorrelated from the value RNG. From 6c8e2ce6979c7ef0f0e20061a4c52d6f9dfce1ee Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 12:25:41 -0700 Subject: [PATCH 46/61] bipartite seeding --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 240 +++++++++++++++++- 1 file changed, 237 insertions(+), 3 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 21351c1736..8fafbfa8f4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -2922,6 +2923,10 @@ template void finalize_fj_cpu_host_initialization( // Above this the O(nnz) seed passes eat a meaningful slice of a short budget, so they are skipped. constexpr int64_t fj_seed_nnz_limit = 8'000'000; +// Budget for the matching seed and the widest exact-one row it will take into the graph. +constexpr double fj_matching_budget_s = 0.45; +constexpr int32_t fj_matching_max_row_width = 20000; + // The aggressive corner pushes harder than the covering seed: more passes, a longer budget, a // tighter clock, and it gives up as soon as a pass changes nothing. constexpr int32_t fj_aggressive_passes = 6; @@ -3197,6 +3202,236 @@ static void apply_aggressive_constraint_seed(fj_cpu_climber_t& fj_cpu) fj_cpu.h_best_assignment = fj_cpu.h_assignment; } +// Treats the exact-one rows as a graph in which each variable is an edge between the two rows it +// appears in. A component that is bipartite and has equally many rows on each side admits a perfect +// matching, and the cheapest one is the assignment satisfying every row in the component at least +// cost. Solved per component as min-cost flow by successive shortest paths, which needs no +// potentials here because augmenting along shortest paths keeps the residual free of negative +// cycles. Components that are not of that shape are left to the search. +template +static void apply_bipartite_matching_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const auto started = std::chrono::steady_clock::now(); + auto timed_out = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_matching_budget_s; + }; + const f_t tol = 1e-6; + + struct exact_one_row_t { + i_t begin, end; + }; + std::vector rows; + for (i_t row = 0; row < fj_cpu.view.pb.n_constraints; ++row) { + if ((row & 0xFFF) == 0 && timed_out()) return; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + if (!isfinite(lb) || !isfinite(ub) || std::abs(lb - ub) > tol) continue; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + if (begin == end || end - begin > fj_matching_max_row_width) continue; + + const f_t scale = fj_cpu.h_coefficients[begin]; + if (!isfinite(scale) || std::abs(scale) <= tol || std::abs(lb / scale - 1) > 1e-5) continue; + + bool uniform_binary = true; + for (i_t p = begin; p < end && uniform_binary; ++p) { + const f_t coeff = fj_cpu.h_coefficients[p]; + const f_t agreement = tol * std::max((f_t)1, std::abs(scale)); + uniform_binary = fj_cpu.h_is_binary_variable[fj_cpu.h_variables[p]] && + std::abs(coeff - scale) <= agreement; + } + if (uniform_binary) rows.push_back({begin, end}); + } + if (rows.size() < 2 || timed_out()) return; + + const i_t n_rows = (i_t)rows.size(); + const i_t n_variables = fj_cpu.view.pb.n_variables; + std::vector degree(n_variables, 0); + std::vector endpoint_a(n_variables, -1); + std::vector endpoint_b(n_variables, -1); + for (i_t row = 0; row < n_rows; ++row) { + for (i_t p = rows[row].begin; p < rows[row].end; ++p) { + const i_t var = fj_cpu.h_variables[p]; + if (degree[var] == 0) endpoint_a[var] = row; + else if (degree[var] == 1) endpoint_b[var] = row; + ++degree[var]; + } + } + + struct edge_t { + int to, reverse, capacity; + f_t cost; + i_t var; + }; + auto add_edge = [](std::vector>& graph, int from, int to, f_t cost, i_t var) { + const int back = (int)graph[to].size(); + graph[from].push_back({to, back, 1, cost, var}); + graph[to].push_back({from, (int)graph[from].size() - 1, 0, -cost, -1}); + }; + + std::vector color(n_rows, -1); + std::vector state(n_variables, -1); + std::vector side_index(n_rows, -1); + std::vector component_rows, component_vars, left, right; + std::queue pending; + bool installed = false; + + for (i_t root = 0; root < n_rows && !timed_out(); ++root) { + if (color[root] >= 0) continue; + + component_rows.clear(); + component_vars.clear(); + color[root] = 0; + pending.push(root); + bool valid = true; + while (!pending.empty()) { + const i_t row = pending.front(); + pending.pop(); + component_rows.push_back(row); + for (i_t p = rows[row].begin; p < rows[row].end; ++p) { + const i_t var = fj_cpu.h_variables[p]; + // A variable outside exactly two rows is not an edge, and a self-loop cannot be 2-coloured. + if (degree[var] != 2 || endpoint_a[var] == endpoint_b[var]) { + valid = false; + continue; + } + if (endpoint_a[var] == row) component_vars.push_back(var); + const i_t other = endpoint_a[var] == row ? endpoint_b[var] : endpoint_a[var]; + if (color[other] < 0) { + color[other] = 1 - color[row]; + pending.push(other); + } else if (color[other] == color[row]) { + valid = false; + } + } + if ((component_rows.size() & 0x3FF) == 0 && timed_out()) return; + } + if (!valid || component_vars.empty()) continue; + + left.clear(); + right.clear(); + for (i_t row : component_rows) + (color[row] == 0 ? left : right).push_back(row); + if (left.size() != right.size()) continue; + for (i_t k = 0; k < (i_t)left.size(); ++k) + side_index[left[k]] = k; + for (i_t k = 0; k < (i_t)right.size(); ++k) + side_index[right[k]] = k; + + const int side = (int)left.size(); + const int source = 2 * side; + const int sink = source + 1; + std::vector> graph(sink + 1); + for (int k = 0; k < side; ++k) { + add_edge(graph, source, k, 0, -1); + add_edge(graph, side + k, sink, 0, -1); + } + // Every perfect matching uses exactly one variable edge per row, so shifting all of them by a + // constant moves every matching's cost equally and leaves the cheapest one unchanged. Shifting + // the negatives away is what lets the potentials below start at zero. + f_t cheapest = 0; + for (i_t var : component_vars) { + const f_t cost = fj_cpu.h_obj_coeffs[var]; + if (!isfinite(cost)) { + valid = false; + break; + } + cheapest = std::min(cheapest, cost); + } + if (!valid) continue; + const f_t shift = -cheapest; + + for (i_t var : component_vars) { + i_t a = endpoint_a[var]; + i_t b = endpoint_b[var]; + if (color[a] == 1) std::swap(a, b); + add_edge(graph, side_index[a], side + side_index[b], fj_cpu.h_obj_coeffs[var] + shift, var); + } + + // Node potentials hold every reduced cost at or above zero, which is what makes Dijkstra + // applicable. All shifted costs start non-negative, so the potentials start at zero. Rounding + // can still leave a tree edge fractionally negative once the potentials move, so relaxation + // below skips settled nodes: that keeps every predecessor older than its successor in + // settlement order, which is what makes the retrace terminate. + int flow = 0; + std::vector potential(graph.size(), 0); + std::vector distance(graph.size()); + std::vector previous_node(graph.size()); + std::vector previous_edge(graph.size()); + std::vector settled(graph.size()); + using heap_entry_t = std::pair; + + while (flow < side && !timed_out()) { + std::fill(distance.begin(), distance.end(), std::numeric_limits::infinity()); + std::fill(previous_node.begin(), previous_node.end(), -1); + std::fill(settled.begin(), settled.end(), 0); + distance[source] = 0; + std::priority_queue, std::greater> heap; + heap.push({0, source}); + + while (!heap.empty()) { + const auto [reached_at, from] = heap.top(); + heap.pop(); + if (settled[from]) continue; + settled[from] = 1; + for (int e = 0; e < (int)graph[from].size(); ++e) { + const auto& edge = graph[from][e]; + if (!edge.capacity || settled[edge.to]) continue; + const f_t reduced = edge.cost + potential[from] - potential[edge.to]; + cuopt_assert(reduced >= -1e-9 * std::max((f_t)1, std::abs(edge.cost)), + "potentials failed to keep the reduced cost non-negative"); + if (reached_at + reduced >= distance[edge.to]) continue; + distance[edge.to] = reached_at + reduced; + previous_node[edge.to] = from; + previous_edge[edge.to] = e; + heap.push({distance[edge.to], edge.to}); + } + } + if (previous_node[sink] < 0) break; + + for (int node = 0; node < (int)graph.size(); ++node) + if (isfinite(distance[node])) potential[node] += distance[node]; + + for (int node = sink; node != source; node = previous_node[node]) { + cuopt_assert(previous_node[node] >= 0, "augmenting path is broken"); + auto& edge = graph[previous_node[node]][previous_edge[node]]; + --edge.capacity; + ++graph[node][edge.reverse].capacity; + } + ++flow; + } + if (flow != side) continue; + + for (i_t var : component_vars) + state[var] = 0; + for (int node = 0; node < side; ++node) + for (const auto& edge : graph[node]) + if (edge.var >= 0 && edge.capacity == 0) state[edge.var] = 1; + installed = true; + } + if (!installed) return; + + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + const auto anchor = fj_cpu.h_assignment; + for (i_t var = 0; var < n_variables; ++var) + if (state[var] >= 0) fj_cpu.h_assignment[var] = state[var]; + + recompute_lhs(fj_cpu); + const i_t candidate = fj_cpu.violated_constraints.size(); + // Kept when it reaches feasibility outright, otherwise only on a strict gain. + if (candidate != 0 && candidate >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + // Every variable to its lower bound, or its upper where the lower is infinite. template static void apply_lower_bound_seed(fj_cpu_climber_t& fj_cpu) @@ -3385,13 +3620,12 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; const f_t obj_weight_floor[4] = {1, 4, 32, 1}; - // One structural start per lane. Lanes 0 and 4 keep the shared anchor, and lane 5 doubles the - // fewer-lock corner until it has a seed of its own. + // One structural start per lane; lanes 0 and 4 keep the shared anchor. switch (lane % 8) { case 1: apply_lock_weighted_seed(climber); break; case 2: apply_aggressive_constraint_seed(climber); break; case 3: apply_greedy_covering_seed(climber); break; - case 5: apply_lock_weighted_seed(climber); break; + case 5: apply_bipartite_matching_seed(climber); break; case 6: apply_objective_corner_seed(climber); break; case 7: apply_lower_bound_seed(climber); break; default: break; From 32b6b522740b0bbe54cfb968719cfa37dc350287 Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 12:45:05 -0700 Subject: [PATCH 47/61] share incumbent in portfolio --- .../feasibility_jump/early_cpufj.cu | 4 ++ .../mip_heuristics/feasibility_jump/fj_cpu.cu | 12 ++++++ .../feasibility_jump/fj_cpu.cuh | 38 +++++++++++++++++++ .../feasibility_jump/fj_cpu_binary.cu | 14 ++++++- 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu index 47a3711fb5..8b1444d8a1 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu @@ -71,6 +71,10 @@ void early_cpufj_t::start() climbers_[k]->improvement_callback = report_incumbent; } + auto shared = std::make_shared>(); + for (int k = 0; k < n_lanes; ++k) + climbers_[k]->shared_incumbent = shared; + CUOPT_LOG_DEBUG("Launching %d early CPUFJ tasks", n_lanes); for (int k = 0; k < n_lanes; ++k) { auto* climber = climbers_[k].get(); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 8fafbfa8f4..ca66be9188 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1146,6 +1146,11 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.h_incumbent_objective, fj_cpu.h_assignment, current_work_units); } fj_cpu.feasible_found = true; + // The true objective of the assignment, not the epsilon-reduced threshold stored above, so + // another lane comparing against it is not misled into adopting something no better. + if (fj_cpu.shared_incumbent) { + fj_cpu.shared_incumbent->publish(fj_cpu.h_incumbent_objective, fj_cpu.h_assignment); + } // Counteract the smooth_weights decay for a lane that is actively improving, and hold the // weight at a scale where base_feas_sum still registers against it. if (fj_cpu.h_objective_weight > 0) { @@ -1719,6 +1724,9 @@ static void perturb(fj_cpu_climber_t& fj_cpu) cuopt_assert(fj_cpu.h_assignment.size() == fj_cpu.h_best_assignment.size(), "incumbent_assignment span would be invalidated"); fj_cpu.h_assignment = fj_cpu.h_best_assignment; + if (fj_cpu.shared_incumbent) { + fj_cpu.shared_incumbent->adopt(fj_cpu.h_best_objective, fj_cpu.h_assignment); + } } // select N variables, assign them a random value between their bounds @@ -3688,6 +3696,10 @@ void build_climber_portfolio(problem_t& problem, climbers[k] = init_fj_cpu_clone(*climbers[0], preemption_flags[k], settings); apply_lane_diversification(*climbers[k], k, base_seed); } + + auto shared = std::make_shared>(); + for (int k = 0; k < n_climbers; ++k) + climbers[k]->shared_incumbent = shared; } #if MIP_INSTANTIATE_FLOAT diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 0ebe059d1d..d81674c991 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -88,6 +88,40 @@ struct host_contiguous_set_t { constexpr double fj_obj_mult_min = 0.25; constexpr double fj_obj_mult_max = 4.0; +// Best feasible assignment found by any lane of one portfolio. A lane publishes its own +// improvements and adopts a better one when it perturbs, so a lane that has stalled resumes from +// the portfolio's progress instead of its own. Lanes run concurrently, so which lane observes +// which incumbent depends on scheduling: a portfolio that shares is not run-to-run reproducible. +template +struct fj_cpu_shared_incumbent_t { + // True when the candidate beat the shared best, in which case it was stored. + bool publish(f_t candidate_objective, const std::vector& candidate) + { + // Unlocked reject first: the publish sites are hot on instances that improve in tiny steps. + if (!(candidate_objective < objective.load(std::memory_order_relaxed))) return false; + std::lock_guard lock(guard); + if (!(candidate_objective < objective.load(std::memory_order_relaxed))) return false; + assignment = candidate; + objective.store(candidate_objective, std::memory_order_relaxed); + return true; + } + + // True when the shared best beat local_objective, in which case it was copied into destination. + bool adopt(f_t local_objective, std::vector& destination) + { + if (!(objective.load(std::memory_order_relaxed) < local_objective)) return false; + std::lock_guard lock(guard); + if (!(objective.load(std::memory_order_relaxed) < local_objective)) return false; + cuopt_assert(assignment.size() == destination.size(), "shared incumbent size mismatch"); + destination = assignment; + return true; + } + + std::mutex guard; + std::vector assignment; + std::atomic objective{std::numeric_limits::infinity()}; +}; + // NOTE: this seems an easy pick for reflection/xmacros once this is available (C++26?) // Maintaining a single source of truth for all members would be nice template @@ -265,6 +299,10 @@ struct fj_cpu_climber_t { std::function&)> diversity_callback{nullptr}; std::string log_prefix{""}; + // Held with the other lanes of the same portfolio. Null when the climber runs alone, which is + // what keeps a solo climber reproducible. + std::shared_ptr> shared_incumbent; + // Work unit tracking for deterministic synchronization std::atomic work_units_elapsed{0.0}; double work_unit_bias{1.5}; // Bias factor to keep CPUFJ ahead of B&B diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index b456559192..2bb49e9cc3 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -573,6 +573,9 @@ struct fj_bin_engine_t { std::vector assign; std::vector best_assign; + std::shared_ptr> shared_incumbent; + // Staging for an adopted assignment, which arrives as f_t. Sized only when sharing is on. + std::vector adopt_buffer; std::vector seed_assign; // restart target std::vector assign_i32; // gather mirror for the SIMD patch (Batch B) @@ -970,6 +973,7 @@ struct fj_bin_engine_t { climber.h_incumbent_objective = (f_t)incumbent_objective; climber.h_best_objective = (f_t)best_objective; climber.feasible_found = true; + if (shared_incumbent) { shared_incumbent->publish((f_t)best_objective, h_best); } // Emitted once per improvement so the benchmark harness can reconstruct the // incumbent trajectory exactly, rather than sampling it at log_interval. CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] new incumbent: objective %.17g", @@ -1395,6 +1399,10 @@ struct fj_bin_engine_t { if (feasible_found) { cuopt_assert((int32_t)best_assign.size() == pb.n_variables, "incumbent size mismatch"); assign = best_assign; + if (shared_incumbent && shared_incumbent->adopt((f_t)best_objective, adopt_buffer)) { + for (int32_t v = 0; v < pb.n_variables; ++v) + assign[v] = (int8_t)(adopt_buffer[v] >= 0.5 ? 1 : 0); + } for (int32_t v = 0; v < pb.n_variables; ++v) assign_i32[v] = assign[v]; } const uint32_t n = (uint32_t)pb.objective_vars.size(); @@ -1462,8 +1470,10 @@ struct fj_bin_engine_t { const double val = (double)h_assign[v]; assign[v] = (int8_t)(val >= 0.5 ? 1 : 0); } - seed_assign = assign; - best_assign = assign; + seed_assign = assign; + best_assign = assign; + shared_incumbent = climber.shared_incumbent; + if (shared_incumbent) adopt_buffer.assign(n, 0); reset_infeasible_checkpoint(); assign_i32.assign(n, 0); for (int32_t v = 0; v < n; ++v) assign_i32[v] = assign[v]; From 428575adad397105fcb8848a4ffd24f0ddc112da Mon Sep 17 00:00:00 2001 From: yboucher Date: Sat, 22 Aug 2026 13:25:12 -0700 Subject: [PATCH 48/61] perturbation gated on objective stall --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 13 +++----- .../feasibility_jump/fj_cpu.cuh | 1 - .../feasibility_jump/fj_cpu_binary.cu | 30 +++++++++++-------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index ca66be9188..cccdb9f813 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1071,8 +1071,6 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.n_variable_updates_window++; fj_cpu.unique_vars_accessed_window.insert(var_idx); - i_t previous_viol = fj_cpu.violated_constraints.size(); - for (auto i = offset_begin; i < offset_end; i++) { cuopt_assert(i < (i_t)fj_cpu.h_reverse_constraints.size(), ""); auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[i].get(); @@ -1113,10 +1111,6 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.h_cstr_version[cstr_idx]++; } - if (previous_viol > 0 && fj_cpu.violated_constraints.empty()) { - fj_cpu.last_feasible_entrance_iter = fj_cpu.iterations; - } - // update the assignment and objective proper fj_cpu.h_assignment[var_idx] = new_val; @@ -2470,9 +2464,10 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w // perturb bool should_perturb = false; if (fj_cpu->violated_constraints.empty() && - fj_cpu->iterations - fj_cpu->last_feasible_entrance_iter > fj_cpu->perturb_interval) { - should_perturb = true; - fj_cpu->last_feasible_entrance_iter = fj_cpu->iterations; + fj_cpu->iterations_since_best > fj_cpu->perturb_interval) { + should_perturb = true; + // Without this the counter stays above the interval and every later iteration perturbs. + fj_cpu->iterations_since_best = 0; } if (score > fj_staged_score_t::zero() && !should_perturb) { diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index d81674c991..054f1b1158 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -226,7 +226,6 @@ struct fj_cpu_climber_t { // objective is re-derived from the assignment. f_t h_objective_sumcomp{0}; f_t h_best_objective; - i_t last_feasible_entrance_iter{0}; i_t iterations; host_contiguous_set_t violated_constraints; host_contiguous_set_t satisfied_constraints; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 2bb49e9cc3..cd8d247153 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -623,7 +623,9 @@ struct fj_bin_engine_t { bool feasible_found{false}; int32_t iters{0}; - int32_t last_feasible_entrance_iter{0}; + // Iterations since best_objective last moved. Counts iterations, unlike + // iterations_at_same_objective, so it is comparable against perturb_interval. + int32_t iters_since_best{0}; int32_t last_restart_iter{0}; int32_t last_kick_iter{0}; int64_t nnz_touched{0}; @@ -854,8 +856,7 @@ struct fj_bin_engine_t { const int8_t new_val = (int8_t)(assign[var] + delta); const int8_t new_flip = (int8_t)(1 - 2 * new_val); const int32_t ob = pb.reverse_offsets[var], oe = pb.reverse_offsets[var + 1]; - const int32_t prev_violated = (int32_t)violated_list.size(); - int64_t own_score = 0; + int64_t own_score = 0; // The tail writes a score delta through int32_t* and calls out to the patch, either of which may // alias a vector's internal pointer as far as the compiler can prove. Without these locals it @@ -939,8 +940,6 @@ struct fj_bin_engine_t { nnz_touched += oe - ob; rows_walked += oe - ob; - if (prev_violated > 0 && violated_list.empty()) last_feasible_entrance_iter = iters; - assign[var] = new_val; assign_i32[var] = new_val; var_score[var] = own_score; @@ -950,9 +949,10 @@ struct fj_bin_engine_t { obj_base_score[var] = flip_objective_base(var); if (violated_list.empty() && incumbent_objective < best_objective) { - best_objective = incumbent_objective; - best_assign = assign; - feasible_found = true; + best_objective = incumbent_objective; + best_assign = assign; + feasible_found = true; + iters_since_best = 0; report_incumbent(climber); } @@ -1426,8 +1426,9 @@ struct fj_bin_engine_t { reset_infeasible_checkpoint(); tabu.clear(iters); recompute_slack(); - last_restart_iter = iters; - last_feasible_entrance_iter = iters; + last_restart_iter = iters; + // The restarted walk gets a full window before the stall gate can perturb it. + iters_since_best = 0; } void init(fj_cpu_climber_t& climber) @@ -1512,6 +1513,7 @@ struct fj_bin_engine_t { iterations_at_same_objective = 0; feasible_found = false; iters = 0; + iters_since_best = 0; last_restart_iter = 0; last_kick_iter = 0; recompute_slack(); @@ -1549,9 +1551,10 @@ struct fj_bin_engine_t { std::tie(move_var, score) = find_move_satisfied(mtm_sat_samples); bool perturb_now = false; - if (violated_list.empty() && iters - last_feasible_entrance_iter > perturb_interval) { - perturb_now = true; - last_feasible_entrance_iter = iters; + if (violated_list.empty() && iters_since_best > perturb_interval) { + perturb_now = true; + // Without this the counter stays above the interval and every later iteration perturbs. + iters_since_best = 0; } if (pair2.first >= 0 && !perturb_now) { @@ -1601,6 +1604,7 @@ struct fj_bin_engine_t { } ++iters; + ++iters_since_best; } compute_saturation(); From 21f2b8972b2de51163e4a4d9f6b69c6bae502e9e Mon Sep 17 00:00:00 2001 From: yboucher Date: Sun, 23 Aug 2026 09:50:18 -0700 Subject: [PATCH 49/61] infeasible pair repair --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 2 + .../feasibility_jump/fj_cpu.cuh | 3 + .../feasibility_jump/fj_cpu_binary.cu | 193 ++++++++++++++++-- 3 files changed, 182 insertions(+), 16 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index cccdb9f813..730f58dee4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -3642,6 +3642,8 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i climber.nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); climber.perturb_interval = std::uniform_int_distribution(50, 500)(rng); + climber.enable_infeasible_repair = (lane % 8 == 1) || (lane % 8 == 5); + climber.h_objective_weight = obj_weight_ladder[lane % 4]; //climber.seed_objective_weight = obj_weight_floor[lane % 4]; } diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 054f1b1158..b450395d72 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -283,6 +283,9 @@ struct fj_cpu_climber_t { i_t mtm_sat_samples{15}; i_t nnz_samples{50000}; i_t perturb_interval{100}; + // Enables the binary engine's infeasible-phase pair repair. Per lane, since the pair scan costs + // iterations that a well-tuned single-flip lane would rather spend elsewhere. + bool enable_infeasible_repair{false}; i_t infeasible_restart_window{300}; i_t infeasible_restart_max_streak{20}; f_t infeasible_restart_degrade_ratio{1.15}; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index cd8d247153..17cb8aaeeb 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -181,6 +181,12 @@ struct fj_bin_problem_t { std::vector objective; std::vector objective_vars; + + // Cardinality census, for the repair-pair gate. A cardinality row is an equality over binaries + // sharing one coefficient, so a variable of degree two across them can only be switched on by + // switching exactly one other off: the exchange a pair can represent. + int32_t n_exchange_vars{0}; + int32_t max_card_degree{0}; }; // Result of the width-independent eligibility scan. @@ -217,6 +223,18 @@ constexpr int32_t fj_bin_kick_restart_guard = 50; constexpr int32_t fj_bin_kick_rows = 3; constexpr int32_t fj_bin_kick_vars_per_row = 2; +// Infeasible-phase pair repair: iterations between attempts, violated rows sampled per attempt, +// and the pool size the O(pool^2) pair scan is capped to. +constexpr int32_t fj_bin_repair_interval = 20; +constexpr int32_t fj_bin_repair_max_rows = 4; +constexpr int32_t fj_bin_repair_max_vars = 12; + +// Structure the pair repair needs before it is worth running: enough variables that are shared by +// exactly two cardinality rows, and no variable shared by so many that closing the exchange takes a +// chain rather than a pair. +constexpr int32_t fj_bin_repair_min_exchange_vars = 64; +constexpr int32_t fj_bin_repair_max_card_degree = 4; + // Candidate draws per 2-opt lift search. constexpr int32_t fj_bin_2opt_candidates = 64; // prefetch distance @@ -553,6 +571,41 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, pb.objective[v] = obj[v]; if (pb.objective[v] != 0.0) pb.objective_vars.push_back(v); } + + // Every variable here is binary, so an equality row whose members share one coefficient reads as + // a cardinality constraint. Counted on the unscaled row: the row scale multiplies bound and + // coefficients alike and leaves the ratio alone. + { + std::vector card_degree(n, 0); + for (int32_t r = 0; r < m; ++r) { + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + if (!std::isfinite(lb) || !std::isfinite(ub) || std::fabs(lb - ub) > tol) continue; + + const int32_t begin = offsets[r]; + const int32_t end = offsets[r + 1]; + if (end - begin < 2) continue; + + const double shared = coeffs[begin]; + if (std::fabs(shared) <= tol) continue; + const double k = lb / shared; + if (k < 1.0 - tol || std::fabs(k - std::round(k)) > tol) continue; + + bool uniform = true; + for (int32_t p = begin; p < end && uniform; ++p) { + const double a = coeffs[p]; + uniform = std::fabs(a - shared) <= tol * std::max(1.0, std::fabs(shared)); + } + if (!uniform) continue; + + for (int32_t p = begin; p < end; ++p) + card_degree[variables[p]]++; + } + for (int32_t v = 0; v < n; ++v) { + if (card_degree[v] == 2) ++pb.n_exchange_vars; + if (card_degree[v] > pb.max_card_degree) pb.max_card_degree = card_degree[v]; + } + } return true; } @@ -649,6 +702,8 @@ struct fj_bin_engine_t { int32_t perturb_interval{100}; int32_t mtm_viol_samples{25}; int32_t mtm_sat_samples{15}; + bool enable_infeasible_repair{false}; + int32_t last_repair_iter{0}; int32_t infeasible_restart_window{300}; int32_t infeasible_restart_max_streak{20}; double infeasible_restart_degrade_ratio{1.15}; @@ -1274,6 +1329,92 @@ struct fj_bin_engine_t { return true; } + // Net change in the violated-row count from flipping both variables. Positive is an improvement. + // Both reverse ranges are row-ascending, so shared rows are counted once with their joint delta. + int32_t paired_flip_violation_delta(int32_t var1, + int8_t delta1, + int32_t var2, + int8_t delta2) const + { + int32_t i = pb.reverse_offsets[var1], ie = pb.reverse_offsets[var1 + 1]; + int32_t j = pb.reverse_offsets[var2], je = pb.reverse_offsets[var2 + 1]; + int32_t net = 0; + + while (i < ie || j < je) { + const int32_t r1 = i < ie ? pb.reverse_constraints[i] : INT32_MAX; + const int32_t r2 = j < je ? pb.reverse_constraints[j] : INT32_MAX; + const int32_t r = r1 < r2 ? r1 : r2; + + int32_t change = 0; + if (r1 == r) change += (int32_t)pb.reverse_coefficients[i++] * delta1; + if (r2 == r) change += (int32_t)pb.reverse_coefficients[j++] * delta2; + + const bool was_violated = row_slack[r] < 0; + const bool now_violated = row_slack[r] - change < 0; + if (was_violated && !now_violated) + ++net; + else if (!was_violated && now_violated) + --net; + } + return net; + } + + // Draws a few violated rows and searches their members for a joint flip that strictly reduces the + // violated-row count. The single-flip path cannot see these: each half may be neutral or worsening + // on its own. Rate-limited by the caller because the pair scan is quadratic in the pool. + std::pair find_infeasible_pair_repair() + { + const std::pair none{-1, -1}; + if (violated_list.empty()) return none; + + sample_buf.clear(); + const int32_t n_viol = (int32_t)violated_list.size(); + const int32_t n_rows = n_viol < fj_bin_repair_max_rows ? n_viol : fj_bin_repair_max_rows; + for (int32_t t = 0; t < n_rows; ++t) + sample_buf.push_back(violated_list[rng.next_u32() % (uint32_t)n_viol]); + + int32_t pool[fj_bin_repair_max_vars]; + int32_t n_pool = 0; + for (int32_t r : sample_buf) { + const int32_t begin = pb.offsets[r]; + const int32_t width = pb.offsets[r + 1] - begin; + if (width == 0) continue; + + // A random cyclic start rather than the CSR prefix. At a repeated local minimum the prefix + // makes the neighbourhood deterministic and leaves the tail of a wide covering row permanently + // invisible, at the same pool size and cost. + const int32_t start = (int32_t)(rng.next_u32() % (uint32_t)width); + for (int32_t q = 0; q < width && n_pool < fj_bin_repair_max_vars; ++q) { + const int32_t v = pb.variables[begin + (start + q) % width]; + bool dup = false; + for (int32_t p = 0; p < n_pool && !dup; ++p) + dup = pool[p] == v; + if (!dup) pool[n_pool++] = v; + } + } + + std::pair best_pair = none; + int32_t best_net = 0; + for (int32_t a = 0; a < n_pool; ++a) { + const int32_t v1 = pool[a]; + if (tabu_blocked(v1, false)) continue; + const int8_t delta1 = (int8_t)(1 - 2 * assign[v1]); + + for (int32_t b = a + 1; b < n_pool; ++b) { + const int32_t v2 = pool[b]; + if (tabu_blocked(v2, false)) continue; + const int8_t delta2 = (int8_t)(1 - 2 * assign[v2]); + const int32_t net = paired_flip_violation_delta(v1, delta1, v2, delta2); + if (net > best_net) { + best_net = net; + best_pair = {v1, v2}; + } + } + } + cuopt_assert(best_pair.first < 0 || best_net > 0, "accepted a repair that gains no row"); + return best_pair; + } + std::pair, int64_t> find_lift_2opt_move() { cuopt_assert(violated_list.empty(), "lift moves require a feasible incumbent"); @@ -1439,9 +1580,13 @@ struct fj_bin_engine_t { tabu_tenure_min = params.tabu_tenure_min; tabu_tenure_max = params.tabu_tenure_max; breakthrough_margin = params.breakthrough_move_epsilon; - perturb_interval = climber.perturb_interval; - mtm_viol_samples = climber.mtm_viol_samples; - mtm_sat_samples = climber.mtm_sat_samples; + perturb_interval = climber.perturb_interval; + mtm_viol_samples = climber.mtm_viol_samples; + mtm_sat_samples = climber.mtm_sat_samples; + enable_infeasible_repair = climber.enable_infeasible_repair && + pb.n_exchange_vars >= fj_bin_repair_min_exchange_vars && + pb.max_card_degree <= fj_bin_repair_max_card_degree; + last_repair_iter = 0; infeasible_restart_window = climber.infeasible_restart_window; infeasible_restart_max_streak = climber.infeasible_restart_max_streak; @@ -1563,20 +1708,36 @@ struct fj_bin_engine_t { } else if (score > 0 && move_var >= 0 && !perturb_now) { apply_move(move_var, (int8_t)(1 - 2 * assign[move_var]), climber); } else { - update_weights(); - const bool kick_ready = !violated_list.empty() && - iters_since_infeasible_improve >= fj_bin_kick_after && - iters - last_kick_iter >= fj_bin_kick_cooldown && - iters - last_restart_iter >= fj_bin_kick_restart_guard; - if (kick_ready) { - infeasible_region_kick(); - last_kick_iter = iters; - } else if (perturb_now) { - perturb(); + // A pair that reduces the violated count takes precedence over reweighting: the weights + // exist to escape a minimum no move can improve, and this found one that can. + bool repaired = false; + if (enable_infeasible_repair && !violated_list.empty() && + iters - last_repair_iter >= fj_bin_repair_interval) { + last_repair_iter = iters; + const auto repair_pair = find_infeasible_pair_repair(); + if (repair_pair.first >= 0) { + apply_move(repair_pair.first, (int8_t)(1 - 2 * assign[repair_pair.first]), climber); + apply_move(repair_pair.second, (int8_t)(1 - 2 * assign[repair_pair.second]), climber); + repaired = true; + } + } + + if (!repaired) { + update_weights(); + const bool kick_ready = !violated_list.empty() && + iters_since_infeasible_improve >= fj_bin_kick_after && + iters - last_kick_iter >= fj_bin_kick_cooldown && + iters - last_restart_iter >= fj_bin_kick_restart_guard; + if (kick_ready) { + infeasible_region_kick(); + last_kick_iter = iters; + } else if (perturb_now) { + perturb(); + } + std::tie(move_var, score) = find_move_violated(1, true); + const int32_t v = move_var >= 0 ? move_var : 0; + apply_move(v, (int8_t)(1 - 2 * assign[v]), climber); } - std::tie(move_var, score) = find_move_violated(1, true); - const int32_t v = move_var >= 0 ? move_var : 0; - apply_move(v, (int8_t)(1 - 2 * assign[v]), climber); } if (iters % climber.log_interval == 0) { From dd70c3258a412109c2f4d5364b2becaf2c3d6575 Mon Sep 17 00:00:00 2001 From: yboucher Date: Sun, 23 Aug 2026 11:35:38 -0700 Subject: [PATCH 50/61] dualsimplex LP seed, 2s --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 125 +++++++++++++++++- .../feasibility_jump/fj_cpu.cuh | 3 + 2 files changed, 123 insertions(+), 5 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 730f58dee4..bc604ad902 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -9,6 +9,9 @@ #include #include +#include +#include +#include #include "feasibility_jump.cuh" #include "feasibility_jump_impl_common.cuh" @@ -36,6 +39,7 @@ #include #include #include +#include #include #include @@ -2368,16 +2372,126 @@ std::unique_ptr> fj_t::create_cpu_climber( constexpr int32_t fj_nnz_per_refresh_stretch = 100000; constexpr int32_t fj_max_refresh_stretch = 8; +// Above this a short LP spends more time moving the matrix than it can pay back as a seed, and the +// wall budget the LP is allowed out of the lane's own. +constexpr int64_t fj_lp_seed_nnz_limit = 8'000'000; +constexpr double fj_lp_seed_budget_s = 2; + +// Rounds the LP relaxation into this lane's start point. Solved by dual simplex on the lane's own +// thread rather than during portfolio construction, so the other seven start searching immediately. +template +static void apply_lp_rounded_seed(fj_cpu_climber_t& fj_cpu) +{ + if (!fj_cpu.use_lp_seed || fj_cpu.pb_ptr == nullptr) return; + if (fj_cpu.view.pb.nnz > fj_lp_seed_nnz_limit) return; + + simplex::user_problem_t relaxation(fj_cpu.pb_ptr->handle_ptr); + fj_cpu.pb_ptr->get_host_user_problem(relaxation); + + simplex_solver_settings_t lp_settings; + lp_settings.relaxation = true; + lp_settings.time_limit = fj_lp_seed_budget_s; + lp_settings.log.log = false; + + std::vector relaxed; + simplex::lp_status_t lp_status = simplex::lp_status_t::UNSET; + double lp_seconds = 0; + + // solve_linear_program_advanced rather than simplex::solve, whose collapsed int return cannot + // separate a limit -- which leaves a usable vertex behind -- from infeasibility, which does not. + // Guarded on f_t because dual simplex is only instantiated for double. + if constexpr (std::is_same_v) { + const f_t lp_start = tic(); + lp_problem_t converted(relaxation.handle_ptr, + relaxation.num_rows, + relaxation.num_cols, + relaxation.A.col_start[relaxation.A.n]); + std::vector new_slacks; + simplex::dualize_info_t dualize_info; + simplex::convert_user_problem(relaxation, lp_settings, converted, new_slacks, dualize_info); + + simplex::lp_solution_t lp_solution(converted.num_rows, converted.num_cols); + std::vector vstatus; + std::vector edge_norms; + lp_status = simplex::solve_linear_program_advanced( + converted, lp_start, lp_settings, lp_solution, vstatus, edge_norms); + relaxed = lp_solution.x; + lp_seconds = toc(lp_start); + } + + // A vertex reached at a limit is dual feasible and still worth rounding. The remaining + // terminations leave nothing to round. + const bool usable = lp_status == simplex::lp_status_t::OPTIMAL || + lp_status == simplex::lp_status_t::TIME_LIMIT || + lp_status == simplex::lp_status_t::ITERATION_LIMIT || + lp_status == simplex::lp_status_t::CONCURRENT_LIMIT || + lp_status == simplex::lp_status_t::WORK_LIMIT; + CUOPT_LOG_DEBUG("%sCPUFJ LP seed: %s after %.3fs of %.3fs%s", + fj_cpu.log_prefix.c_str(), + simplex::lp_status_to_string(lp_status).c_str(), + lp_seconds, + fj_lp_seed_budget_s, + usable ? "" : ", discarded"); + if (!usable) return; + + const i_t n_variables = fj_cpu.view.pb.n_variables; + // convert_user_problem appends slacks, so the model's own variables are the leading columns. + cuopt_assert((i_t)relaxed.size() >= n_variables, "dual simplex returned too few columns"); + + std::vector candidate(n_variables); + for (i_t var = 0; var < n_variables; ++var) { + cuopt_assert(isfinite(relaxed[var]), "dual simplex returned a non-finite value"); + const auto bounds = fj_cpu.h_var_bounds[var].get(); + const f_t lower = get_lower(bounds); + const f_t upper = get_upper(bounds); + f_t value = std::clamp(relaxed[var], lower, upper); + if (is_integer_var(fj_cpu, var)) { + value = round(value); + // Rounding can leave the bounds, and a variable with no integral value inside them cannot be + // seeded at all without breaking the engine's integrality invariant. + if (value < lower || value > upper) return; + } + candidate[var] = value; + } + + fj_cpu.h_assignment = candidate; + fj_cpu.h_best_assignment = candidate; + recompute_lhs(fj_cpu); + + // The rounded point can already be integral-feasible. It never passed through apply_move, so the + // incumbent is recorded here through the same contract that path uses. + if (fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { + fj_cpu.h_best_objective = + fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; + fj_cpu.feasible_found = true; + CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", + fj_cpu.log_prefix.c_str(), + fj_cpu.h_best_objective); + if (fj_cpu.improvement_callback) { + fj_cpu.improvement_callback(fj_cpu.h_incumbent_objective, + fj_cpu.h_assignment, + fj_cpu.work_units_elapsed.load(std::memory_order_acquire)); + } + if (fj_cpu.shared_incumbent) { + fj_cpu.shared_incumbent->publish(fj_cpu.h_incumbent_objective, fj_cpu.h_assignment); + } + } +} + template void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double work_unit_limit) { + const auto solve_start = std::chrono::high_resolution_clock::now(); // problem fits the binary fastpath shape? run it (engine is solve-local) if (try_cpufj_binary_solve(*fj_cpu, in_time_limit, work_unit_limit)) return; - i_t local_mins = 0; - auto loop_start = std::chrono::high_resolution_clock::now(); + apply_lp_rounded_seed(*fj_cpu); + + i_t local_mins = 0; + // The LP comes out of this lane's own budget; every other lane's clock starts where it did. + auto loop_start = fj_cpu->use_lp_seed ? solve_start : std::chrono::high_resolution_clock::now(); auto time_limit = std::chrono::milliseconds(static_cast(std::floor(in_time_limit * 1000.0))); - auto loop_time_start = std::chrono::high_resolution_clock::now(); + auto loop_time_start = loop_start; fj_cpu->rng.seed(fj_cpu->settings.seed); @@ -3623,14 +3737,15 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; const f_t obj_weight_floor[4] = {1, 4, 32, 1}; - // One structural start per lane; lanes 0 and 4 keep the shared anchor. + // One structural start per lane; lanes 0 and 4 keep the shared anchor. Lane 7 also keeps it here, + // because its replacement is an LP solved inside that lane's own task rather than in setup. + climber.use_lp_seed = lane % 8 == 7; switch (lane % 8) { case 1: apply_lock_weighted_seed(climber); break; case 2: apply_aggressive_constraint_seed(climber); break; case 3: apply_greedy_covering_seed(climber); break; case 5: apply_bipartite_matching_seed(climber); break; case 6: apply_objective_corner_seed(climber); break; - case 7: apply_lower_bound_seed(climber); break; default: break; } diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index b450395d72..61d1bfb5e4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -283,6 +283,9 @@ struct fj_cpu_climber_t { i_t mtm_sat_samples{15}; i_t nnz_samples{50000}; i_t perturb_interval{100}; + // One lane replaces its start with a rounded LP relaxation, solved inside that lane's own task so + // portfolio construction does not wait on an LP. + bool use_lp_seed{false}; // Enables the binary engine's infeasible-phase pair repair. Per lane, since the pair scan costs // iterations that a well-tuned single-flip lane would rather spend elsewhere. bool enable_infeasible_repair{false}; From e885c1161e4f60ec9c0c40c592aa86a2f7735839 Mon Sep 17 00:00:00 2001 From: yboucher Date: Sun, 23 Aug 2026 12:41:39 -0700 Subject: [PATCH 51/61] some optimization work --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 120 ++++++++++++------ .../feasibility_jump/fj_cpu.cuh | 3 + .../feasibility_jump/fj_cpu_binary.cu | 7 +- .../feasibility_jump/fj_cpu_binary.cuh | 7 + .../fj_cpu_binary_kernels.cpp | 31 +++++ 5 files changed, 125 insertions(+), 43 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index bc604ad902..3e89277a7f 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -1075,41 +1075,72 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.n_variable_updates_window++; fj_cpu.unique_vars_accessed_window.insert(var_idx); + const size_t nnz_touched = (size_t)(offset_end - offset_begin); + fj_cpu.h_reverse_constraints.byte_loads += nnz_touched * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_touched * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_touched * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_touched * sizeof(f_t); + fj_cpu.h_lhs.byte_stores += nnz_touched * sizeof(f_t); + fj_cpu.h_lhs_sumcomp.byte_loads += nnz_touched * sizeof(f_t); + fj_cpu.h_lhs_sumcomp.byte_stores += nnz_touched * sizeof(f_t); + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + f_t* const row_sumcomp = fj_cpu.view.incumbent_lhs_sumcomp.data(); + for (auto i = offset_begin; i < offset_end; i++) { cuopt_assert(i < (i_t)fj_cpu.h_reverse_constraints.size(), ""); - auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[i].get(); + const auto [c_lb, c_ub] = cstr_bounds[i]; - auto cstr_idx = fj_cpu.h_reverse_constraints[i]; - auto cstr_coeff = fj_cpu.h_reverse_coefficients[i]; + const i_t cstr_idx = rev_cstr[i]; + const f_t cstr_coeff = rev_coeff[i]; - f_t old_lhs = fj_cpu.h_lhs[cstr_idx]; + const f_t old_lhs = row_lhs[cstr_idx]; // Kahan compensated summation - f_t y = cstr_coeff * delta - fj_cpu.h_lhs_sumcomp[cstr_idx]; - f_t t = old_lhs + y; - fj_cpu.h_lhs_sumcomp[cstr_idx] = (t - old_lhs) - y; - fj_cpu.h_lhs[cstr_idx] = t; - f_t new_lhs = fj_cpu.h_lhs[cstr_idx]; - f_t old_cost = fj_cpu.view.excess_score(cstr_idx, old_lhs, c_lb, c_ub); - f_t new_cost = fj_cpu.view.excess_score(cstr_idx, new_lhs, c_lb, c_ub); - f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + const f_t y = cstr_coeff * delta - row_sumcomp[cstr_idx]; + const f_t t = old_lhs + y; + const f_t new_sumcomp = (t - old_lhs) - y; + row_sumcomp[cstr_idx] = new_sumcomp; + row_lhs[cstr_idx] = t; + + const f_t old_cost = fj_cpu.view.excess_score(cstr_idx, old_lhs, c_lb, c_ub); + const f_t new_cost = fj_cpu.view.excess_score(cstr_idx, t, c_lb, c_ub); + const f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); // trigger early lhs recomputation if the sumcomp term gets too large // to avoid large numerical errors - if (fabs(fj_cpu.h_lhs_sumcomp[cstr_idx]) > BIGVAL_THRESHOLD) - fj_cpu.trigger_early_lhs_recomputation = true; + if (fabs(new_sumcomp) > BIGVAL_THRESHOLD) fj_cpu.trigger_early_lhs_recomputation = true; + + const bool was_violated = fj_cpu.violated_constraints.contains(cstr_idx); + const bool now_violated = new_cost < -cstr_tolerance; - if (new_cost < -cstr_tolerance && !fj_cpu.violated_constraints.contains(cstr_idx)) { + // total_violations sums the excess over the violated set alone, so a row crossing the boundary + // contributes its whole cost rather than a difference. Kahan compensated, as h_lhs is: this is + // now the only place the total is maintained between refreshes. + const f_t viol_delta = + (now_violated ? new_cost : f_t{0}) - (was_violated ? old_cost : f_t{0}); + if (viol_delta != f_t{0}) { + const f_t viol_old = fj_cpu.total_violations; + const f_t viol_y = viol_delta - fj_cpu.total_violations_sumcomp; + const f_t viol_t = viol_old + viol_y; + fj_cpu.total_violations_sumcomp = (viol_t - viol_old) - viol_y; + fj_cpu.total_violations = viol_t; + } + + if (now_violated && !was_violated) { fj_cpu.violated_constraints.insert(cstr_idx); cuopt_assert(fj_cpu.satisfied_constraints.contains(cstr_idx), ""); fj_cpu.satisfied_constraints.remove(cstr_idx); - } else if (!(new_cost < -cstr_tolerance) && fj_cpu.violated_constraints.contains(cstr_idx)) { + } else if (!now_violated && was_violated) { cuopt_assert(!fj_cpu.satisfied_constraints.contains(cstr_idx), ""); fj_cpu.violated_constraints.remove(cstr_idx); fj_cpu.satisfied_constraints.insert(cstr_idx); } cuopt_assert(isfinite(delta), "delta should be finite"); - cuopt_assert(isfinite(fj_cpu.h_lhs[cstr_idx]), "assignment should be finite"); + cuopt_assert(isfinite(t), "assignment should be finite"); // Invalidate related cached move scores fj_cpu.h_cstr_version[cstr_idx]++; @@ -1413,7 +1444,8 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) fj_cpu.violated_constraints.clear(); fj_cpu.satisfied_constraints.clear(); - fj_cpu.total_violations = 0; + fj_cpu.total_violations = 0; + fj_cpu.total_violations_sumcomp = 0; for (i_t cstr_idx = 0; cstr_idx < fj_cpu.view.pb.n_constraints; ++cstr_idx) { auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); auto c_lb = fj_cpu.h_cstr_lb[cstr_idx]; @@ -1596,20 +1628,37 @@ static thrust::tuple find_lift_move( if (delta * obj_coeff >= 0) continue; auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); - bool breaks_a_row = false; + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + const f_t* const row_sumcomp = fj_cpu.view.incumbent_lhs_sumcomp.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + + bool breaks_a_row = false; + i_t scanned = 0; for (i_t j = offset_begin; j < offset_end; ++j) { - auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[j].get(); - const i_t cstr_idx = fj_cpu.h_reverse_constraints[j]; - const f_t cstr_coeff = fj_cpu.h_reverse_coefficients[j]; - const f_t lhs = fj_cpu.h_lhs[cstr_idx]; - const f_t sumcomp = fj_cpu.h_lhs_sumcomp[cstr_idx]; - const f_t new_lhs = lhs + (cstr_coeff * delta - sumcomp); + ++scanned; + const auto [c_lb, c_ub] = cstr_bounds[j]; + const i_t cstr_idx = rev_cstr[j]; + const f_t cstr_coeff = rev_coeff[j]; + const f_t lhs = row_lhs[cstr_idx]; + const f_t sumcomp = row_sumcomp[cstr_idx]; + const f_t new_lhs = lhs + (cstr_coeff * delta - sumcomp); if (fj_cpu.view.excess_score(cstr_idx, new_lhs, c_lb, c_ub) < -fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub)) { breaks_a_row = true; break; } } + + const size_t nnz_scanned = (size_t)scanned; + fj_cpu.h_reverse_constraints.byte_loads += nnz_scanned * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_scanned * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_scanned * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_scanned * sizeof(f_t); + fj_cpu.h_lhs_sumcomp.byte_loads += nnz_scanned * sizeof(f_t); + if (breaks_a_row) continue; } else { f_t lfd_lb = get_lower(fj_cpu.h_var_bounds[var_idx].get()) - val; @@ -2156,13 +2205,14 @@ static void finalize_fj_cpu_host_initialization_from_template( fj_cpu.cached_cstr_bounds = tmpl.cached_cstr_bounds; fj_cpu.obj_magnitude = tmpl.obj_magnitude; - fj_cpu.h_lhs = tmpl.h_lhs; - fj_cpu.h_lhs_sumcomp = tmpl.h_lhs_sumcomp; - fj_cpu.violated_constraints = tmpl.violated_constraints; - fj_cpu.satisfied_constraints = tmpl.satisfied_constraints; - fj_cpu.total_violations = tmpl.total_violations; - fj_cpu.h_incumbent_objective = tmpl.h_incumbent_objective; - fj_cpu.h_objective_sumcomp = tmpl.h_objective_sumcomp; + fj_cpu.h_lhs = tmpl.h_lhs; + fj_cpu.h_lhs_sumcomp = tmpl.h_lhs_sumcomp; + fj_cpu.violated_constraints = tmpl.violated_constraints; + fj_cpu.satisfied_constraints = tmpl.satisfied_constraints; + fj_cpu.total_violations = tmpl.total_violations; + fj_cpu.total_violations_sumcomp = tmpl.total_violations_sumcomp; + fj_cpu.h_incumbent_objective = tmpl.h_incumbent_objective; + fj_cpu.h_objective_sumcomp = tmpl.h_objective_sumcomp; fj_cpu.n_binary_vars = tmpl.n_binary_vars; fj_cpu.n_integer_vars = tmpl.n_integer_vars; @@ -2620,12 +2670,6 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w ++fj_cpu->n_local_minima_window; } - // number of violated constraints is usually small (<100). recomputing from all LHSs is cheap - // and more numerically precise than just adding to the accumulator in apply_move - fj_cpu->total_violations = 0; - for (auto cstr_idx : fj_cpu->violated_constraints) { - fj_cpu->total_violations += fj_cpu->view.excess_score(cstr_idx, fj_cpu->h_lhs[cstr_idx]); - } if (fj_cpu->iterations % fj_cpu->log_interval == 0) { CUOPT_LOG_DEBUG( "%sCPUFJ iteration: %d/%d, local mins: %d, best_objective: %g, viol: %zu, obj weight %g, " diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 61d1bfb5e4..5c65f0e383 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -232,6 +232,9 @@ struct fj_cpu_climber_t { bool feasible_found{false}; bool trigger_early_lhs_recomputation{false}; f_t total_violations{0}; + // Kahan compensation for total_violations, mirroring h_lhs_sumcomp. Reset wherever the total is + // re-derived from the violated set. + f_t total_violations_sumcomp{0}; // Timing data structures std::vector find_lift_move_times; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 17cb8aaeeb..8f7ec35a4e 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -1205,11 +1205,8 @@ struct fj_bin_engine_t { // field where it can only separate variables already tied on the base. The winner's score is // then taken from full_score so the caller sees the true value. ensure_objective_base(); - const int64_t* const obj_p = obj_base_score.data(); - const int64_t* const var_p = var_score.data(); - int64_t* const comb_p = combined_score.data(); - for (int32_t v = 0; v < pb.n_variables; ++v) - comb_p[v] = var_p[v] + obj_p[v]; + int64_t* const comb_p = combined_score.data(); + fj_bin_add_scores(var_score.data(), obj_base_score.data(), pb.n_variables, comb_p); int32_t saved_var[fj_bin_tabu_t::ring_size]; int64_t saved_score[fj_bin_tabu_t::ring_size]; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh index 524ead8780..08005817aa 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -121,4 +121,11 @@ void fj_bin_argmax(const int64_t* var_score, int32_t& best_var, int64_t& best_score); +// combined[v] = var_score[v] + obj_score[v] over n variables, which is the full score once the +// objective weight is nonzero. The three arrays must not overlap. +void fj_bin_add_scores(const int64_t* var_score, + const int64_t* obj_score, + int32_t n, + int64_t* combined); + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp index f140041ac6..fd8a93ad73 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -456,6 +456,26 @@ void ArgmaxImpl(const int64_t* HWY_RESTRICT var_score, *best_score = bs; } +// combined[v] = var_score[v] + obj_score[v] over all n variables. Materialized rather than fused +// into the argmax because block_tabu writes sentinels into the result and restores them afterwards, +// so the array has to outlive the scan. None of the three has SIMD padding, hence the scalar tail. +void AddScoresImpl(const int64_t* HWY_RESTRICT var_score, + const int64_t* HWY_RESTRICT obj_score, + int32_t n, + int64_t* HWY_RESTRICT combined) +{ + const hn::ScalableTag d; + const int32_t step = (int32_t)hn::Lanes(d); + const int32_t nblk = n - (n % step); + + for (int32_t v = 0; v < nblk; v += step) { + hn::StoreU(hn::Add(hn::LoadU(d, var_score + v), hn::LoadU(d, obj_score + v)), d, combined + v); + } + for (int32_t v = nblk; v < n; ++v) { + combined[v] = var_score[v] + obj_score[v]; + } +} + } // namespace HWY_NAMESPACE } // namespace cuopt::mathematical_optimization::mip HWY_AFTER_NAMESPACE(); @@ -472,6 +492,7 @@ HWY_EXPORT_T(PatchRowI16, PatchRowDispatchImpl); HWY_EXPORT_T(WalkRowsI8, WalkRowsImpl); HWY_EXPORT_T(WalkRowsI16, WalkRowsImpl); HWY_EXPORT(ArgmaxImpl); +HWY_EXPORT(AddScoresImpl); // HWY_DYNAMIC_DISPATCH resolves the target on every call, and the hwy::GetChosenTarget() call it // expands to is a real out-of-line call: it clobbers the argument registers, so the compiler spills @@ -527,6 +548,8 @@ static fj_bin_walk_fn_t fj_bin_walk_fn(int8_t) { return fj_bin_walk_i8; static fj_bin_walk_fn_t fj_bin_walk_fn(int16_t) { return fj_bin_walk_i16; } static const auto fj_bin_argmax_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(ArgmaxImpl)); +static const auto fj_bin_add_scores_fn = + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(AddScoresImpl)); template int32_t fj_bin_walk_rows(int32_t* row_slack, @@ -600,5 +623,13 @@ void fj_bin_argmax(const int64_t* var_score, fj_bin_argmax_fn(var_score, n, tile, &best_var, &best_score); } +void fj_bin_add_scores(const int64_t* var_score, + const int64_t* obj_score, + int32_t n, + int64_t* combined) +{ + fj_bin_add_scores_fn(var_score, obj_score, n, combined); +} + } // namespace cuopt::mathematical_optimization::mip #endif // HWY_ONCE From 0795c61f2aa8d80d28e8c3eaa26f594749b60e0e Mon Sep 17 00:00:00 2001 From: yboucher Date: Mon, 24 Aug 2026 07:06:24 -0700 Subject: [PATCH 52/61] some optimization efforts --- .../linear_programming/cuopt/run_cpufj.cu | 124 ++++++++ .../mip_heuristics/feasibility_jump/fj_cpu.cu | 293 +++++++++++++----- .../feasibility_jump/fj_cpu.cuh | 50 ++- 3 files changed, 380 insertions(+), 87 deletions(-) diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu index 1ab13ca28b..67dafa847b 100644 --- a/benchmarks/linear_programming/cuopt/run_cpufj.cu +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -147,6 +147,68 @@ int main(int argc, char** argv) problem.n_constraints, problem.nnz); + // Taken from the host-side parse, so it is independent of everything under target_code. + { + const auto& col_indices = mps_data_model.get_constraint_matrix_indices(); + const auto& row_lb = mps_data_model.get_constraint_lower_bounds(); + const auto& row_ub = mps_data_model.get_constraint_upper_bounds(); + const int64_t nnz = (int64_t)col_indices.size(); + + const i_t n_cols = mps_data_model.get_n_variables(); + std::vector degree(n_cols, 0); + for (i_t index : col_indices) { + if (index >= 0 && index < n_cols) ++degree[index]; + } + std::sort(degree.begin(), degree.end()); + + const i_t max_degree = degree.empty() ? 0 : degree.back(); + auto quantile = [&](double q) { + return degree.empty() + ? 0 + : degree[std::min(degree.size() - 1, (size_t)(q * degree.size()))]; + }; + int64_t top10 = 0; + for (size_t k = 0; k < 10 && k < degree.size(); ++k) + top10 += degree[degree.size() - 1 - k]; + const double mean_degree = n_cols > 0 ? (double)nnz / n_cols : 0.0; + std::printf("census cols: n=%d degree max=%d p99=%d p90=%d median=%d mean=%.1f" + " widest=%.1f%% top10=%.1f%% of nnz hub=%.0fx mean\n", + n_cols, + max_degree, + quantile(0.99), + quantile(0.90), + quantile(0.50), + mean_degree, + nnz > 0 ? 100.0 * max_degree / nnz : 0.0, + nnz > 0 ? 100.0 * top10 / nnz : 0.0, + mean_degree > 0 ? max_degree / mean_degree : 0.0); + + const i_t n_rows = (i_t)std::min(row_lb.size(), row_ub.size()); + i_t lb_only = 0, ub_only = 0, equality = 0, ranged = 0, free_rows = 0; + for (i_t r = 0; r < n_rows; ++r) { + const bool has_lb = std::isfinite((double)row_lb[r]); + const bool has_ub = std::isfinite((double)row_ub[r]); + if (has_lb && has_ub) { + ++(row_lb[r] == row_ub[r] ? equality : ranged); + } else if (has_lb) { + ++lb_only; + } else if (has_ub) { + ++ub_only; + } else { + ++free_rows; + } + } + std::printf("census rows: n=%d lb_only=%d ub_only=%d equality=%d ranged=%d free=%d" + " one_sided=%.1f%%\n", + n_rows, + lb_only, + ub_only, + equality, + ranged, + free_rows, + n_rows > 0 ? 100.0 * (lb_only + ub_only) / n_rows : 0.0); + } + // FROZEN -- defines t=0 for the benchmark. Everything above it (the MPS parse, // problem construction under problem/, and the name anonymisation) is outside // target_code; everything below it is editable. A marker any later would leave @@ -308,6 +370,68 @@ int main(int argc, char** argv) bks_user ? std::to_string(*bks_user).c_str() : (cuopt_bench::is_known_infeasible(path) ? "known infeasible" : "unknown")); + std::printf("\n climber | moves | apply nnz | nnz/move | bitmap elems | ratio |" + " bump/apply | bump/weight | mtm inval | cache hit%%\n"); + std::printf("---------+-----------+------------+----------+--------------+-------+" + "------------+-------------+-----------+-----------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& c = *climbers[k]; + const int64_t bitmap = 2 * c.n_moves_applied * (int64_t)c.view.pb.n_variables; + const int64_t probes = c.hit_count + c.miss_count; + std::printf(" %7d | %9lld | %10lld | %8.1f | %12lld | %5.0f | %10lld | %11lld | %9lld |" + " %9.2f\n", + k, + (long long)c.n_moves_applied, + (long long)c.apply_move_nnz, + c.n_moves_applied > 0 ? (double)c.apply_move_nnz / c.n_moves_applied : 0.0, + (long long)bitmap, + c.apply_move_nnz > 0 ? (double)bitmap / c.apply_move_nnz : 0.0, + (long long)c.n_version_bumps_apply, + (long long)c.n_version_bumps_weights, + (long long)c.n_mtm_cache_invalidations, + probes > 0 ? 100.0 * c.hit_count / probes : 0.0); + } + + std::printf("\n climber | mtm calls | row entries | ent/call | capped ent | capped/call |" + " score calls | score nnz | nnz/score | nnz budget\n"); + std::printf("---------+-----------+-------------+----------+-------------+-------------+" + "-------------+-----------+-----------+-----------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& c = *climbers[k]; + std::printf(" %7d | %9lld | %11lld | %8.0f | %11lld | %11.0f | %11lld | %9lld | %9.1f |" + " %10d\n", + k, + (long long)c.n_mtm_calls, + (long long)c.mtm_row_entries, + c.n_mtm_calls > 0 ? (double)c.mtm_row_entries / c.n_mtm_calls : 0.0, + (long long)c.mtm_entries_capped, + c.n_mtm_calls > 0 ? (double)c.mtm_entries_capped / c.n_mtm_calls : 0.0, + (long long)c.n_compute_score_calls, + (long long)c.compute_score_nnz, + c.n_compute_score_calls > 0 + ? (double)c.compute_score_nnz / c.n_compute_score_calls + : 0.0, + c.nnz_samples); + } + + std::printf("\n climber | refresh period | lhs total | periodic | bigval | perturb | restart |" + " epi vars | epi projections\n"); + std::printf("---------+----------------+-----------+----------+--------+---------+---------+" + "----------+----------------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& c = *climbers[k]; + std::printf(" %7d | %14d | %9lld | %8lld | %6lld | %7lld | %7lld | %8zu | %15lld\n", + k, + c.lhs_refresh_period_used, + (long long)c.n_lhs_recompute_total, + (long long)c.n_lhs_recompute_periodic, + (long long)c.n_lhs_recompute_bigval, + (long long)c.n_lhs_recompute_perturb, + (long long)c.n_lhs_recompute_restart, + c.epigraph_vars.size(), + (long long)c.n_epigraph_projections); + } + std::printf("\nSUMMARY: %d/%d crossed (%.0f%%) wall=%.1fs total_iters=%.0f agg_iters/s=%.0f\n", crossed, n_climbers, diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 3e89277a7f..2ee9378a9f 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -104,22 +104,19 @@ thrust::tuple get_mtm_for_bound(const typename fj_t::climber } template -thrust::tuple get_mtm_for_constraint( - const typename fj_t::climber_data_t::view_t& fj, - i_t var_idx, - i_t cstr_idx, - f_t cstr_coeff, - f_t c_lb, - f_t c_ub, - const ArrayType& assignment, - const ArrayType& lhs_vector) +thrust::tuple get_mtm_for_constraint(i_t var_idx, + i_t cstr_idx, + f_t cstr_coeff, + f_t c_lb, + f_t c_ub, + const ArrayType& assignment, + const ArrayType& lhs_vector, + f_t cstr_tolerance) { f_t sign = -1; f_t delta_ij = 0; f_t slack = 0; - f_t cstr_tolerance = fj.get_corrected_tolerance(cstr_idx, c_lb, c_ub); - f_t old_val = assignment[var_idx]; // process each bound as two separate constraints @@ -150,7 +147,7 @@ thrust::tuple get_mtm_for_constraint( } template -std::pair feas_score_constraint(const typename fj_t::climber_data_t::view_t& fj, +std::pair feas_score_constraint(fj_cpu_climber_t& fj_cpu, f_t delta, i_t cstr_idx, f_t cstr_coeff, @@ -158,8 +155,10 @@ std::pair feas_score_constraint(const typename fj_t::climber f_t c_ub, f_t current_lhs, f_t left_weight, - f_t right_weight) + f_t right_weight, + f_t cstr_tolerance) { + const auto& fj = fj_cpu.view; cuopt_assert(isfinite(delta), "invalid delta"); cuopt_assert(cstr_coeff != 0 && isfinite(cstr_coeff), "invalid coefficient"); @@ -170,21 +169,15 @@ std::pair feas_score_constraint(const typename fj_t::climber cuopt_assert(isfinite(c_lb) || isfinite(c_ub), "no range"); // Independent of bound_idx. - const f_t moved_lhs = current_lhs + cstr_coeff * delta; - const f_t cstr_tolerance = fj.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + const f_t moved_lhs = current_lhs + cstr_coeff * delta; const bool old_viol = fj.excess_score(cstr_idx, current_lhs, c_lb, c_ub) < -cstr_tolerance; const bool new_viol = fj.excess_score(cstr_idx, moved_lhs, c_lb, c_ub) < -cstr_tolerance; for (i_t bound_idx = 0; bound_idx < 2; ++bound_idx) { if (!isfinite(bounds[bound_idx])) continue; - // factor to correct the lhs/rhs to turn a lb <= lhs <= ub constraint into - // two virtual leq constraints "lhs <= ub" and "-lhs <= -lb" in order to match - // the convention of the paper - - // TODO: broadcast left/right weights to a csr_offset-indexed table? local minimums - // usually occur on a rarer basis (around 50 iteratiosn to 1 local minimum) - // likely unreasonable and overkill however + // factor to correct the lhs/rhs to turn a lb <= lhs <= ub constraint into two virtual leq + // constraints "lhs <= ub" and "-lhs <= -lb", to match the convention of the paper f_t cstr_weight = bound_idx == 0 ? left_weight : right_weight; f_t sign = bound_idx == 0 ? -1 : 1; f_t rhs = bounds[bound_idx] * sign; @@ -221,12 +214,12 @@ std::pair feas_score_constraint(const typename fj_t::climber // simple improvement else if (!old_sat && !new_sat && old_lhs > new_lhs) { cuopt_assert(old_viol && new_viol, ""); - base_feas += (i_t)(cstr_weight * fj.settings->parameters.excess_improvement_weight); + base_feas += (i_t)(cstr_weight * fj_cpu.settings.parameters.excess_improvement_weight); } // simple worsening else if (!old_sat && !new_sat && old_lhs < new_lhs) { cuopt_assert(old_viol && new_viol, ""); - base_feas -= (i_t)(cstr_weight * fj.settings->parameters.excess_improvement_weight); + base_feas -= (i_t)(cstr_weight * fj_cpu.settings.parameters.excess_improvement_weight); } // robustness score bonus if this would leave some strick slack @@ -415,9 +408,9 @@ static void log_regression_features(fj_cpu_climber_t& fj_cpu, double eval_intensity = (double)fj_cpu.nnz_processed_window / 1000.0; // Cache and locality metrics - i_t cache_hits_window = fj_cpu.hit_count - fj_cpu.hit_count_window_start; - i_t cache_misses_window = fj_cpu.miss_count - fj_cpu.miss_count_window_start; - i_t total_cache_accesses = cache_hits_window + cache_misses_window; + int64_t cache_hits_window = fj_cpu.hit_count - fj_cpu.hit_count_window_start; + int64_t cache_misses_window = fj_cpu.miss_count - fj_cpu.miss_count_window_start; + int64_t total_cache_accesses = cache_hits_window + cache_misses_window; double cache_hit_rate = total_cache_accesses > 0 ? (double)cache_hits_window / total_cache_accesses : 0.0; @@ -623,18 +616,22 @@ static inline std::pair compute_score(fj_cpu_climber_t); fj_cpu.h_lhs.byte_loads += nnz_read * sizeof(f_t); fj_cpu.h_cstr_left_weights.byte_loads += nnz_read * sizeof(f_t); fj_cpu.h_cstr_right_weights.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.h_cstr_tolerance.byte_loads += nnz_read * sizeof(f_t); const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); const f_t* const weight_l = fj_cpu.view.cstr_left_weights.data(); const f_t* const weight_r = fj_cpu.view.cstr_right_weights.data(); + const f_t* const row_tol = fj_cpu.h_cstr_tolerance.data(); const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); for (i_t i = offset_begin; i < offset_end; i++) { @@ -644,7 +641,7 @@ static inline std::pair compute_score(fj_cpu_climber_t(fj_cpu.view, + auto [cstr_base_feas, cstr_bonus_robust] = feas_score_constraint(fj_cpu, delta, cstr_idx, cstr_coeff, @@ -652,7 +649,8 @@ static inline std::pair compute_score(fj_cpu_climber_t(fj_cpu.view, + feas_score_constraint(fj_cpu, lhs_delta, cstr_idx, 1, @@ -740,7 +738,8 @@ static fj_staged_score_t two_opt_compute_pair_score( fj_cpu.h_cstr_ub[cstr_idx], fj_cpu.h_lhs[cstr_idx], fj_cpu.h_cstr_left_weights[cstr_idx], - fj_cpu.h_cstr_right_weights[cstr_idx]); + fj_cpu.h_cstr_right_weights[cstr_idx], + fj_cpu.h_cstr_tolerance[cstr_idx]); base_feas_sum += cstr_base_feas; bonus_robust_sum += cstr_bonus_robust; } @@ -1031,6 +1030,7 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) } // Invalidate related cached move scores + ++fj_cpu.n_version_bumps_weights; fj_cpu.h_cstr_version[cstr_idx]++; } @@ -1076,6 +1076,9 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.unique_vars_accessed_window.insert(var_idx); const size_t nnz_touched = (size_t)(offset_end - offset_begin); + ++fj_cpu.n_moves_applied; + fj_cpu.apply_move_nnz += (int64_t)nnz_touched; + fj_cpu.n_version_bumps_apply += (int64_t)nnz_touched; fj_cpu.h_reverse_constraints.byte_loads += nnz_touched * sizeof(i_t); fj_cpu.h_reverse_coefficients.byte_loads += nnz_touched * sizeof(f_t); fj_cpu.cached_cstr_bounds.byte_loads += nnz_touched * sizeof(std::pair); @@ -1083,10 +1086,12 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.h_lhs.byte_stores += nnz_touched * sizeof(f_t); fj_cpu.h_lhs_sumcomp.byte_loads += nnz_touched * sizeof(f_t); fj_cpu.h_lhs_sumcomp.byte_stores += nnz_touched * sizeof(f_t); + fj_cpu.h_cstr_tolerance.byte_loads += nnz_touched * sizeof(f_t); const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + const f_t* const row_tol = fj_cpu.h_cstr_tolerance.data(); f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); f_t* const row_sumcomp = fj_cpu.view.incumbent_lhs_sumcomp.data(); @@ -1107,7 +1112,7 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, const f_t old_cost = fj_cpu.view.excess_score(cstr_idx, old_lhs, c_lb, c_ub); const f_t new_cost = fj_cpu.view.excess_score(cstr_idx, t, c_lb, c_ub); - const f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + const f_t cstr_tolerance = row_tol[cstr_idx]; // trigger early lhs recomputation if the sumcomp term gets too large // to avoid large numerical errors @@ -1204,9 +1209,46 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, // CUOPT_LOG_TRACE("CPU: tabu noinc_until: %d\n", fj_cpu.h_tabu_noinc_until[var_idx]); } - std::fill(fj_cpu.flip_move_computed.begin(), fj_cpu.flip_move_computed.end(), false); - std::fill(fj_cpu.var_bitmap.begin(), fj_cpu.var_bitmap.end(), false); - fj_cpu.iter_mtm_vars.clear(); + ++fj_cpu.flip_move_epoch; +} + +// Tightest value the rows of a certified epigraph variable imply. Satisfies all of them at once and +// leaves the objective as small as they allow, which is why it is sound from an infeasible point. +template +static f_t project_epigraph_variable(fj_cpu_climber_t& fj_cpu, i_t var_idx) +{ + cuopt_assert(fj_cpu.epigraph_push[var_idx] != 0, "variable is not a certified epigraph variable"); + const bool push_up = fj_cpu.epigraph_push[var_idx] > 0; + const f_t current = fj_cpu.h_assignment[var_idx]; + const auto bounds = fj_cpu.h_var_bounds[var_idx].get(); + f_t target = push_up ? get_lower(bounds) : get_upper(bounds); + + auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); + const size_t nnz_read = (size_t)(offset_end - offset_begin); + fj_cpu.h_reverse_constraints.byte_loads += nnz_read * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_read * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_read * sizeof(f_t); + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + + for (i_t p = offset_begin; p < offset_end; ++p) { + const f_t coeff = rev_coeff[p]; + if (coeff == f_t{0}) continue; + const auto [c_lb, c_ub] = cstr_bounds[p]; + const f_t rest = row_lhs[rev_cstr[p]] - coeff * current; + const f_t bound = ((coeff > f_t{0}) == push_up) ? c_lb : c_ub; + const f_t implied = (bound - rest) / coeff; + if (!isfinite(implied)) continue; + target = push_up ? max(target, implied) : min(target, implied); + } + + target = std::min(std::max(target, get_lower(bounds)), get_upper(bounds)); + cuopt_assert(isfinite(target), "epigraph projection is not finite"); + return target; } template @@ -1220,34 +1262,43 @@ static thrust::tuple find_mtm_move( fj_move_t best_move = fj_move_t{-1, 0}; fj_staged_score_t best_score = fj_staged_score_t::invalid(); - // collect all the variables that are involved in the target constraints + ++fj_cpu.n_mtm_calls; + + // Each row contributes at most its share of the sampling budget. The gate below sits inside the + // walk, so an uncapped wide row is walked in full whatever the budget says. + const i_t per_row_cap = + std::max(1, fj_cpu.nnz_samples / std::max(1, (i_t)target_cstrs.size())); + + i_t entries = 0; for (size_t cstr_idx : target_cstrs) { auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); - for (auto i = offset_begin; i < offset_end; i++) { - i_t var_idx = fj_cpu.h_variables[i]; - if (fj_cpu.var_bitmap[var_idx]) continue; - fj_cpu.iter_mtm_vars.push_back(var_idx); - fj_cpu.var_bitmap[var_idx] = true; - } - } - // estimate the amount of nnzs to consider - i_t nnz_sum = 0; - for (auto var_idx : fj_cpu.iter_mtm_vars) { - auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); - nnz_sum += offset_end - offset_begin; + const i_t width = offset_end - offset_begin; + entries += std::min(width, per_row_cap); + fj_cpu.mtm_entries_capped += (int64_t)std::max(0, width - per_row_cap); } + fj_cpu.mtm_row_entries += (int64_t)entries; + + // The exact sum over the candidate variables costs one random offset read each to set a single + // sampling rate. The mean reverse degree estimates it in constant time. + const f_t mean_reverse_degree = + (f_t)fj_cpu.h_coefficients.size() / (f_t)std::max(1, fj_cpu.view.pb.n_variables); + const f_t nnz_sum = (f_t)entries * mean_reverse_degree; f_t nnz_pick_probability = 1; - if (nnz_sum > fj_cpu.nnz_samples) nnz_pick_probability = (f_t)fj_cpu.nnz_samples / nnz_sum; + if (nnz_sum > (f_t)fj_cpu.nnz_samples) nnz_pick_probability = (f_t)fj_cpu.nnz_samples / nnz_sum; for (size_t cstr_idx : target_cstrs) { - auto c_lb = fj_cpu.h_cstr_lb[cstr_idx]; - auto c_ub = fj_cpu.h_cstr_ub[cstr_idx]; - f_t cstr_tol = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + f_t cstr_tol = fj_cpu.h_cstr_tolerance[cstr_idx]; cuopt_assert(cstr_idx < fj_cpu.h_cstr_lb.size(), "cstr_idx is out of bounds"); auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); - for (auto i = offset_begin; i < offset_end; i++) { + const i_t width = offset_end - offset_begin; + const i_t visit = std::min(width, per_row_cap); + const i_t start = visit == width + ? offset_begin + : offset_begin + (i_t)(rng.next_u32() % (uint32_t)width); + for (i_t q = 0, i = start; q < visit; + ++q, i = (i + 1 == offset_end ? offset_begin : i + 1)) { // early cached check cuopt_assert(fj_cpu.cached_mtm_moves_version[i] <= fj_cpu.h_cstr_version[cstr_idx], "cached move newer than its constraint"); @@ -1280,23 +1331,23 @@ static thrust::tuple find_mtm_move( // Special case for binary variables if (fj_cpu.h_is_binary_variable[var_idx]) { - if (fj_cpu.flip_move_computed[var_idx]) continue; - fj_cpu.flip_move_computed[var_idx] = true; - new_val = 1 - val; + if (fj_cpu.flip_move_stamp[var_idx] == fj_cpu.flip_move_epoch) continue; + fj_cpu.flip_move_stamp[var_idx] = fj_cpu.flip_move_epoch; + new_val = 1 - val; } else { auto cstr_coeff = fj_cpu.h_coefficients[i]; f_t c_lb = fj_cpu.h_cstr_lb[cstr_idx]; f_t c_ub = fj_cpu.h_cstr_ub[cstr_idx]; auto [delta, sign, slack, cstr_tolerance] = - get_mtm_for_constraint(fj_cpu.view, - var_idx, + get_mtm_for_constraint(var_idx, cstr_idx, cstr_coeff, c_lb, c_ub, fj_cpu.h_assignment, - fj_cpu.h_lhs); + fj_cpu.h_lhs, + cstr_tol); if (is_integer_var(fj_cpu, var_idx)) { new_val = cstr_coeff * sign > 0 ? floor(val + delta + fj_cpu.view.pb.tolerances.integrality_tolerance) @@ -1434,6 +1485,7 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) { CPUFJ_NVTX_RANGE("CPUFJ::recompute_lhs"); cuopt_assert(fj_cpu.h_lhs.size() == fj_cpu.view.pb.n_constraints, "h_lhs size mismatch"); + ++fj_cpu.n_lhs_recompute_total; // clamp to var bounds - defensive; apply_move should already have clamped appropriately for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) { @@ -1458,7 +1510,7 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) fj_kahan_babushka_neumaier_sum(delta_it + offset_begin, delta_it + offset_end); fj_cpu.h_lhs_sumcomp[cstr_idx] = 0; - f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + f_t cstr_tolerance = fj_cpu.h_cstr_tolerance[cstr_idx]; f_t new_cost = fj_cpu.view.excess_score(cstr_idx, fj_cpu.h_lhs[cstr_idx]); if (new_cost < -cstr_tolerance) { fj_cpu.violated_constraints.insert(cstr_idx); @@ -1513,8 +1565,7 @@ static bool paired_flip_keeps_feasible( } const f_t new_lhs = fj_cpu.h_lhs[r] + (change - fj_cpu.h_lhs_sumcomp[r]); - if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < - -fj_cpu.view.get_corrected_tolerance(r, c_lb, c_ub)) + if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < -(f_t)fj_cpu.h_cstr_tolerance[r]) return false; } return true; @@ -1559,8 +1610,7 @@ static thrust::tuple find_lift_2opt_mov const i_t r = fj_cpu.h_reverse_constraints[k]; const f_t new_lhs = fj_cpu.h_lhs[r] + ((f_t)fj_cpu.h_reverse_coefficients[k] * delta1 - fj_cpu.h_lhs_sumcomp[r]); - if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < - -fj_cpu.view.get_corrected_tolerance(r, c_lb, c_ub)) { + if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < -(f_t)fj_cpu.h_cstr_tolerance[r]) { if (broken >= 0) multiple = true; else @@ -1633,6 +1683,7 @@ static thrust::tuple find_lift_move( const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); const f_t* const row_sumcomp = fj_cpu.view.incumbent_lhs_sumcomp.data(); + const f_t* const row_tol = fj_cpu.h_cstr_tolerance.data(); const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); bool breaks_a_row = false; @@ -1645,8 +1696,7 @@ static thrust::tuple find_lift_move( const f_t lhs = row_lhs[cstr_idx]; const f_t sumcomp = row_sumcomp[cstr_idx]; const f_t new_lhs = lhs + (cstr_coeff * delta - sumcomp); - if (fj_cpu.view.excess_score(cstr_idx, new_lhs, c_lb, c_ub) < - -fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub)) { + if (fj_cpu.view.excess_score(cstr_idx, new_lhs, c_lb, c_ub) < -row_tol[cstr_idx]) { breaks_a_row = true; break; } @@ -1669,7 +1719,7 @@ static thrust::tuple find_lift_move( auto cstr_coeff = fj_cpu.h_reverse_coefficients[j]; f_t c_lb = fj_cpu.h_cstr_lb[cstr_idx]; f_t c_ub = fj_cpu.h_cstr_ub[cstr_idx]; - f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + f_t cstr_tolerance = fj_cpu.h_cstr_tolerance[cstr_idx]; cuopt_assert(c_lb <= c_ub, "invalid bounds"); cuopt_assert(fj_cpu.view.cstr_satisfied(cstr_idx, fj_cpu.h_lhs[cstr_idx]), "cstr should be satisfied"); @@ -1781,13 +1831,14 @@ static void perturb(fj_cpu_climber_t& fj_cpu) std::sample(fj_cpu.h_objective_vars.begin(), fj_cpu.h_objective_vars.end(), std::back_inserter(sampled_vars), - 2, + std::max(1, fj_cpu.perturb_vars), fj_cpu.rng); raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); for (auto var_idx : sampled_vars) randomize_variable(fj_cpu, var_idx, rng); + ++fj_cpu.n_lhs_recompute_perturb; recompute_lhs(fj_cpu); } @@ -1803,6 +1854,7 @@ static void reset_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) template static void invalidate_mtm_cache(fj_cpu_climber_t& fj_cpu) { + ++fj_cpu.n_mtm_cache_invalidations; for (size_t c = 0; c < fj_cpu.h_cstr_version.size(); ++c) fj_cpu.h_cstr_version[c]++; } @@ -1813,6 +1865,7 @@ static void restart_from_infeasible_checkpoint(fj_cpu_climber_t& fj_cp cuopt_assert(fj_cpu.h_assignment.size() == fj_cpu.h_best_infeasible_assignment.size(), "incumbent_assignment span would be invalidated"); fj_cpu.h_assignment = fj_cpu.h_best_infeasible_assignment; + ++fj_cpu.n_lhs_recompute_restart; recompute_lhs(fj_cpu); invalidate_mtm_cache(fj_cpu); } @@ -1859,6 +1912,7 @@ static void track_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) randomize_variable(fj_cpu, var_idx, rng); + ++fj_cpu.n_lhs_recompute_restart; recompute_lhs(fj_cpu); invalidate_mtm_cache(fj_cpu); reset_infeasible_checkpoint(fj_cpu); @@ -2015,6 +2069,45 @@ static void init_fj_cpu_from_template(fj_cpu_climber_t& fj_cpu, tmpl.view.pb.tolerances); } +// Certifies the epigraph variables: continuous, in the objective, and appearing in every one of +// their rows only on the side the objective pulls away from, with that direction unbounded. +template +static void certify_epigraph_variables(fj_cpu_climber_t& fj_cpu, i_t n_variables) +{ + fj_cpu.epigraph_push.assign(n_variables, 0); + fj_cpu.epigraph_vars.clear(); + + for (i_t var = 0; var < n_variables; ++var) { + if (is_integer_var(fj_cpu, var)) continue; + const f_t obj_coeff = fj_cpu.h_obj_coeffs[var]; + if (obj_coeff == f_t{0}) continue; + + const auto [begin, end] = reverse_range_for_var(fj_cpu, var); + if (begin == end) continue; + + // A positive coefficient is minimised by pushing the variable down, so its rows must be the + // only thing holding it up, and it must be free to rise as far as they demand. + const bool push_up = obj_coeff > f_t{0}; + const auto bounds = fj_cpu.h_var_bounds[var].get(); + if (isfinite(push_up ? get_upper(bounds) : get_lower(bounds))) continue; + + bool certified = true; + for (i_t p = begin; p < end && certified; ++p) { + const i_t row = fj_cpu.h_reverse_constraints[p]; + const f_t coeff = fj_cpu.h_reverse_coefficients[p]; + const bool has_lb = isfinite((f_t)fj_cpu.h_cstr_lb[row]); + const bool has_ub = isfinite((f_t)fj_cpu.h_cstr_ub[row]); + if (coeff == f_t{0}) continue; + certified = push_up ? ((coeff > 0 && has_lb && !has_ub) || (coeff < 0 && has_ub && !has_lb)) + : ((coeff > 0 && has_ub && !has_lb) || (coeff < 0 && has_lb && !has_ub)); + } + if (!certified) continue; + + fj_cpu.epigraph_push[var] = push_up ? 1 : -1; + fj_cpu.epigraph_vars.push_back(var); + } +} + template static void set_host_data_view( fj_cpu_climber_t& fj_cpu, @@ -2107,9 +2200,16 @@ static void wire_fj_cpu_host_views( fj_cpu.cached_mtm_moves_version.assign(fj_cpu.h_coefficients.size(), -1); fj_cpu.h_cstr_version.assign(n_constraints, 0); - fj_cpu.flip_move_computed.resize(n_variables, false); - fj_cpu.var_bitmap.resize(n_variables, false); - fj_cpu.iter_mtm_vars.reserve(n_variables); + fj_cpu.flip_move_stamp.assign(n_variables, 0); + fj_cpu.flip_move_epoch = 1; + + fj_cpu.h_cstr_tolerance.resize(n_constraints); + for (i_t row = 0; row < n_constraints; ++row) { + fj_cpu.h_cstr_tolerance[row] = + fj_cpu.view.get_corrected_tolerance(row, fj_cpu.h_cstr_lb[row], fj_cpu.h_cstr_ub[row]); + } + + certify_epigraph_variables(fj_cpu, n_variables); } template @@ -2167,10 +2267,6 @@ void finalize_fj_cpu_host_initialization( } fj_cpu.h_binrow_offsets[n_constraints] = fj_cpu.h_binrow_vars.size(); - fj_cpu.flip_move_computed.resize(n_variables, false); - fj_cpu.var_bitmap.resize(n_variables, false); - fj_cpu.iter_mtm_vars.reserve(n_variables); - // Must precede recompute_lhs, which is what first populates them. fj_cpu.violated_constraints.resize(n_constraints); fj_cpu.satisfied_constraints.resize(n_constraints); @@ -2201,8 +2297,13 @@ static void finalize_fj_cpu_host_initialization_from_template( cuopt_assert(tmpl.cached_cstr_bounds.size() == fj_cpu.h_reverse_coefficients.size(), "template cached bounds mismatch"); + cuopt_assert(tmpl.h_binrow_offsets.size() == static_cast(n_constraints + 1), + "template binrow offsets mismatch"); + fj_cpu.h_objective_vars = tmpl.h_objective_vars; fj_cpu.cached_cstr_bounds = tmpl.cached_cstr_bounds; + fj_cpu.h_binrow_offsets = tmpl.h_binrow_offsets; + fj_cpu.h_binrow_vars = tmpl.h_binrow_vars; fj_cpu.obj_magnitude = tmpl.obj_magnitude; fj_cpu.h_lhs = tmpl.h_lhs; @@ -2561,7 +2662,18 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w const i_t nnz_stretch = std::min( (i_t)fj_cpu->h_coefficients.size() / fj_nnz_per_refresh_stretch, fj_max_refresh_stretch); const i_t refresh_period = fj_cpu->settings.parameters.lhs_refresh_period * (1 + nnz_stretch); + //const i_t refresh_period = 5000 * (1 + nnz_stretch); cuopt_assert(refresh_period > 0, "refresh period overflowed"); + fj_cpu->lhs_refresh_period_used = refresh_period; + + // Whatever the seed left behind, these rows are satisfiable on their own, so the walk should not + // start with them in the violated set competing for the sampler's attention. + for (i_t var : fj_cpu->epigraph_vars) { + const f_t delta = project_epigraph_variable(*fj_cpu, var) - (f_t)fj_cpu->h_assignment[var]; + if (delta == f_t{0}) continue; + apply_move(*fj_cpu, var, delta, false); + ++fj_cpu->n_epigraph_projections; + } while (!fj_cpu->halted && !fj_cpu->preemption_flag.load()) { // Check if 5 seconds have passed @@ -2584,9 +2696,13 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w // periodically recompute the LHS and violation scores // to correct any accumulated numerical errors - if (fj_cpu->iterations % refresh_period == 0 || fj_cpu->trigger_early_lhs_recomputation) { + if (fj_cpu->trigger_early_lhs_recomputation) { + ++fj_cpu->n_lhs_recompute_bigval; recompute_lhs(*fj_cpu); fj_cpu->trigger_early_lhs_recomputation = false; + } else if (fj_cpu->iterations % refresh_period == 0) { + ++fj_cpu->n_lhs_recompute_periodic; + recompute_lhs(*fj_cpu); } fj_move_t move = fj_move_t{-1, 0}; @@ -2624,6 +2740,17 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w thrust::tie(move, score) = find_mtm_move_sat(*fj_cpu, fj_cpu->mtm_sat_samples); if (score > fj_staged_score_t::zero()) is_mtm_sat = true; } + // The scorers target one row at a time, so on an epigraph variable they climb toward the bound + // its rows already imply. The projection lands there in one move at the same O(degree) cost. + if (move.var_idx >= 0 && fj_cpu->epigraph_push[move.var_idx] != 0) { + const f_t projected = project_epigraph_variable(*fj_cpu, move.var_idx) - + (f_t)fj_cpu->h_assignment[move.var_idx]; + if (projected != f_t{0}) { + move.value = projected; + ++fj_cpu->n_epigraph_projections; + } + } + // if we're in the feasible region but haven't found improvements in the last n iterations, // perturb bool should_perturb = false; @@ -3800,6 +3927,20 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i climber.mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); climber.nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); climber.perturb_interval = std::uniform_int_distribution(50, 500)(rng); + //climber.perturb_vars = std::uniform_int_distribution(2, 8)(rng); + + // The objective weight below is inert until a lane crosses, so without these the whole portfolio + // runs one weight decay, one tabu tenure and one restart policy while it is still infeasible. + // const double smoothing_ladder[8] = {0.0003, 0.0, 0.001, 0.003, 0.0001, 0.0006, 0.002, 0.0003}; + // const int tabu_min_ladder[8] = {3, 1, 5, 3, 2, 6, 4, 3}; + // const int tabu_max_ladder[8] = {13, 7, 21, 13, 10, 25, 17, 13}; + // const i_t restart_window_ladder[8] = {300, 150, 500, 300, 200, 600, 400, 300}; + // const f_t degrade_ratio_ladder[8] = {1.15, 1.05, 1.30, 1.15, 1.08, 1.40, 1.20, 1.15}; + // climber.settings.parameters.weight_smoothing_probability = smoothing_ladder[lane % 8]; + // climber.settings.parameters.tabu_tenure_min = tabu_min_ladder[lane % 8]; + // climber.settings.parameters.tabu_tenure_max = tabu_max_ladder[lane % 8]; + // climber.infeasible_restart_window = restart_window_ladder[lane % 8]; + // climber.infeasible_restart_degrade_ratio = degrade_ratio_ladder[lane % 8]; climber.enable_infeasible_repair = (lane % 8 == 1) || (lane % 8 == 5); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 5c65f0e383..c251388f1f 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -142,6 +142,7 @@ struct fj_cpu_climber_t { ADD_INSTRUMENTED(h_var_bounds), ADD_INSTRUMENTED(h_cstr_lb), ADD_INSTRUMENTED(h_cstr_ub), + ADD_INSTRUMENTED(h_cstr_tolerance), ADD_INSTRUMENTED(h_var_types), ADD_INSTRUMENTED(h_is_binary_variable), ADD_INSTRUMENTED(h_objective_vars), @@ -163,8 +164,7 @@ struct fj_cpu_climber_t { ADD_INSTRUMENTED(h_assignment), ADD_INSTRUMENTED(h_best_assignment), ADD_INSTRUMENTED(h_best_infeasible_assignment), - ADD_INSTRUMENTED(cached_cstr_bounds), - ADD_INSTRUMENTED(iter_mtm_vars)}; + ADD_INSTRUMENTED(cached_cstr_bounds)}; #undef ADD_INSTRUMENTED } @@ -189,6 +189,8 @@ struct fj_cpu_climber_t { ins_vector::type> h_var_bounds; ins_vector h_cstr_lb; ins_vector h_cstr_ub; + // get_corrected_tolerance of each row, held because the bounds it derives from never move. + ins_vector h_cstr_tolerance; ins_vector h_var_types; ins_vector h_is_binary_variable; ins_vector h_objective_vars; @@ -244,14 +246,41 @@ struct fj_cpu_climber_t { std::vector update_weights_times; std::vector compute_score_times; - i_t hit_count{0}; - i_t miss_count{0}; + int64_t hit_count{0}; + int64_t miss_count{0}; i_t candidate_move_hits[3] = {0}; i_t candidate_move_misses[3] = {0}; - // vector is actually likely beneficial here since we're memory bound - std::vector flip_move_computed; + // Hot-loop accounting, reported off the clock by the standalone harness. + int64_t n_moves_applied{0}; + int64_t apply_move_nnz{0}; + int64_t n_mtm_calls{0}; + // Row entries find_mtm_move visits, and the ones the per-row cap kept it from visiting. + int64_t mtm_row_entries{0}; + int64_t mtm_entries_capped{0}; + int64_t n_compute_score_calls{0}; + int64_t compute_score_nnz{0}; + int64_t n_version_bumps_apply{0}; + int64_t n_version_bumps_weights{0}; + int64_t n_mtm_cache_invalidations{0}; + int64_t n_lhs_recompute_total{0}; + int64_t n_lhs_recompute_periodic{0}; + int64_t n_lhs_recompute_bigval{0}; + int64_t n_lhs_recompute_perturb{0}; + int64_t n_lhs_recompute_restart{0}; + i_t lhs_refresh_period_used{0}; + + // A variable's flip move has already been considered when its stamp equals flip_move_epoch, + // which advances once per applied move. An epoch avoids clearing an n_variables bitmap per move. + std::vector flip_move_stamp; + int64_t flip_move_epoch{1}; + + // Continuous objective variables bounded by their rows only opposite the objective's pull, so + // the tightest row bound is their value. epigraph_push is +1 pushing up, -1 pushing down. + std::vector epigraph_push; + std::vector epigraph_vars; + int64_t n_epigraph_projections{0}; // CSR nnz offset -> (delta, score) std::vector> cached_mtm_moves; @@ -264,9 +293,6 @@ struct fj_cpu_climber_t { // std::pair better compile down to 16 bytes!! GCC do your job! ins_vector> cached_cstr_bounds; - std::vector var_bitmap; - ins_vector iter_mtm_vars; - // Scratch reused by the binary 2-opt search, which runs at every local minimum std::vector two_opt_target_cstrs; std::vector two_opt_first_vars; @@ -286,6 +312,8 @@ struct fj_cpu_climber_t { i_t mtm_sat_samples{15}; i_t nnz_samples{50000}; i_t perturb_interval{100}; + // Number of variables randomized by one perturbation. + i_t perturb_vars{2}; // One lane replaces its start with a rounded LP relaxation, solved inside that lane's own task so // portfolio construction does not wait on an LP. bool use_lp_seed{false}; @@ -330,8 +358,8 @@ struct fj_cpu_climber_t { i_t iterations_since_best{0}; // Cache and locality tracking - i_t hit_count_window_start{0}; - i_t miss_count_window_start{0}; + int64_t hit_count_window_start{0}; + int64_t miss_count_window_start{0}; std::unordered_set unique_cstrs_accessed_window; std::unordered_set unique_vars_accessed_window; From d32d935b6b2c19d28f92c566a3422277734479c9 Mon Sep 17 00:00:00 2001 From: yboucher Date: Mon, 24 Aug 2026 08:08:17 -0700 Subject: [PATCH 53/61] bounds prop in CPUFJ --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 190 +++++++++++++++++- .../feasibility_jump/fj_cpu.cuh | 3 + 2 files changed, 191 insertions(+), 2 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 2ee9378a9f..7e38334340 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -2528,6 +2528,178 @@ constexpr int32_t fj_max_refresh_stretch = 8; constexpr int64_t fj_lp_seed_nnz_limit = 8'000'000; constexpr double fj_lp_seed_budget_s = 2; +constexpr int32_t fj_bound_prop_rounds = 10; +// A deduction is committed only when it moves a bound by more than this many absolute tolerances. +constexpr double fj_bound_prop_commit_scale = 1e3; + +// Raises a lower bound to a deduced limit. Returns whether the domain moved. +template +static bool tighten_lower_bound(fj_cpu_climber_t& fj_cpu, + std::vector& lower, + const std::vector& upper, + i_t var, + f_t limit, + f_t commit_threshold) +{ + if (!isfinite(limit)) return false; + if (is_integer_var(fj_cpu, var)) + limit = ceil(limit - fj_cpu.view.pb.tolerances.integrality_tolerance); + if (limit > upper[var]) return false; + if (limit <= lower[var] + commit_threshold) return false; + lower[var] = limit; + return true; +} + +// Lowers an upper bound to a deduced limit. Returns whether the domain moved. +template +static bool tighten_upper_bound(fj_cpu_climber_t& fj_cpu, + const std::vector& lower, + std::vector& upper, + i_t var, + f_t limit, + f_t commit_threshold) +{ + if (!isfinite(limit)) return false; + if (is_integer_var(fj_cpu, var)) + limit = floor(limit + fj_cpu.view.pb.tolerances.integrality_tolerance); + if (limit < lower[var]) return false; + if (limit >= upper[var] - commit_threshold) return false; + upper[var] = limit; + return true; +} + +// Narrows this lane's domains by activity propagation, then reclassifies: an integer squeezed to +// [0,1] becomes eligible for the binary engine. +template +static void apply_bound_propagation(fj_cpu_climber_t& fj_cpu) +{ + if (!fj_cpu.use_bound_prop) return; + + const i_t n_variables = fj_cpu.view.pb.n_variables; + const i_t n_constraints = fj_cpu.view.pb.n_constraints; + const f_t commit = + (f_t)fj_bound_prop_commit_scale * fj_cpu.view.pb.tolerances.absolute_tolerance; + + std::vector lower(n_variables); + std::vector upper(n_variables); + for (i_t var = 0; var < n_variables; ++var) { + auto bounds = fj_cpu.h_var_bounds[var].get(); + lower[var] = get_lower(bounds); + upper[var] = get_upper(bounds); + } + + bool changed = true; + int32_t pass = 0; + for (; changed && pass < fj_bound_prop_rounds; ++pass) { + changed = false; + for (i_t row = 0; row < n_constraints; ++row) { + const f_t row_lb = fj_cpu.h_cstr_lb[row]; + const f_t row_ub = fj_cpu.h_cstr_ub[row]; + const bool has_lb = isfinite(row_lb); + const bool has_ub = isfinite(row_ub); + if (!has_lb && !has_ub) continue; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + + f_t min_activity = 0; + f_t max_activity = 0; + bool finite_min = true; + bool finite_max = true; + for (i_t p = begin; p < end; ++p) { + const f_t coeff = fj_cpu.h_coefficients[p]; + if (coeff == f_t{0}) continue; + const i_t var = fj_cpu.h_variables[p]; + const f_t min_x = coeff > 0 ? lower[var] : upper[var]; + const f_t max_x = coeff > 0 ? upper[var] : lower[var]; + finite_min &= isfinite(min_x); + finite_max &= isfinite(max_x); + if (finite_min) min_activity += coeff * min_x; + if (finite_max) max_activity += coeff * max_x; + } + + const bool from_row_ub = finite_min && has_ub; + const bool from_row_lb = finite_max && has_lb; + if (!from_row_ub && !from_row_lb) continue; + + // The activities are not refreshed as the loop below narrows the row's own variables, and a + // stale bound is the looser one, so a deduction taken against it is the weaker one. + for (i_t p = begin; p < end; ++p) { + const f_t coeff = fj_cpu.h_coefficients[p]; + if (coeff == f_t{0}) continue; + const i_t var = fj_cpu.h_variables[p]; + + if (from_row_ub) { + const f_t rest = min_activity - coeff * (coeff > 0 ? lower[var] : upper[var]); + const f_t limit = (row_ub - rest) / coeff; + changed |= coeff > 0 ? tighten_upper_bound(fj_cpu, lower, upper, var, limit, commit) + : tighten_lower_bound(fj_cpu, lower, upper, var, limit, commit); + } + if (from_row_lb) { + const f_t rest = max_activity - coeff * (coeff > 0 ? upper[var] : lower[var]); + const f_t limit = (row_lb - rest) / coeff; + changed |= coeff > 0 ? tighten_lower_bound(fj_cpu, lower, upper, var, limit, commit) + : tighten_upper_bound(fj_cpu, lower, upper, var, limit, commit); + } + } + } + } + + fj_cpu.h_binary_indices.clear(); + fj_cpu.n_binary_vars = 0; + fj_cpu.n_integer_vars = 0; + i_t tightened = 0; + bool clamped = false; + for (i_t var = 0; var < n_variables; ++var) { + auto bounds = fj_cpu.h_var_bounds[var].get(); + cuopt_assert(!(lower[var] < get_lower(bounds)), "propagation widened a lower bound"); + cuopt_assert(!(upper[var] > get_upper(bounds)), "propagation widened an upper bound"); + cuopt_assert(!(lower[var] > upper[var]), "propagation emptied a domain"); + const bool moved = lower[var] != get_lower(bounds) || upper[var] != get_upper(bounds); + + // Same rule as problem_t::compute_binary_var_table, fixed binaries included: a domain narrowed + // to a point is no longer binary. + const bool integer = is_integer_var(fj_cpu, var); + const bool binary = integer && fj_cpu.view.pb.integer_equal(lower[var], (f_t)0) && + fj_cpu.view.pb.integer_equal(upper[var], (f_t)1); + fj_cpu.h_is_binary_variable[var] = binary; + if (binary) { + fj_cpu.h_binary_indices.push_back(var); + ++fj_cpu.n_binary_vars; + } else if (integer) { + ++fj_cpu.n_integer_vars; + } + if (!moved) continue; + + ++tightened; + fj_cpu.h_var_bounds[var] = typename type_2::type{lower[var], upper[var]}; + + const f_t value = fj_cpu.h_assignment[var]; + const f_t clamped_value = std::clamp(value, lower[var], upper[var]); + if (clamped_value != value) { + cuopt_assert(!integer || fj_cpu.view.pb.is_integer(clamped_value), + "bound clamp broke integrality"); + fj_cpu.h_assignment[var] = clamped_value; + clamped = true; + } + fj_cpu.h_best_assignment[var] = + std::clamp((f_t)fj_cpu.h_best_assignment[var], lower[var], upper[var]); + } + + // h_binary_indices reallocated, so the span over it would otherwise dangle. + fj_cpu.view.pb.binary_indices = + raft::device_span(fj_cpu.h_binary_indices.data(), fj_cpu.h_binary_indices.size()); + + if (clamped) recompute_lhs(fj_cpu); + + CUOPT_LOG_DEBUG("%sCPUFJ bound prop: %d passes, %d domains tightened, %d binary of %d integer", + fj_cpu.log_prefix.c_str(), + pass, + tightened, + fj_cpu.n_binary_vars, + fj_cpu.n_binary_vars + fj_cpu.n_integer_vars); +} + // Rounds the LP relaxation into this lane's start point. Solved by dual simplex on the lane's own // thread rather than during portfolio construction, so the other seven start searching immediately. template @@ -2633,14 +2805,24 @@ template void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double work_unit_limit) { const auto solve_start = std::chrono::high_resolution_clock::now(); + // Precedes the dispatch below because a variable it squeezes to [0,1] can bring the whole model + // into the binary engine's shape. + apply_bound_propagation(*fj_cpu); + const f_t prop_seconds = + fj_cpu->use_bound_prop + ? std::chrono::duration(std::chrono::high_resolution_clock::now() - solve_start).count() + : f_t{0}; + // problem fits the binary fastpath shape? run it (engine is solve-local) - if (try_cpufj_binary_solve(*fj_cpu, in_time_limit, work_unit_limit)) return; + if (try_cpufj_binary_solve(*fj_cpu, in_time_limit - prop_seconds, work_unit_limit)) return; apply_lp_rounded_seed(*fj_cpu); i_t local_mins = 0; // The LP comes out of this lane's own budget; every other lane's clock starts where it did. - auto loop_start = fj_cpu->use_lp_seed ? solve_start : std::chrono::high_resolution_clock::now(); + auto loop_start = (fj_cpu->use_lp_seed || fj_cpu->use_bound_prop) + ? solve_start + : std::chrono::high_resolution_clock::now(); auto time_limit = std::chrono::milliseconds(static_cast(std::floor(in_time_limit * 1000.0))); auto loop_time_start = loop_start; @@ -3911,6 +4093,10 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i // One structural start per lane; lanes 0 and 4 keep the shared anchor. Lane 7 also keeps it here, // because its replacement is an LP solved inside that lane's own task rather than in setup. climber.use_lp_seed = lane % 8 == 7; + + // Half the portfolio searches the propagated model, half the model as parsed. Off the LP lane, so + // one lane does not carry both setup passes. + climber.use_bound_prop = lane % 2 == 0; switch (lane % 8) { case 1: apply_lock_weighted_seed(climber); break; case 2: apply_aggressive_constraint_seed(climber); break; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index c251388f1f..9bd02103b2 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -317,6 +317,9 @@ struct fj_cpu_climber_t { // One lane replaces its start with a rounded LP relaxation, solved inside that lane's own task so // portfolio construction does not wait on an LP. bool use_lp_seed{false}; + // Half the lanes narrow their own domains by activity propagation before searching, so the + // portfolio covers both the propagated and the as-parsed model. + bool use_bound_prop{false}; // Enables the binary engine's infeasible-phase pair repair. Per lane, since the pair scan costs // iterations that a well-tuned single-flip lane would rather spend elsewhere. bool enable_infeasible_repair{false}; From 98130fc17bcb6366f88253b862816e26449cbcab Mon Sep 17 00:00:00 2001 From: yboucher Date: Mon, 24 Aug 2026 08:45:40 -0700 Subject: [PATCH 54/61] ddfw scheme --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 59 +++++++++++++++++++ .../feasibility_jump/fj_cpu.cuh | 2 + 2 files changed, 61 insertions(+) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 7e38334340..e61bc18697 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -973,6 +973,58 @@ static void smooth_weights(fj_cpu_climber_t& fj_cpu) constexpr int32_t fj_weight_escalate_after = 2000; constexpr int32_t fj_weight_escalate_max = 100; +// Satisfied neighbours sampled per violated row for the donation, and the floor a donor keeps. +constexpr int32_t fj_weight_donor_samples = 4; +constexpr double fj_weight_donation_floor = 1.0; + +// DDFW donation: reach through a variable of this violated row to a satisfied neighbour and take +// the bump back off its heavier side, so total weight stays roughly conserved. +template +static void donate_row_weight(fj_cpu_climber_t& fj_cpu, + i_t cstr_idx, + f_t delta, + raft::random::PCGenerator& rng) +{ + const auto [row_begin, row_end] = range_for_constraint(fj_cpu, cstr_idx); + const uint32_t row_width = (uint32_t)(row_end - row_begin); + // What a donor has to carry to still hold the floor once the delta comes off it. + const f_t donor_minimum = (f_t)fj_weight_donation_floor + delta; + i_t donor = -1; + bool donor_left = true; + f_t donor_weight = 0; + + for (i_t sample = 0; row_width > 0 && sample < fj_weight_donor_samples; ++sample) { + const i_t var_idx = fj_cpu.h_variables[row_begin + (i_t)(rng.next_u32() % row_width)]; + const auto [col_begin, col_end] = reverse_range_for_var(fj_cpu, var_idx); + if (col_end <= col_begin) continue; + const i_t candidate = fj_cpu.h_reverse_constraints[ + col_begin + (i_t)(rng.next_u32() % (uint32_t)(col_end - col_begin))]; + if (candidate == cstr_idx || !fj_cpu.satisfied_constraints.contains(candidate)) continue; + + const f_t left = fj_cpu.h_cstr_left_weights[candidate]; + const f_t right = fj_cpu.h_cstr_right_weights[candidate]; + const bool take_left = left >= right; + const f_t weight = take_left ? left : right; + if (weight < donor_minimum) continue; + if (donor >= 0 && weight <= donor_weight) continue; + + donor = candidate; + donor_left = take_left; + donor_weight = weight; + } + if (donor < 0) return; + + const f_t donated = donor_weight - delta; + cuopt_assert(donated >= (f_t)fj_weight_donation_floor, "donation broke the weight floor"); + if (donor_left) { + fj_cpu.h_cstr_left_weights[donor] = donated; + } else { + fj_cpu.h_cstr_right_weights[donor] = donated; + } + ++fj_cpu.n_version_bumps_weights; + fj_cpu.h_cstr_version[donor]++; +} + template static i_t weight_escalation_delta(const fj_cpu_climber_t& fj_cpu) { @@ -1029,6 +1081,11 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) fj_cpu.max_weight = max(fj_cpu.max_weight, new_weight); } + // Only before this lane's first crossing: past that the search oscillates in and out of + // feasibility, and draining satisfied rows costs the objective phase. + if (fj_cpu.use_weight_donation && !fj_cpu.feasible_found) + donate_row_weight(fj_cpu, cstr_idx, delta, rng); + // Invalidate related cached move scores ++fj_cpu.n_version_bumps_weights; fj_cpu.h_cstr_version[cstr_idx]++; @@ -4097,6 +4154,8 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i // Half the portfolio searches the propagated model, half the model as parsed. Off the LP lane, so // one lane does not carry both setup passes. climber.use_bound_prop = lane % 2 == 0; + + climber.use_weight_donation = (lane % 8 == 5) || (lane % 8 == 6); switch (lane % 8) { case 1: apply_lock_weighted_seed(climber); break; case 2: apply_aggressive_constraint_seed(climber); break; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 9bd02103b2..ceddadbbdc 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -320,6 +320,8 @@ struct fj_cpu_climber_t { // Half the lanes narrow their own domains by activity propagation before searching, so the // portfolio covers both the propagated and the as-parsed model. bool use_bound_prop{false}; + // Two lanes move weight from satisfied rows into the violated ones while still infeasible. + bool use_weight_donation{false}; // Enables the binary engine's infeasible-phase pair repair. Per lane, since the pair scan costs // iterations that a well-tuned single-flip lane would rather spend elsewhere. bool enable_infeasible_repair{false}; From e4caa19ece4d207441695580110a8bbe26cd12ef Mon Sep 17 00:00:00 2001 From: yboucher Date: Mon, 24 Aug 2026 09:09:26 -0700 Subject: [PATCH 55/61] encode small domain integers as binary to enable the FJ binary fastpath --- .../feasibility_jump/fj_cpu_binary.cu | 345 +++++++++++++++++- 1 file changed, 334 insertions(+), 11 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index 8f7ec35a4e..b648b8a0b4 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -187,6 +188,15 @@ struct fj_bin_problem_t { // switching exactly one other off: the exchange a pair can represent. int32_t n_exchange_vars{0}; int32_t max_card_degree{0}; + + // Empty unless encoded, when every engine variable is one bit of a bounded general integer and + // original[j] = var_offset[j] + sum of bit_weight[b] * assign[b] over the bits b owned by j. + bool encoded{false}; + int32_t n_original{0}; + std::vector var_offset; + std::vector bit_owner; + std::vector bit_weight; + std::vector orig_objective; }; // Result of the width-independent eligibility scan. @@ -610,6 +620,237 @@ static bool fj_bin_narrow(const fj_cpu_climber_t& c, } +// Bit budget for one general integer's domain. +constexpr int32_t fj_bin_encode_max_bits = 16; +// Cap on the bit-variable count relative to the model's variable count, bounding the SIMD sweep. +constexpr int64_t fj_bin_encode_max_growth = 6; + +// Bits needed to represent the integers 0..W inclusive. +static inline int32_t fj_bin_encode_nbits(int64_t W) +{ + int32_t bits = 0; + while (((int64_t)1 << bits) - 1 < W) ++bits; + return bits; +} + +// Encodes an all-integer model with bounded general integers into bits: x in [L,U] becomes +// x = L + sum_k w_k b_k over weights 1, 2, ..., 2^(nbits-2), R, with R closing the range at W = U-L. +template +static bool fj_bin_encode(const fj_cpu_climber_t& c, + fj_bin_problem_t& pb, + int& coefficient_bits) +{ + const int32_t n = c.view.pb.n_variables; + const int32_t m = c.view.pb.n_constraints; + if (n <= 0 || m <= 0) return false; + + const double tol = c.view.pb.tolerances.integrality_tolerance; + + const auto& var_bounds = c.h_var_bounds; + const auto& var_types = c.h_var_types; + const auto& offsets = c.h_offsets; + const auto& variables = c.h_variables; + const auto& coeffs = c.h_coefficients; + const auto& cstr_lb = c.h_cstr_lb; + const auto& cstr_ub = c.h_cstr_ub; + const auto& left_w = c.h_cstr_left_weights; + const auto& right_w = c.h_cstr_right_weights; + const auto& obj = c.h_obj_coeffs; + + std::vector lower(n); + std::vector upper(n); + std::vector nbits(n); + std::vector bit_start(n); + int64_t total_bits = 0; + for (int32_t v = 0; v < n; ++v) { + if (var_types[v] != var_t::INTEGER) return false; + auto bounds = var_bounds[v]; + const double x = (double)cuopt::get_lower(bounds); + const double y = (double)cuopt::get_upper(bounds); + if (!std::isfinite(x) || !std::isfinite(y) || y < x) return false; + if (!is_integer(x, tol) || !is_integer(y, tol)) return false; + + lower[v] = std::round(x); + upper[v] = std::round(y); + const int64_t W = (int64_t)(upper[v] - lower[v]); + + nbits[v] = fj_bin_encode_nbits(W); + if (nbits[v] > fj_bin_encode_max_bits) return false; + bit_start[v] = (int32_t)total_bits; + total_bits += nbits[v]; + } + if (total_bits <= 0 || total_bits > (int64_t)INT32_MAX / 2) return false; + if (total_bits > fj_bin_encode_max_growth * (int64_t)n) return false; + + const int32_t n_bits = (int32_t)total_bits; + + pb.encoded = true; + pb.n_original = n; + pb.var_offset = lower; + pb.orig_objective.assign(n, 0.0); + pb.bit_owner.assign(n_bits, 0); + pb.bit_weight.assign(n_bits, 0.0); + for (int32_t v = 0; v < n; ++v) { + int64_t covered = 0; + const int64_t W = (int64_t)(upper[v] - lower[v]); + for (int32_t k = 0; k < nbits[v]; ++k) { + const int64_t w = k + 1 < nbits[v] ? (int64_t)1 << k : W - covered; + covered += w; + pb.bit_owner[bit_start[v] + k] = v; + pb.bit_weight[bit_start[v] + k] = (double)w; + } + cuopt_assert(covered == W, "bit weights do not close the domain exactly"); + } + + pb.n_variables = n_bits; + pb.offsets.assign(1, 0); + pb.bound.clear(); + pb.cmax.clear(); + pb.initial_weight.clear(); + pb.variables.clear(); + pb.coefficients.clear(); + + std::vector incoming_weight; + std::vector row_values; + double max_abs_coefficient = 0; + + // One side of one row, as a'b <= bound in bit space with sum(a_j L_j) folded into the bound. + auto emit = [&](int32_t r, double side_bound, long side, double weight) -> bool { + double fixed = 0; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) + fixed += coeffs[k] * lower[variables[k]]; + const double folded_bound = side_bound - fixed; + + row_values.clear(); + bool integral = is_integer(folded_bound, tol); + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + row_values.push_back(coeffs[k]); + if (!is_integer(coeffs[k], tol)) integral = false; + } + row_values.push_back(folded_bound); + + double s = 1.0; + if (!integral) { + s = find_scaling_rational( + row_values, 1.0 / tol, fj_bin_scale_cap, (double)fj_bin_scale_cap, tol); + if (!std::isfinite(s) || s <= 0.0) return false; + } + + coef_t row_cmax = 1; + double row_abs_sum = 0; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + const int32_t v = variables[k]; + const double a = s * coeffs[k]; + if (!is_integer(a, tol)) return false; + const long ai = std::lround(a); + for (int32_t bk = 0; bk < nbits[v]; ++bk) { + const int32_t bit = bit_start[v] + bk; + const long scaled = side * ai * std::lround(pb.bit_weight[bit]); + const long abs_a = std::labs(scaled); + // Bounded by magnitude, so cmax below and the negated side both stay representable. + if (abs_a > (long)std::numeric_limits::max()) return false; + pb.variables.push_back(bit); + pb.coefficients.push_back((coef_t)scaled); + + if (abs_a > (long)row_cmax) row_cmax = (coef_t)abs_a; + row_abs_sum += (double)abs_a; + if ((double)abs_a > max_abs_coefficient) max_abs_coefficient = (double)abs_a; + } + } + if (row_abs_sum > (double)(INT32_MAX / 2)) return false; + + const double scaled_bound = side * s * folded_bound; + if (!is_integer(scaled_bound, tol)) return false; + const double bound = std::round(scaled_bound); + if (!fj_bin_in_int32(bound)) return false; + // A bit assignment can drive lhs anywhere in [-row_abs_sum, row_abs_sum]. + if (!fj_bin_in_int32(bound - row_abs_sum) || !fj_bin_in_int32(bound + row_abs_sum)) return false; + + pb.offsets.push_back((int32_t)pb.variables.size()); + pb.bound.push_back((int32_t)bound); + pb.cmax.push_back(row_cmax); + incoming_weight.push_back(weight); + return true; + }; + + for (int32_t r = 0; r < m; ++r) { + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + if (std::isfinite(lb) && !emit(r, lb, -1, left_w[r])) return false; + if (std::isfinite(ub) && !emit(r, ub, 1, right_w[r])) return false; + } + pb.n_constraints = (int32_t)pb.bound.size(); + if (pb.n_constraints <= 0) return false; + pb.nnz = (int32_t)pb.variables.size(); + + if (max_abs_coefficient <= 127.0) { + coefficient_bits = 8; + } else if (max_abs_coefficient <= 32767.0) { + coefficient_bits = 16; + } else { + return false; + } + + pb.variables.resize(pb.nnz + fj_bin_simd_padding, 0); + pb.coefficients.resize(pb.nnz + fj_bin_simd_padding, (coef_t)0); + + double w_min = std::numeric_limits::infinity(); + for (double w : incoming_weight) { + if (w > 0 && w < w_min) w_min = w; + } + double scale = 1.0; + if (std::isfinite(w_min) && w_min > 0) { + scale = (double)fj_bin_ddfw_init / w_min; + if (scale < 1.0) scale = 1.0; + } + for (double w : incoming_weight) { + int32_t scaled = w > 0 ? (int32_t)std::lround(w * scale) : fj_bin_ddfw_init; + if (scaled < 1) scaled = 1; + pb.initial_weight.push_back(scaled); + } + + pb.reverse_offsets.assign(n_bits + 1, 0); + for (int32_t k = 0; k < pb.nnz; ++k) pb.reverse_offsets[pb.variables[k] + 1]++; + for (int32_t v = 0; v < n_bits; ++v) pb.reverse_offsets[v + 1] += pb.reverse_offsets[v]; + pb.reverse_constraints.resize(pb.nnz); + pb.reverse_coefficients.resize(pb.nnz); + pb.reverse_to_csr.resize(pb.nnz); + pb.incident_row_cmax.resize(pb.nnz); + { + std::vector cursor(pb.reverse_offsets.begin(), pb.reverse_offsets.begin() + n_bits); + for (int32_t r = 0; r < pb.n_constraints; ++r) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t slot = cursor[pb.variables[k]]++; + pb.reverse_constraints[slot] = r; + pb.reverse_coefficients[slot] = pb.coefficients[k]; + pb.reverse_to_csr[slot] = k; + pb.incident_row_cmax[slot] = pb.cmax[r]; + } + } + } + const int32_t rpad = fj_bin_pf_dist > fj_bin_simd_padding ? fj_bin_pf_dist : fj_bin_simd_padding; + pb.reverse_constraints.resize(pb.nnz + rpad, 0); + pb.reverse_coefficients.resize(pb.nnz + rpad, (coef_t)0); + pb.incident_row_cmax.resize(pb.nnz + rpad, (coef_t)1); + + pb.objective.assign(n_bits, 0.0); + pb.objective_vars.clear(); + for (int32_t v = 0; v < n; ++v) { + pb.orig_objective[v] = obj[v]; + if (obj[v] == 0.0) continue; + for (int32_t bk = 0; bk < nbits[v]; ++bk) { + const int32_t bit = bit_start[v] + bk; + pb.objective[bit] = obj[v] * pb.bit_weight[bit]; + if (pb.objective[bit] != 0.0) pb.objective_vars.push_back(bit); + } + } + + // The cardinality census only reads as a count on rows of plain binaries. + pb.n_exchange_vars = 0; + pb.max_card_degree = 0; + return true; +} + // The integer engine. Feasibility is an exact compare against one bound per row, so there is no // tolerance arithmetic and no compensated summation anywhere below. template @@ -671,6 +912,9 @@ struct fj_bin_engine_t { // Mean absolute nonzero objective coefficient; the unit of the objective score term. double obj_magnitude{1.0}; double incumbent_objective{0}; + // sum(obj_j * L_j), folded out of the encoded objective and carried here so both tracked + // objectives hold the model's own value. Zero on the all-binary path. + double objective_offset{0}; double best_objective{std::numeric_limits::infinity()}; int32_t max_weight{1}; bool feasible_found{false}; @@ -744,7 +988,7 @@ struct fj_bin_engine_t { } } - double objective = 0; + double objective = objective_offset; for (int32_t v = 0; v < pb.n_variables; ++v) objective += pb.objective[v] * (double)best_assign[v]; const double drift = std::fabs(objective - best_objective); @@ -845,7 +1089,7 @@ struct fj_bin_engine_t { row_slack[r] = slack; if (slack < 0) set_violated(r); } - incumbent_objective = 0; + incumbent_objective = objective_offset; for (int32_t v = 0; v < pb.n_variables; ++v) incumbent_objective += pb.objective[v] * assign[v]; nnz_touched += pb.nnz; rebuild_scores(); @@ -1021,9 +1265,22 @@ struct fj_bin_engine_t { { auto& h_assign = climber.h_assignment; auto& h_best = climber.h_best_assignment; - for (int32_t v = 0; v < pb.n_variables; ++v) { - h_assign[v] = (f_t)assign[v]; - h_best[v] = (f_t)assign[v]; + if (pb.encoded) { + for (int32_t v = 0; v < pb.n_original; ++v) { + h_assign[v] = (f_t)pb.var_offset[v]; + h_best[v] = (f_t)pb.var_offset[v]; + } + for (int32_t b = 0; b < pb.n_variables; ++b) { + if (!assign[b]) continue; + const int32_t v = pb.bit_owner[b]; + h_assign[v] += (f_t)pb.bit_weight[b]; + h_best[v] += (f_t)pb.bit_weight[b]; + } + } else { + for (int32_t v = 0; v < pb.n_variables; ++v) { + h_assign[v] = (f_t)assign[v]; + h_best[v] = (f_t)assign[v]; + } } climber.h_incumbent_objective = (f_t)incumbent_objective; climber.h_best_objective = (f_t)best_objective; @@ -1537,7 +1794,10 @@ struct fj_bin_engine_t { if (feasible_found) { cuopt_assert((int32_t)best_assign.size() == pb.n_variables, "incumbent size mismatch"); assign = best_assign; - if (shared_incumbent && shared_incumbent->adopt((f_t)best_objective, adopt_buffer)) { + // The shared buffer holds decoded integers, so the flat 0/1 read below only lines up when + // engine variables are the model's own variables. + if (!pb.encoded && shared_incumbent && + shared_incumbent->adopt((f_t)best_objective, adopt_buffer)) { for (int32_t v = 0; v < pb.n_variables; ++v) assign[v] = (int8_t)(adopt_buffer[v] >= 0.5 ? 1 : 0); } @@ -1608,10 +1868,33 @@ struct fj_bin_engine_t { const int32_t n = pb.n_variables, m = pb.n_constraints; const auto& h_assign = climber.h_assignment; - assign.resize(n); - for (int32_t v = 0; v < n; ++v) { - const double val = (double)h_assign[v]; - assign[v] = (int8_t)(val >= 0.5 ? 1 : 0); + assign.assign(n, 0); + if (pb.encoded) { + // Descending weight, so the bit pattern reproduces the start value wherever it is + // representable: with exact closure that is every integer of the domain. + std::vector> bits_of(pb.n_original); + for (int32_t b = 0; b < n; ++b) bits_of[pb.bit_owner[b]].push_back(b); + for (int32_t v = 0; v < pb.n_original; ++v) { + long residual = std::lround((double)h_assign[v] - pb.var_offset[v]); + if (residual < 0) residual = 0; + auto& bits = bits_of[v]; + std::sort(bits.begin(), bits.end(), [&](int32_t a, int32_t b) { + return pb.bit_weight[a] > pb.bit_weight[b]; + }); + for (int32_t b : bits) { + const long w = std::lround(pb.bit_weight[b]); + if (w <= residual) { + assign[b] = 1; + residual -= w; + } + } + cuopt_assert(residual == 0, "greedy bit encode left the start value unrepresented"); + } + } else { + for (int32_t v = 0; v < n; ++v) { + const double val = (double)h_assign[v]; + assign[v] = (int8_t)(val >= 0.5 ? 1 : 0); + } } seed_assign = assign; best_assign = assign; @@ -1645,6 +1928,12 @@ struct fj_bin_engine_t { cuopt_assert(std::isfinite(obj_magnitude) && obj_magnitude > 0, "objective magnitude unit must be finite and positive"); + objective_offset = 0; + if (pb.encoded) { + for (int32_t v = 0; v < pb.n_original; ++v) + objective_offset += pb.orig_objective[v] * pb.var_offset[v]; + } + argmax_tile = fj_bin_argmax_tile(); objective_weight = seeded_weight > 0 ? seeded_weight : 0; seed_objective_weight = objective_weight; @@ -1748,7 +2037,13 @@ struct fj_bin_engine_t { } if (iters % climber.diversity_callback_interval == 0 && climber.diversity_callback) { auto& h_assign = climber.h_assignment; - for (int32_t v = 0; v < pb.n_variables; ++v) h_assign[v] = (f_t)assign[v]; + if (pb.encoded) { + for (int32_t v = 0; v < pb.n_original; ++v) h_assign[v] = (f_t)pb.var_offset[v]; + for (int32_t b = 0; b < pb.n_variables; ++b) + if (assign[b]) h_assign[pb.bit_owner[b]] += (f_t)pb.bit_weight[b]; + } else { + for (int32_t v = 0; v < pb.n_variables; ++v) h_assign[v] = (f_t)assign[v]; + } climber.diversity_callback((f_t)incumbent_objective, h_assign); } @@ -1806,6 +2101,34 @@ bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, const fj_bin_scan_t scan = fj_bin_scan(climber); if (scan.reject != fj_binary_reject_t::none) { + // A non-binary variable is the one rejection the encoding can answer: the model may still be + // all-integer with finite domains. Every other reason fails the encoded model just the same. + if (scan.reject == fj_binary_reject_t::non_binary_var) { + // The width the encoded coefficients need is only known once they are built, so probe with + // int16 and rebuild on int8 for the narrower kernel when that is enough. + fj_bin_engine_t probe; + int bits = 0; + if (fj_bin_encode(climber, probe.pb, bits)) { + if (bits == 8) { + fj_bin_engine_t engine8; + int bits8 = 0; + if (fj_bin_encode(climber, engine8.pb, bits8)) { + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path enabled (encoded int8): %d bits, %d rows", + climber.log_prefix.c_str(), + engine8.pb.n_variables, + engine8.pb.n_constraints); + engine8.solve(climber, time_limit, work_unit_limit); + return true; + } + } + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path enabled (encoded int16): %d bits, %d rows", + climber.log_prefix.c_str(), + probe.pb.n_variables, + probe.pb.n_constraints); + probe.solve(climber, time_limit, work_unit_limit); + return true; + } + } CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s (row %d, var %d)", climber.log_prefix.c_str(), fj_binary_reject_name(scan.reject), From 555848a976f0927357a528caa46ba1b6ab620ac1 Mon Sep 17 00:00:00 2001 From: yboucher Date: Mon, 24 Aug 2026 09:54:01 -0700 Subject: [PATCH 56/61] feaspump in early cpufj --- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 312 ++++++++++++------ 1 file changed, 219 insertions(+), 93 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index e61bc18697..360c9da6ab 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -2583,7 +2583,133 @@ constexpr int32_t fj_max_refresh_stretch = 8; // Above this a short LP spends more time moving the matrix than it can pay back as a seed, and the // wall budget the LP is allowed out of the lane's own. constexpr int64_t fj_lp_seed_nnz_limit = 8'000'000; -constexpr double fj_lp_seed_budget_s = 2; +constexpr double fj_lp_pump_max_budget_s = 2.0; +constexpr double fj_lp_pump_budget_share = 0.25; +constexpr int32_t fj_lp_pump_projections = 3; + +// One dual simplex solve of a relaxation on the calling thread. Reports whether the returned point +// is usable: a vertex reached at a limit is dual feasible and still worth rounding. +template +static bool solve_lp_relaxation(const simplex::user_problem_t& relaxation, + double time_limit, + std::vector& x) +{ + simplex::lp_status_t status = simplex::lp_status_t::UNSET; + double seconds = 0; + + // solve_linear_program_advanced, whose status separates a limit -- which leaves a usable vertex + // behind -- from infeasibility. Guarded on f_t because dual simplex is only built for double. + if constexpr (std::is_same_v) { + simplex_solver_settings_t lp_settings; + lp_settings.relaxation = true; + lp_settings.time_limit = time_limit; + lp_settings.log.log = false; + // The portfolio already pins one CPU per lane, and the simplex default is + // omp_get_max_threads() - 1, which would open a second portfolio inside this lane's worker. + lp_settings.num_threads = 1; + + const f_t lp_start = tic(); + lp_problem_t converted(relaxation.handle_ptr, + relaxation.num_rows, + relaxation.num_cols, + relaxation.A.col_start[relaxation.A.n]); + std::vector new_slacks; + simplex::dualize_info_t dualize_info; + simplex::convert_user_problem(relaxation, lp_settings, converted, new_slacks, dualize_info); + + simplex::lp_solution_t lp_solution(converted.num_rows, converted.num_cols); + std::vector vstatus; + std::vector edge_norms; + status = simplex::solve_linear_program_advanced( + converted, lp_start, lp_settings, lp_solution, vstatus, edge_norms); + x = std::move(lp_solution.x); + seconds = toc(lp_start); + } + + const bool usable = status == simplex::lp_status_t::OPTIMAL || + status == simplex::lp_status_t::TIME_LIMIT || + status == simplex::lp_status_t::ITERATION_LIMIT || + status == simplex::lp_status_t::CONCURRENT_LIMIT || + status == simplex::lp_status_t::WORK_LIMIT; + CUOPT_LOG_DEBUG("CPUFJ LP relaxation: %s after %.3fs of %.3fs%s", + simplex::lp_status_to_string(status).c_str(), + seconds, + time_limit, + usable ? "" : ", discarded"); + return usable; +} + +// The L1 distance to a rounded point, as an exact LP. Every integer x gains a d with the pair +// x - d <= r and -x - d <= -r, so minimising sum(d) minimises sum(abs(x - r)). +template +static simplex::user_problem_t make_lp_distance_problem( + const simplex::user_problem_t& base, + fj_cpu_climber_t& fj_cpu, + const std::vector& rounded) +{ + std::vector integer_vars; + for (i_t var = 0; var < fj_cpu.view.pb.n_variables; ++var) + if (is_integer_var(fj_cpu, var)) integer_vars.push_back(var); + const i_t n_distance = (i_t)integer_vars.size(); + + simplex::user_problem_t result(base.handle_ptr); + result.num_rows = base.num_rows + 2 * n_distance; + result.num_cols = base.num_cols + n_distance; + + // The model's own objective is dropped: this LP measures distance alone. + result.objective.assign(result.num_cols, f_t{0}); + for (i_t k = 0; k < n_distance; ++k) result.objective[base.num_cols + k] = f_t{1}; + + result.lower = base.lower; + result.upper = base.upper; + result.lower.resize(result.num_cols, f_t{0}); + result.upper.resize(result.num_cols, std::numeric_limits::infinity()); + + result.rhs = base.rhs; + result.row_sense = base.row_sense; + result.rhs.reserve(result.num_rows); + result.row_sense.reserve(result.num_rows); + for (i_t k = 0; k < n_distance; ++k) { + result.rhs.push_back(rounded[integer_vars[k]]); + result.row_sense.push_back('L'); + result.rhs.push_back(-rounded[integer_vars[k]]); + result.row_sense.push_back('L'); + } + result.range_rows = base.range_rows; + result.range_value = base.range_value; + result.num_range_rows = base.num_range_rows; + + const i_t base_nnz = base.A.col_start[base.A.n]; + csc_matrix_t matrix(result.num_rows, result.num_cols, base_nnz + 4 * n_distance); + i_t out = 0; + i_t next_integer = 0; + for (i_t j = 0; j < base.num_cols; ++j) { + matrix.col_start[j] = out; + for (i_t p = base.A.col_start[j]; p < base.A.col_start[j + 1]; ++p) { + matrix.i[out] = base.A.i[p]; + matrix.x[out++] = base.A.x[p]; + } + if (next_integer < n_distance && integer_vars[next_integer] == j) { + const i_t row = base.num_rows + 2 * next_integer++; + matrix.i[out] = row; + matrix.x[out++] = f_t{1}; + matrix.i[out] = row + 1; + matrix.x[out++] = f_t{-1}; + } + } + for (i_t k = 0; k < n_distance; ++k) { + matrix.col_start[base.num_cols + k] = out; + const i_t row = base.num_rows + 2 * k; + matrix.i[out] = row; + matrix.x[out++] = f_t{-1}; + matrix.i[out] = row + 1; + matrix.x[out++] = f_t{-1}; + } + matrix.col_start[result.num_cols] = out; + cuopt_assert(out == base_nnz + 4 * n_distance, "distance problem nonzero count mismatch"); + result.A = std::move(matrix); + return result; +} constexpr int32_t fj_bound_prop_rounds = 10; // A deduction is committed only when it moves a bound by more than this many absolute tolerances. @@ -2757,105 +2883,102 @@ static void apply_bound_propagation(fj_cpu_climber_t& fj_cpu) fj_cpu.n_binary_vars + fj_cpu.n_integer_vars); } -// Rounds the LP relaxation into this lane's start point. Solved by dual simplex on the lane's own -// thread rather than during portfolio construction, so the other seven start searching immediately. +// A bounded feasibility pump for the LP lane, run on the lane's own thread. An integral-feasible +// projection is published; otherwise FJ starts from the least violated rounding the pump saw. template -static void apply_lp_rounded_seed(fj_cpu_climber_t& fj_cpu) +static void apply_lp_rounded_seed(fj_cpu_climber_t& fj_cpu, f_t lane_time_limit) { if (!fj_cpu.use_lp_seed || fj_cpu.pb_ptr == nullptr) return; if (fj_cpu.view.pb.nnz > fj_lp_seed_nnz_limit) return; - simplex::user_problem_t relaxation(fj_cpu.pb_ptr->handle_ptr); - fj_cpu.pb_ptr->get_host_user_problem(relaxation); + const double budget = + std::min(fj_lp_pump_max_budget_s, fj_lp_pump_budget_share * (double)lane_time_limit); + if (budget <= 0) return; - simplex_solver_settings_t lp_settings; - lp_settings.relaxation = true; - lp_settings.time_limit = fj_lp_seed_budget_s; - lp_settings.log.log = false; + simplex::user_problem_t base(fj_cpu.pb_ptr->handle_ptr); + fj_cpu.pb_ptr->get_host_user_problem(base); - std::vector relaxed; - simplex::lp_status_t lp_status = simplex::lp_status_t::UNSET; - double lp_seconds = 0; + const auto started = std::chrono::steady_clock::now(); + const i_t n_variables = fj_cpu.view.pb.n_variables; - // solve_linear_program_advanced rather than simplex::solve, whose collapsed int return cannot - // separate a limit -- which leaves a usable vertex behind -- from infeasibility, which does not. - // Guarded on f_t because dual simplex is only instantiated for double. - if constexpr (std::is_same_v) { - const f_t lp_start = tic(); - lp_problem_t converted(relaxation.handle_ptr, - relaxation.num_rows, - relaxation.num_cols, - relaxation.A.col_start[relaxation.A.n]); - std::vector new_slacks; - simplex::dualize_info_t dualize_info; - simplex::convert_user_problem(relaxation, lp_settings, converted, new_slacks, dualize_info); + std::vector rounded; + std::vector selected; + f_t selected_violation = -std::numeric_limits::infinity(); - simplex::lp_solution_t lp_solution(converted.num_rows, converted.num_cols); - std::vector vstatus; - std::vector edge_norms; - lp_status = simplex::solve_linear_program_advanced( - converted, lp_start, lp_settings, lp_solution, vstatus, edge_norms); - relaxed = lp_solution.x; - lp_seconds = toc(lp_start); - } + for (int32_t projection = 0; projection < fj_lp_pump_projections; ++projection) { + const double remaining = + budget - std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + if (remaining <= 0) break; - // A vertex reached at a limit is dual feasible and still worth rounding. The remaining - // terminations leave nothing to round. - const bool usable = lp_status == simplex::lp_status_t::OPTIMAL || - lp_status == simplex::lp_status_t::TIME_LIMIT || - lp_status == simplex::lp_status_t::ITERATION_LIMIT || - lp_status == simplex::lp_status_t::CONCURRENT_LIMIT || - lp_status == simplex::lp_status_t::WORK_LIMIT; - CUOPT_LOG_DEBUG("%sCPUFJ LP seed: %s after %.3fs of %.3fs%s", - fj_cpu.log_prefix.c_str(), - simplex::lp_status_to_string(lp_status).c_str(), - lp_seconds, - fj_lp_seed_budget_s, - usable ? "" : ", discarded"); - if (!usable) return; + // Projection 0 is the plain relaxation; the rest chase the previous rounding. + const auto distance = projection == 0 ? simplex::user_problem_t(base.handle_ptr) + : make_lp_distance_problem(base, fj_cpu, rounded); + const auto& relaxation = projection == 0 ? base : distance; - const i_t n_variables = fj_cpu.view.pb.n_variables; - // convert_user_problem appends slacks, so the model's own variables are the leading columns. - cuopt_assert((i_t)relaxed.size() >= n_variables, "dual simplex returned too few columns"); + std::vector x; + if (!solve_lp_relaxation(relaxation, remaining, x)) break; + // convert_user_problem appends slacks, so the model's own variables are the leading columns. + if ((i_t)x.size() < n_variables) break; - std::vector candidate(n_variables); - for (i_t var = 0; var < n_variables; ++var) { - cuopt_assert(isfinite(relaxed[var]), "dual simplex returned a non-finite value"); - const auto bounds = fj_cpu.h_var_bounds[var].get(); - const f_t lower = get_lower(bounds); - const f_t upper = get_upper(bounds); - f_t value = std::clamp(relaxed[var], lower, upper); - if (is_integer_var(fj_cpu, var)) { - value = round(value); - // Rounding can leave the bounds, and a variable with no integral value inside them cannot be - // seeded at all without breaking the engine's integrality invariant. - if (value < lower || value > upper) return; + rounded.resize(n_variables); + cuopt::pcgenerator_t rng(fj_cpu.settings.seed); + bool valid = true; + for (i_t var = 0; var < n_variables && valid; ++var) { + const auto bounds = fj_cpu.h_var_bounds[var].get(); + const f_t lower = get_lower(bounds); + const f_t upper = get_upper(bounds); + f_t value = std::clamp(x[var], lower, upper); + if (!isfinite(value)) { + valid = false; + break; + } + if (is_integer_var(fj_cpu, var)) { + // Rounded up with probability equal to the fractional part, so successive projections of + // the same point explore different corners. + const f_t fraction = value - floor(value); + value = rng.next_double() < fraction ? ceil(value) : floor(value); + // A variable with no integral value inside its bounds cannot be seeded at all without + // breaking the engine's integrality invariant. + valid = value >= lower && value <= upper; + } + rounded[var] = value; } - candidate[var] = value; - } - - fj_cpu.h_assignment = candidate; - fj_cpu.h_best_assignment = candidate; - recompute_lhs(fj_cpu); + if (!valid) break; - // The rounded point can already be integral-feasible. It never passed through apply_move, so the - // incumbent is recorded here through the same contract that path uses. - if (fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { - fj_cpu.h_best_objective = - fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; - fj_cpu.feasible_found = true; - CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", - fj_cpu.log_prefix.c_str(), - fj_cpu.h_best_objective); - if (fj_cpu.improvement_callback) { - fj_cpu.improvement_callback(fj_cpu.h_incumbent_objective, - fj_cpu.h_assignment, - fj_cpu.work_units_elapsed.load(std::memory_order_acquire)); + fj_cpu.h_assignment = rounded; + recompute_lhs(fj_cpu); + // total_violations sums a non-positive excess, so the greater value is the closer point. + if (fj_cpu.total_violations > selected_violation) { + selected_violation = fj_cpu.total_violations; + selected = rounded; } - if (fj_cpu.shared_incumbent) { - fj_cpu.shared_incumbent->publish(fj_cpu.h_incumbent_objective, fj_cpu.h_assignment); + + // The rounded point can already be integral-feasible. It never passed through apply_move, so + // the incumbent is recorded here through the same contract that path uses. + if (fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { + fj_cpu.h_best_assignment = rounded; + fj_cpu.h_best_objective = + fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; + fj_cpu.feasible_found = true; + CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", + fj_cpu.log_prefix.c_str(), + fj_cpu.h_best_objective); + if (fj_cpu.improvement_callback) { + fj_cpu.improvement_callback(fj_cpu.h_incumbent_objective, + fj_cpu.h_assignment, + fj_cpu.work_units_elapsed.load(std::memory_order_acquire)); + } + if (fj_cpu.shared_incumbent) { + fj_cpu.shared_incumbent->publish(fj_cpu.h_incumbent_objective, fj_cpu.h_assignment); + } + return; } } + + if (selected.empty()) return; + fj_cpu.h_assignment = selected; + fj_cpu.h_best_assignment = selected; + recompute_lhs(fj_cpu); } template @@ -2865,15 +2988,19 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w // Precedes the dispatch below because a variable it squeezes to [0,1] can bring the whole model // into the binary engine's shape. apply_bound_propagation(*fj_cpu); - const f_t prop_seconds = - fj_cpu->use_bound_prop + // Also ahead of the dispatch, so an all-binary model gets the same LP-derived start. + apply_lp_rounded_seed(*fj_cpu, in_time_limit); + + const bool paid_setup = fj_cpu->use_bound_prop || fj_cpu->use_lp_seed; + const f_t setup_seconds = + paid_setup ? std::chrono::duration(std::chrono::high_resolution_clock::now() - solve_start).count() : f_t{0}; + const f_t remaining = std::max(f_t{0}, in_time_limit - setup_seconds); + if (remaining <= f_t{0}) return; // problem fits the binary fastpath shape? run it (engine is solve-local) - if (try_cpufj_binary_solve(*fj_cpu, in_time_limit - prop_seconds, work_unit_limit)) return; - - apply_lp_rounded_seed(*fj_cpu); + if (try_cpufj_binary_solve(*fj_cpu, remaining, work_unit_limit)) return; i_t local_mins = 0; // The LP comes out of this lane's own budget; every other lane's clock starts where it did. @@ -4147,12 +4274,11 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; const f_t obj_weight_floor[4] = {1, 4, 32, 1}; - // One structural start per lane; lanes 0 and 4 keep the shared anchor. Lane 7 also keeps it here, - // because its replacement is an LP solved inside that lane's own task rather than in setup. - climber.use_lp_seed = lane % 8 == 7; + // One structural start per lane; lanes 0, 4 and 7 keep the shared anchor here. Lane 4's is + // replaced inside its own task by the LP pump, so construction does not wait on an LP. + climber.use_lp_seed = lane % 8 == 4; - // Half the portfolio searches the propagated model, half the model as parsed. Off the LP lane, so - // one lane does not carry both setup passes. + // Half the portfolio searches the propagated model, half the model as parsed. climber.use_bound_prop = lane % 2 == 0; climber.use_weight_donation = (lane % 8 == 5) || (lane % 8 == 6); From b6f4f43625ff9e9046fecf1b60d16948d992fec7 Mon Sep 17 00:00:00 2001 From: yboucher Date: Mon, 24 Aug 2026 12:44:48 -0700 Subject: [PATCH 57/61] cpufj batching --- .../feasibility_jump_impl_common.cuh | 5 +- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 344 +++++++++++++++++- .../feasibility_jump/fj_cpu.cuh | 26 ++ .../feasibility_jump/fj_cpu_binary.cu | 2 +- cpp/src/utilities/macros.cuh | 2 +- 5 files changed, 370 insertions(+), 9 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh index 046e138c5b..6535794e08 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh @@ -196,7 +196,6 @@ HDI f_t get_breakthrough_move(typename fj_t::climber_data_t::view_t fj auto bounds = fj.pb.variable_bounds[var_idx]; f_t v_lb = get_lower(bounds); f_t v_ub = get_upper(bounds); - cuopt_assert(isfinite(v_lb) || isfinite(v_ub), "unexpected free variable"); cuopt_assert(v_lb <= v_ub, "invalid bounds"); cuopt_assert(fj.pb.check_variable_within_bounds(var_idx, fj.incumbent_assignment[var_idx]), "invalid incumbent assignment"); @@ -220,10 +219,12 @@ HDI f_t get_breakthrough_move(typename fj_t::climber_data_t::view_t fj new_val = old_val + delta_ij; } - // fallback + // A positive coefficient gives a negative delta, so only the lower bound can be the one broken, + // and a broken bound is finite. Free and half-free variables therefore land here finite too. if (!fj.pb.check_variable_within_bounds(var_idx, new_val)) { new_val = obj_coeff > 0 ? v_lb : v_ub; } + cuopt_assert(isfinite(new_val), "breakthrough move left the representable range"); return new_val; } diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 360c9da6ab..4f1500031b 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -160,7 +160,8 @@ std::pair feas_score_constraint(fj_cpu_climber_t& fj_cpu, { const auto& fj = fj_cpu.view; cuopt_assert(isfinite(delta), "invalid delta"); - cuopt_assert(cstr_coeff != 0 && isfinite(cstr_coeff), "invalid coefficient"); + // A model may store explicit zeros, and a zero coefficient contributes nothing to the row. + cuopt_assert(isfinite(cstr_coeff), "invalid coefficient"); f_t base_feas = 0; f_t bonus_robust = 0; @@ -385,6 +386,11 @@ static void precompute_problem_features(fj_cpu_climber_t& fj_cpu) fj_cpu.problem_density = (double)total_nnz / ((double)n_vars * n_cstrs); } +// Greedy first-fit colouring of the variable co-occurrence graph, where each row is a clique. The +// adjacency is walked per variable and never stored: the clique expansion is far larger than nnz. +template +static void compute_variable_coloring(fj_cpu_climber_t& fj_cpu); + template static void log_regression_features(fj_cpu_climber_t& fj_cpu, double time_window_ms, @@ -548,6 +554,207 @@ static inline std::pair range_for_constraint(fj_cpu_climber_t +static void compute_variable_coloring(fj_cpu_climber_t& fj_cpu) +{ + const i_t n_vars = fj_cpu.view.pb.n_variables; + const i_t n_cstrs = fj_cpu.view.pb.n_constraints; + + i_t max_row_length = 0; + double clique_edges = 0; + for (i_t row = 0; row < n_cstrs; ++row) { + const i_t length = fj_cpu.h_offsets[row + 1] - fj_cpu.h_offsets[row]; + max_row_length = std::max(max_row_length, length); + if (length > 1) clique_edges += (double)length * (length - 1) / 2.0; + } + if (n_vars <= 0 || max_row_length <= 0) return; + + const double class_size = (double)n_vars / max_row_length; + const double edges_per_nnz = clique_edges / std::max(1, (double)fj_cpu.view.pb.nnz); + if (class_size < fj_batch_min_class_size || edges_per_nnz > fj_batch_max_edges_per_nnz) { + CUOPT_LOG_DEBUG("CPUFJ move batching declined: class size %.2f, clique edges/nnz %.2f", + class_size, + edges_per_nnz); + return; + } + + const auto started = std::chrono::steady_clock::now(); + fj_cpu.h_var_color.assign(n_vars, -1); + fj_cpu.n_colors = 0; + std::vector neighbor_stamp(n_vars, -1); + std::vector color_stamp(n_vars, -1); + + for (i_t var = 0; var < n_vars; ++var) { + const auto [rev_begin, rev_end] = reverse_range_for_var(fj_cpu, var); + for (i_t p = rev_begin; p < rev_end; ++p) { + const auto [begin, end] = + range_for_constraint(fj_cpu, fj_cpu.h_reverse_constraints[p]); + for (i_t k = begin; k < end; ++k) { + const i_t other = fj_cpu.h_variables[k]; + if (other == var || neighbor_stamp[other] == var) continue; + neighbor_stamp[other] = var; + const i_t taken = fj_cpu.h_var_color[other]; + if (taken >= 0) color_stamp[taken] = var; + } + } + + i_t color = 0; + while (color < fj_cpu.n_colors && color_stamp[color] == var) ++color; + if (color == fj_cpu.n_colors) ++fj_cpu.n_colors; + fj_cpu.h_var_color[var] = color; + } + + fj_cpu.h_var_best_score.assign(n_vars, fj_staged_score_t::invalid()); + fj_cpu.h_var_best_delta.assign(n_vars, f_t{0}); + fj_cpu.h_var_best_stamp.assign(n_vars, 0); + fj_cpu.h_var_best_rowsum.assign(n_vars, 0); + fj_cpu.h_var_bucket_stamp.assign(n_vars, 0); + fj_cpu.batch_size_hist.assign(fj_batch_hist_bins, 0); + fj_cpu.h_color_candidates.assign(fj_cpu.n_colors, {}); + fj_cpu.h_color_epoch.assign(fj_cpu.n_colors, 0); + fj_cpu.var_best_epoch = 1; + + CUOPT_LOG_DEBUG("CPUFJ move batching: %d colours over %d variables in %.3f ms", + fj_cpu.n_colors, + n_vars, + std::chrono::duration(std::chrono::steady_clock::now() - + started) + .count()); +} + +// Sum of the versions of the rows a variable appears in. Versions only ever increase, so an +// unchanged sum means no incident row has been touched. +template +static inline int64_t incident_row_version_sum(fj_cpu_climber_t& fj_cpu, i_t var_idx) +{ + const auto [begin, end] = reverse_range_for_var(fj_cpu, var_idx); + int64_t sum = 0; + for (i_t p = begin; p < end; ++p) + sum += fj_cpu.h_cstr_version[fj_cpu.h_reverse_constraints[p]]; + return sum; +} + +// Records a candidate move for its variable. The table keeps a best per variable, independent of +// the argmax the caller is tracking, which is what lets a batch be assembled later. +template +static inline void record_var_best_move(fj_cpu_climber_t& fj_cpu, + i_t var_idx, + fj_staged_score_t score, + f_t delta) +{ + if (!fj_cpu.use_move_batching) return; + if (!(score > fj_staged_score_t::zero())) return; + + const bool current = fj_cpu.h_var_best_stamp[var_idx] == fj_cpu.var_best_epoch; + if (current && !(score > fj_cpu.h_var_best_score[var_idx])) return; + + fj_cpu.h_var_best_score[var_idx] = score; + fj_cpu.h_var_best_delta[var_idx] = delta; + fj_cpu.h_var_best_stamp[var_idx] = fj_cpu.var_best_epoch; + fj_cpu.h_var_best_rowsum[var_idx] = incident_row_version_sum(fj_cpu, var_idx); + + const i_t color = fj_cpu.h_var_color[var_idx]; + cuopt_assert(color >= 0 && color < fj_cpu.n_colors, "variable has no colour"); + if (fj_cpu.h_color_epoch[color] != fj_cpu.var_best_epoch) { + fj_cpu.h_color_candidates[color].clear(); + fj_cpu.h_color_epoch[color] = fj_cpu.var_best_epoch; + } + if (fj_cpu.h_var_bucket_stamp[var_idx] == fj_cpu.var_best_epoch) return; + fj_cpu.h_var_bucket_stamp[var_idx] = fj_cpu.var_best_epoch; + fj_cpu.h_color_candidates[color].push_back(var_idx); +} + +// Retires the whole table in constant time. Called wherever the weights or the assignment move far +// enough that every cached score is suspect. +template +static inline void retire_var_best_moves(fj_cpu_climber_t& fj_cpu) +{ + if (!fj_cpu.use_move_batching) return; + ++fj_cpu.var_best_epoch; +} + +// Companions per batch attempt, as min, median, max and mean. A median landing in the saturating +// last bin reads as that bin's index, and max_batch_size carries the true tail. +template +static void log_batch_distribution(const fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.n_batch_attempts == 0) return; + + int32_t smallest = -1; + int32_t median = -1; + int64_t seen = 0; + for (size_t bin = 0; bin < fj_cpu.batch_size_hist.size(); ++bin) { + if (fj_cpu.batch_size_hist[bin] == 0) continue; + if (smallest < 0) smallest = (int32_t)bin; + seen += fj_cpu.batch_size_hist[bin]; + if (median < 0 && 2 * seen > fj_cpu.n_batch_attempts) median = (int32_t)bin; + } + + CUOPT_LOG_DEBUG( + "%sCPUFJ batch companions: min %d median %d max %lld mean %.3f over %lld attempts, %lld total, " + "%d colours, batching %s", + fj_cpu.log_prefix.c_str(), + smallest, + median, + (long long)fj_cpu.max_batch_size, + (double)fj_cpu.n_batched_moves / (double)fj_cpu.n_batch_attempts, + (long long)fj_cpu.n_batch_attempts, + (long long)fj_cpu.n_batched_moves, + fj_cpu.n_colors, + fj_cpu.use_move_batching ? "on" : "off"); +} + +// Companions for the chosen move: same colour, so they share no row with it or with each other and +// their recorded scores and deltas hold as the batch is applied. Excludes the chosen move itself. +template +static void collect_move_batch(fj_cpu_climber_t& fj_cpu, + fj_move_t chosen, + std::vector& batch) +{ + batch.clear(); + if (!fj_cpu.use_move_batching) return; + + const i_t color = fj_cpu.h_var_color[chosen.var_idx]; + cuopt_assert(color >= 0 && color < fj_cpu.n_colors, "chosen move has no colour"); + if (fj_cpu.h_color_epoch[color] != fj_cpu.var_best_epoch) return; + + for (i_t var_idx : fj_cpu.h_color_candidates[color]) { + if (var_idx == chosen.var_idx) continue; + if (fj_cpu.h_var_best_stamp[var_idx] != fj_cpu.var_best_epoch) continue; + if (!(fj_cpu.h_var_best_score[var_idx] > fj_staged_score_t::zero())) continue; + if (fj_cpu.h_var_best_rowsum[var_idx] != incident_row_version_sum(fj_cpu, var_idx)) + continue; + + batch.push_back({var_idx, fj_cpu.h_var_best_delta[var_idx]}); + // Invalidated so a second pass over the bucket cannot apply the move twice. + fj_cpu.h_var_best_stamp[var_idx] = 0; + } + + ++fj_cpu.n_batch_attempts; + fj_cpu.n_batched_moves += (int64_t)batch.size(); + ++fj_cpu.batch_size_hist[std::min(batch.size(), fj_cpu.batch_size_hist.size() - 1)]; + if ((int64_t)batch.size() > fj_cpu.max_batch_size) + fj_cpu.max_batch_size = (int64_t)batch.size(); + if (fj_cpu.n_batch_attempts == fj_batch_probe_attempts && + (double)fj_cpu.n_batched_moves < fj_batch_min_yield * (double)fj_batch_probe_attempts) { + fj_cpu.use_move_batching = false; + CUOPT_LOG_DEBUG("%sCPUFJ move batching off: %lld companions over %lld attempts", + fj_cpu.log_prefix.c_str(), + (long long)fj_cpu.n_batched_moves, + (long long)fj_cpu.n_batch_attempts); + } +} + template static inline bool check_variable_within_bounds(fj_cpu_climber_t& fj_cpu, i_t var_idx, @@ -559,6 +766,69 @@ static inline bool check_variable_within_bounds(fj_cpu_climber_t& fj_c return within_bounds; } +// Names the first variable whose assignment sits outside its own bounds, so the writer that left it +// there is identified by the call site. Scans, and is only reached through cuopt_func_call. +template +static void audit_assignment_bounds(fj_cpu_climber_t& fj_cpu, const char* site) +{ + for (i_t var = 0; var < fj_cpu.view.pb.n_variables; ++var) { + const f_t val = fj_cpu.h_assignment[var]; + auto bounds = fj_cpu.h_var_bounds[var].get(); + const bool inbox = fj_cpu.view.pb.check_variable_within_bounds(var, val); + const bool integral = + var_t::INTEGER != fj_cpu.h_var_types[var] || fj_cpu.view.pb.is_integer(val); + if (inbox && integral) continue; + + // stderr and flushed, so the abort below cannot swallow it. + std::fprintf(stderr, + "%sCPUFJ %s left var %d at %.17g outside [%.17g, %.17g], integer %d\n", + fj_cpu.log_prefix.c_str(), + site, + (int)var, + (double)val, + (double)get_lower(bounds), + (double)get_upper(bounds), + (int)(var_t::INTEGER == fj_cpu.h_var_types[var])); + std::fflush(stderr); + cuopt_assert(false, "assignment left the variable bounds"); + return; + } +} + +// Reports the first objective variable get_breakthrough_move would reject, reading the value both +// from the climber's vector and through the view span so a bad value is told from a stale span. +template +static void audit_breakthrough_inputs(fj_cpu_climber_t& fj_cpu) +{ + for (auto var_idx : fj_cpu.h_objective_vars) { + const f_t viewed = fj_cpu.view.incumbent_assignment[var_idx]; + if (fj_cpu.view.pb.check_variable_within_bounds(var_idx, viewed)) continue; + + const f_t direct = fj_cpu.h_assignment[var_idx]; + auto bounds = fj_cpu.h_var_bounds[var_idx].get(); + auto viewed_bnd = fj_cpu.view.pb.variable_bounds[var_idx]; + // stderr and flushed, so the abort below cannot swallow it. + std::fprintf(stderr, + "%sCPUFJ breakthrough input var %d: direct %.17g viewed %.17g nan %d, bounds " + "direct [%.17g, %.17g] viewed [%.17g, %.17g], obj %.17g, degree %d, integer %d\n", + fj_cpu.log_prefix.c_str(), + (int)var_idx, + (double)direct, + (double)viewed, + (int)(viewed != viewed), + (double)get_lower(bounds), + (double)get_upper(bounds), + (double)get_lower(viewed_bnd), + (double)get_upper(viewed_bnd), + (double)fj_cpu.h_obj_coeffs[var_idx], + (int)(fj_cpu.h_reverse_offsets[var_idx + 1] - fj_cpu.h_reverse_offsets[var_idx]), + (int)(var_t::INTEGER == fj_cpu.h_var_types[var_idx])); + std::fflush(stderr); + cuopt_assert(false, "breakthrough move input out of bounds"); + return; + } +} + template static inline bool is_integer_var(fj_cpu_climber_t& fj_cpu, i_t var_idx) { @@ -639,6 +909,8 @@ static inline std::pair compute_score(fj_cpu_climber_t(fj_cpu, @@ -1043,6 +1315,8 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); bool smoothing = rng.next_float() <= fj_cpu.settings.parameters.weight_smoothing_probability; + retire_var_best_moves(fj_cpu); + if (smoothing) { smooth_weights(fj_cpu); return; @@ -1210,6 +1484,9 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, // update the assignment and objective proper fj_cpu.h_assignment[var_idx] = new_val; + // The clamp above passes a NaN straight through, and every comparison against one is false. + cuopt_assert(fj_cpu.view.pb.check_variable_within_bounds(var_idx, new_val), + "apply_move left the variable bounds"); // Kahan compensated summation, as for h_lhs. The incumbent objective is reported as-is, so it // cannot carry the drift of a long uncompensated chain of deltas. @@ -1248,6 +1525,8 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.h_objective_weight = min((f_t)fj_obj_weight_incumbent_cap, fj_cpu.h_objective_weight + (f_t)fj_obj_weight_incumbent_bump); + // The weight enters every score, and row versions cannot see it move. + retire_var_best_moves(fj_cpu); } } @@ -1438,6 +1717,7 @@ static thrust::tuple find_mtm_move( // reject this move if it would increase the target variable to a numerically unstable value if (fj_cpu.view.move_numerically_stable( val, new_val, infeasibility, fj_cpu.total_violations)) { + record_var_best_move(fj_cpu, var_idx, score, delta); if (best_score < score) { best_score = score; best_move = move; @@ -1451,6 +1731,7 @@ static thrust::tuple find_mtm_move( fj_cpu.h_best_objective < std::numeric_limits::infinity() && fj_cpu.h_incumbent_objective >= fj_cpu.h_best_objective + fj_cpu.settings.parameters.breakthrough_move_epsilon) { + cuopt_func_call(audit_breakthrough_inputs(fj_cpu)); for (auto var_idx : fj_cpu.h_objective_vars) { f_t old_val = fj_cpu.h_assignment[var_idx]; f_t new_val = get_breakthrough_move(fj_cpu.view, var_idx); @@ -1473,6 +1754,7 @@ static thrust::tuple find_mtm_move( if (fj_cpu.view.move_numerically_stable( old_val, new_val, infeasibility, fj_cpu.total_violations)) { + record_var_best_move(fj_cpu, var_idx, score, delta); if (best_score < score) { best_score = score; best_move = move; @@ -1880,6 +2162,7 @@ static void perturb(fj_cpu_climber_t& fj_cpu) fj_cpu.h_assignment = fj_cpu.h_best_assignment; if (fj_cpu.shared_incumbent) { fj_cpu.shared_incumbent->adopt(fj_cpu.h_best_objective, fj_cpu.h_assignment); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "shared adopt")); } } @@ -1897,6 +2180,7 @@ static void perturb(fj_cpu_climber_t& fj_cpu) ++fj_cpu.n_lhs_recompute_perturb; recompute_lhs(fj_cpu); + retire_var_best_moves(fj_cpu); } template @@ -1925,6 +2209,7 @@ static void restart_from_infeasible_checkpoint(fj_cpu_climber_t& fj_cp ++fj_cpu.n_lhs_recompute_restart; recompute_lhs(fj_cpu); invalidate_mtm_cache(fj_cpu); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "checkpoint restore")); } // Nonzeros per extra restart window, the cap on that, and how many windows a lane waits. @@ -1974,6 +2259,7 @@ static void track_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) invalidate_mtm_cache(fj_cpu); reset_infeasible_checkpoint(fj_cpu); fj_cpu.restores_since_improvement = 0; + cuopt_func_call(audit_assignment_bounds(fj_cpu, "randomized restart")); CUOPT_LOG_DEBUG("%sCPUFJ randomized restart at iteration %d", fj_cpu.log_prefix.c_str(), @@ -2291,6 +2577,11 @@ void finalize_fj_cpu_host_initialization( fj_cpu.h_objective_vars.resize(end - fj_cpu.h_objective_vars.begin()); fj_cpu.view.objective_vars = raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); + // get_breakthrough_move divides by the coefficient of every variable in here. + for (auto var_idx : fj_cpu.h_objective_vars) { + cuopt_assert(fj_cpu.h_obj_coeffs[var_idx] != f_t{0}, "null coefficient in the objective vars"); + cuopt_assert(isfinite((f_t)fj_cpu.h_obj_coeffs[var_idx]), "non-finite objective coefficient"); + } f_t abs_obj_sum = 0; for (auto var_idx : fj_cpu.h_objective_vars) { @@ -2332,6 +2623,7 @@ void finalize_fj_cpu_host_initialization( // Precompute static problem features for regression model precompute_problem_features(fj_cpu); + compute_variable_coloring(fj_cpu); } template @@ -2372,6 +2664,21 @@ static void finalize_fj_cpu_host_initialization_from_template( fj_cpu.h_incumbent_objective = tmpl.h_incumbent_objective; fj_cpu.h_objective_sumcomp = tmpl.h_objective_sumcomp; + // The colouring is structural, so it carries over; the score table is this climber's own. + fj_cpu.h_var_color = tmpl.h_var_color; + fj_cpu.n_colors = tmpl.n_colors; + if (fj_cpu.n_colors > 0) { + fj_cpu.h_var_best_score.assign(n_variables, fj_staged_score_t::invalid()); + fj_cpu.h_var_best_delta.assign(n_variables, f_t{0}); + fj_cpu.h_var_best_stamp.assign(n_variables, 0); + fj_cpu.h_var_best_rowsum.assign(n_variables, 0); + fj_cpu.h_var_bucket_stamp.assign(n_variables, 0); + fj_cpu.batch_size_hist.assign(fj_batch_hist_bins, 0); + fj_cpu.h_color_candidates.assign(fj_cpu.n_colors, {}); + fj_cpu.h_color_epoch.assign(fj_cpu.n_colors, 0); + fj_cpu.var_best_epoch = 1; + } + fj_cpu.n_binary_vars = tmpl.n_binary_vars; fj_cpu.n_integer_vars = tmpl.n_integer_vars; fj_cpu.avg_var_degree = tmpl.avg_var_degree; @@ -2503,6 +2810,14 @@ static std::unique_ptr> init_fj_cpu_from_host_lp( template static void sanity_checks(fj_cpu_climber_t& fj_cpu) { + // Assigning any of these wrappers from a plain vector rebinds its buffer and strands the span. + cuopt_assert(fj_cpu.view.incumbent_assignment.data() == fj_cpu.h_assignment.data(), + "incumbent_assignment span no longer covers h_assignment"); + cuopt_assert(fj_cpu.view.incumbent_lhs.data() == fj_cpu.h_lhs.data(), + "incumbent_lhs span no longer covers h_lhs"); + cuopt_assert(fj_cpu.view.pb.variable_bounds.data() == fj_cpu.h_var_bounds.data(), + "variable_bounds span no longer covers h_var_bounds"); + // Check that each variable is within its bounds for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) { f_t val = fj_cpu.h_assignment[var_idx]; @@ -2874,6 +3189,7 @@ static void apply_bound_propagation(fj_cpu_climber_t& fj_cpu) raft::device_span(fj_cpu.h_binary_indices.data(), fj_cpu.h_binary_indices.size()); if (clamped) recompute_lhs(fj_cpu); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "bound prop")); CUOPT_LOG_DEBUG("%sCPUFJ bound prop: %d passes, %d domains tightened, %d binary of %d integer", fj_cpu.log_prefix.c_str(), @@ -2945,7 +3261,9 @@ static void apply_lp_rounded_seed(fj_cpu_climber_t& fj_cpu, f_t lane_t } if (!valid) break; - fj_cpu.h_assignment = rounded; + // Copied in place: assigning the wrapper from a plain vector rebinds its buffer and leaves the + // incumbent_assignment span on freed memory. + std::copy(rounded.begin(), rounded.end(), fj_cpu.h_assignment.begin()); recompute_lhs(fj_cpu); // total_violations sums a non-positive excess, so the greater value is the closer point. if (fj_cpu.total_violations > selected_violation) { @@ -2956,7 +3274,7 @@ static void apply_lp_rounded_seed(fj_cpu_climber_t& fj_cpu, f_t lane_t // The rounded point can already be integral-feasible. It never passed through apply_move, so // the incumbent is recorded here through the same contract that path uses. if (fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { - fj_cpu.h_best_assignment = rounded; + std::copy(rounded.begin(), rounded.end(), fj_cpu.h_best_assignment.begin()); fj_cpu.h_best_objective = fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; fj_cpu.feasible_found = true; @@ -2976,9 +3294,10 @@ static void apply_lp_rounded_seed(fj_cpu_climber_t& fj_cpu, f_t lane_t } if (selected.empty()) return; - fj_cpu.h_assignment = selected; - fj_cpu.h_best_assignment = selected; + std::copy(selected.begin(), selected.end(), fj_cpu.h_assignment.begin()); + std::copy(selected.begin(), selected.end(), fj_cpu.h_best_assignment.begin()); recompute_lhs(fj_cpu); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "lp pump")); } template @@ -3003,6 +3322,7 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w if (try_cpufj_binary_solve(*fj_cpu, remaining, work_unit_limit)) return; i_t local_mins = 0; + std::vector batch_moves; // The LP comes out of this lane's own budget; every other lane's clock starts where it did. auto loop_start = (fj_cpu->use_lp_seed || fj_cpu->use_bound_prop) ? solve_start @@ -3128,6 +3448,13 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w } if (score > fj_staged_score_t::zero() && !should_perturb) { + // A 2-opt lift already commits two coupled moves, and its second half is scored against the + // state before both, so it stays on its own. + if (lift_companion.var_idx < 0) { + collect_move_batch(*fj_cpu, move, batch_moves); + for (const auto& batched : batch_moves) + apply_move(*fj_cpu, batched.var_idx, batched.value, false); + } apply_move(*fj_cpu, move.var_idx, move.value, false); if (lift_companion.var_idx >= 0) { apply_move(*fj_cpu, lift_companion.var_idx, lift_companion.value, false); @@ -3226,6 +3553,7 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w (long long)fj_cpu->n_checkpoint_restores, (long long)fj_cpu->n_checkpoint_snapshots, fj_cpu->max_restores_since_improvement); + log_batch_distribution(*fj_cpu); #if CPUFJ_TIMING_TRACE // Print final timing statistics @@ -4282,6 +4610,12 @@ void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, i climber.use_bound_prop = lane % 2 == 0; climber.use_weight_donation = (lane % 8 == 5) || (lane % 8 == 6); + + // Only where the colouring came out; n_colors is zero when the structure declined it. + climber.use_move_batching = + climber.n_colors > 0 && ((lane % 8 == 2) || (lane % 8 == 6)); + climber.use_move_batching = true; + if (climber.n_colors == 0) climber.use_move_batching = false; switch (lane % 8) { case 1: apply_lock_weighted_seed(climber); break; case 2: apply_aggressive_constraint_seed(climber); break; diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index ceddadbbdc..df86602d45 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -233,6 +233,32 @@ struct fj_cpu_climber_t { host_contiguous_set_t satisfied_constraints; bool feasible_found{false}; bool trigger_early_lhs_recomputation{false}; + + // Move batching over a colouring of the variable co-occurrence graph, where each row is a clique. + // Same colour means no shared row, so a batch of same-coloured moves has disjoint row support. + bool use_move_batching{false}; + i_t n_colors{0}; + std::vector h_var_color; + // Per variable, the best move seen since the epoch below, and the sum of its incident row + // versions at that moment. The entry is usable while both still match. + std::vector h_var_best_score; + std::vector h_var_best_delta; + std::vector h_var_best_stamp; + std::vector h_var_best_rowsum; + int64_t var_best_epoch{1}; + // Variables that entered the table with a positive score, bucketed by colour. Stale entries are + // skipped at selection, so each bucket carries the epoch it was last cleared in. + std::vector> h_color_candidates; + std::vector h_color_epoch; + // Membership is stamped separately from validity: a variable consumed by a batch is invalidated + // while staying in its bucket, so it cannot be enqueued twice in one epoch. + std::vector h_var_bucket_stamp; + int64_t n_batch_attempts{0}; + int64_t n_batched_moves{0}; + // Companions per attempt, in unit bins. The last bin saturates, so max_batch_size carries the + // tail exactly. + std::vector batch_size_hist; + int64_t max_batch_size{0}; f_t total_violations{0}; // Kahan compensation for total_violations, mirroring h_lhs_sumcomp. Reset wherever the total is // re-derived from the violated set. diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu index b648b8a0b4..7fe3bb1e39 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -329,7 +329,7 @@ static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) [&reverse_offsets, &reverse_constraints](int32_t v) { const auto first = reverse_constraints.begin() + reverse_offsets[v]; const auto last = reverse_constraints.begin() + reverse_offsets[v + 1]; - return thrust::adjacent_find(thrust::host, first, last) == last; + return std::adjacent_find(first, last) == last; }), "duplicate variable in CSR row"); diff --git a/cpp/src/utilities/macros.cuh b/cpp/src/utilities/macros.cuh index d36832015a..380851627f 100644 --- a/cpp/src/utilities/macros.cuh +++ b/cpp/src/utilities/macros.cuh @@ -14,7 +14,7 @@ // 3) heavy #ifdef ASSERT_MODE #include -#define cuopt_assert(val, msg) assert(val&& msg) +#define cuopt_assert(val, msg) assert((val) && msg) #define cuopt_func_call(func) func; #else #define cuopt_assert(val, msg) From c28f2ab6c3263d1852485680ce892327a0e26932 Mon Sep 17 00:00:00 2001 From: yboucher Date: Mon, 24 Aug 2026 13:26:09 -0700 Subject: [PATCH 58/61] fix Werror=unused flags --- cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 4f1500031b..5e16a8c4b3 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -184,8 +184,10 @@ std::pair feas_score_constraint(fj_cpu_climber_t& fj_cpu, f_t rhs = bounds[bound_idx] * sign; f_t old_lhs = current_lhs * sign; f_t new_lhs = moved_lhs * sign; - f_t old_slack = rhs - old_lhs; - f_t new_slack = rhs - new_lhs; + [[maybe_unused]] + f_t old_slack = rhs - old_lhs; + [[maybe_unused]] + f_t new_slack = rhs - new_lhs; cuopt_assert(isfinite(cstr_weight), "invalid weight"); cuopt_assert(cstr_weight >= 0, "invalid weight"); @@ -1839,8 +1841,6 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) fj_cpu.total_violations_sumcomp = 0; for (i_t cstr_idx = 0; cstr_idx < fj_cpu.view.pb.n_constraints; ++cstr_idx) { auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); - auto c_lb = fj_cpu.h_cstr_lb[cstr_idx]; - auto c_ub = fj_cpu.h_cstr_ub[cstr_idx]; auto delta_it = thrust::make_transform_iterator(thrust::make_counting_iterator(0), [&fj_cpu](i_t j) { return fj_cpu.h_coefficients[j] * fj_cpu.h_assignment[fj_cpu.h_variables[j]]; @@ -2578,7 +2578,7 @@ void finalize_fj_cpu_host_initialization( fj_cpu.view.objective_vars = raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); // get_breakthrough_move divides by the coefficient of every variable in here. - for (auto var_idx : fj_cpu.h_objective_vars) { + for ([[maybe_unused]] auto var_idx : fj_cpu.h_objective_vars) { cuopt_assert(fj_cpu.h_obj_coeffs[var_idx] != f_t{0}, "null coefficient in the objective vars"); cuopt_assert(isfinite((f_t)fj_cpu.h_obj_coeffs[var_idx]), "non-finite objective coefficient"); } From 72d73355ddcabe8847581b4598915e234c9821e7 Mon Sep 17 00:00:00 2001 From: yboucher Date: Tue, 25 Aug 2026 03:56:49 -0700 Subject: [PATCH 59/61] match main post rebase --- cpp/src/branch_and_bound/branch_and_bound.cpp | 16 +++------------- cpp/src/branch_and_bound/branch_and_bound.hpp | 8 ++------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/cpp/src/branch_and_bound/branch_and_bound.cpp b/cpp/src/branch_and_bound/branch_and_bound.cpp index c532e37918..29174148b7 100644 --- a/cpp/src/branch_and_bound/branch_and_bound.cpp +++ b/cpp/src/branch_and_bound/branch_and_bound.cpp @@ -524,8 +524,7 @@ void branch_and_bound_t::update_user_bound(f_t lower_bound) template bool branch_and_bound_t::set_solution_from_heuristics(const std::vector& solution, - heuristics_origin_t origin, - f_t* solver_objective) + heuristics_origin_t origin) { mutex_original_lp_.lock(); if (solution.size() != original_problem_.num_cols) { @@ -538,7 +537,6 @@ bool branch_and_bound_t::set_solution_from_heuristics(const std::vecto f_t obj = compute_objective(original_lp_, crushed_solution); mutex_original_lp_.unlock(); - if (solver_objective != nullptr) { *solver_objective = obj; } bool is_feasible = false; bool attempt_repair = false; bool success = false; @@ -689,18 +687,10 @@ void branch_and_bound_t::set_solution_from_submip( log_prefix, compute_user_objective(lp, obj)); - // `obj` is in the sub-MIP's own space (fixed variables, own presolve offset), so it cannot be - // handed to solution_callback alongside a user-space assignment. - f_t original_lp_objective = std::numeric_limits::quiet_NaN(); - bool success = - set_solution_from_heuristics(user_sol, heuristics_origin_t::SUBMIP, &original_lp_objective); + bool success = set_solution_from_heuristics(user_sol, heuristics_origin_t::SUBMIP); if (success) { submip_stats.save_success(fixrate); - cuopt_assert(std::isfinite(original_lp_objective), - "SubMIP incumbent objective must be finite when accepted"); - if (settings_.solution_callback != nullptr) { - settings_.solution_callback(user_sol, original_lp_objective); - } + if (settings_.solution_callback != nullptr) { settings_.solution_callback(user_sol, obj); } } } diff --git a/cpp/src/branch_and_bound/branch_and_bound.hpp b/cpp/src/branch_and_bound/branch_and_bound.hpp index 5b95195bb5..7cf5ed3680 100644 --- a/cpp/src/branch_and_bound/branch_and_bound.hpp +++ b/cpp/src/branch_and_bound/branch_and_bound.hpp @@ -132,12 +132,8 @@ class branch_and_bound_t { } } - // Set a solution based on the user problem during the course of the solve. - // When non-null, `solver_objective` receives the objective of `solution` in original_lp_ space, - // which is the space every solution_callback consumer expects. - bool set_solution_from_heuristics(const std::vector& solution, - heuristics_origin_t origin, - f_t* solver_objective = nullptr); + // Set a solution based on the user problem during the course of the solve + bool set_solution_from_heuristics(const std::vector& solution, heuristics_origin_t origin); // Apply a solution found by a CPU FJ worker. void set_solution_from_cpu_fj(f_t obj, const std::vector& assignment, double work_units); From 67ccc3522e95f10e1881327c801ece5e54fd5c8f Mon Sep 17 00:00:00 2001 From: yboucher Date: Tue, 25 Aug 2026 05:19:34 -0700 Subject: [PATCH 60/61] run cpufj portfolio before root LP solve --- cpp/src/branch_and_bound/branch_and_bound.cpp | 50 +++++++- .../mip_heuristics/feasibility_jump/fj_cpu.cu | 108 ++++++++++++++++-- .../feasibility_jump/fj_cpu_worker.cuh | 16 ++- cpp/src/mip_heuristics/mip_constants.hpp | 4 + cpp/src/mip_heuristics/root_heuristics.hpp | 61 +++++++++- 5 files changed, 221 insertions(+), 18 deletions(-) diff --git a/cpp/src/branch_and_bound/branch_and_bound.cpp b/cpp/src/branch_and_bound/branch_and_bound.cpp index 29174148b7..18b45afc19 100644 --- a/cpp/src/branch_and_bound/branch_and_bound.cpp +++ b/cpp/src/branch_and_bound/branch_and_bound.cpp @@ -2430,6 +2430,7 @@ void branch_and_bound_t::solve_submip(diving_worker_t* worke f_t work_limit = 1.0; submip_fj_cpu_worker.create_worker(submip_bnb.original_lp_, submip_bnb.var_types_, + submip_bnb.original_problem_.num_cols, initial_guess, submip_bnb.settings_, std::format("{} [CPU FJ]", log_prefix), @@ -2891,6 +2892,7 @@ void branch_and_bound_t::recursive_submip(diving_worker_t* w f_t work_limit = 1.0; submip_fj_cpu_worker.create_worker(worker->leaf_problem, var_types, + original_problem_.num_cols, worker->leaf_solution.x, settings_, std::format("{} [CPU FJ]", log_prefix), @@ -2963,12 +2965,31 @@ void branch_and_bound_t::launch_root_heuristics( f_t work_limit = std::numeric_limits::infinity(); f_t time_limit = settings_.time_limit - toc(exploration_stats_.start_time); + // Odd passes start from the incumbent, even ones from the relaxation. The size guard covers a + // concurrent pass having grown the LP past the crush the incumbent was last taken through. + std::vector fj_seed; + if (cut_pass % 2 == 1) { + mutex_upper_.lock(); + if (incumbent_.has_incumbent && incumbent_.x.size() == (size_t)lp.num_cols) { + fj_seed = incumbent_.x; + } + mutex_upper_.unlock(); + } + if (fj_seed.empty()) { fj_seed = sol; } + current_heuristic->fj_cpu_worker_.improvement_callback = [this](f_t obj, const std::vector& assignment, double work_units) { set_solution_from_cpu_fj(obj, assignment, work_units); }; - current_heuristic->fj_cpu_worker_.create_worker( - lp, var_types_, sol, settings_, "[RootCut CPUFJ] "); + current_heuristic->fj_cpu_worker_.create_worker(lp, + var_types_, + original_problem_.num_cols, + fj_seed, + settings_, + "[RootCut CPUFJ " + std::to_string(cut_pass) + + "] ", + /*seed=*/-1, + /*lane=*/cut_pass); ++(*worker_count); #pragma omp task priority(CUOPT_DEFAULT_TASK_PRIORITY) \ @@ -3525,6 +3546,29 @@ mip_status_t branch_and_bound_t::solve(mip_solution_t& solut lp_status_t root_status = lp_status_t::UNSET; solving_root_relaxation_ = true; + // Started here so the lanes run through the root LP and every cut pass. No relaxation exists + // yet, so they seed from the anchor. + root_heuristics_t root_heuristics(settings_.num_threads - 1); + const i_t n_root_fj_lanes = + std::clamp(settings_.num_threads / 4, 0, CUOPT_MIP_ROOT_CPUFJ_MAX_LANES); + const f_t root_fj_time_limit = settings_.time_limit - toc(exploration_stats_.start_time); + if (!settings_.deterministic && n_root_fj_lanes > 0 && root_fj_time_limit > 0) { + root_heuristics.start_persistent_lanes( + original_lp_, + var_types_, + original_problem_.num_cols, + {}, + settings_, + n_root_fj_lanes, + root_fj_time_limit, + (int64_t)settings_.random_seed, + [this](f_t obj, const std::vector& assignment, double work_units) { + cuopt_assert(assignment.size() == (size_t)original_problem_.num_cols, + "root CPU FJ lanes must report a slack-free assignment"); + set_solution_from_cpu_fj(obj, assignment, work_units); + }); + } + f_t root_relax_start_time = tic(); if (!enable_concurrent_lp_root_solve()) { @@ -3682,8 +3726,6 @@ mip_status_t branch_and_bound_t::solve(mip_solution_t& solut compute_user_objective(original_lp_, root_relax_objective); } - root_heuristics_t root_heuristics(settings_.num_threads - 1); - f_t cut_generation_start_time = tic(); i_t cut_pool_size = 0; for (i_t cut_pass = 0; cut_pass < settings_.max_cut_passes; cut_pass++) { diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index 5e16a8c4b3..c375b16441 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -2694,10 +2694,72 @@ static void finalize_fj_cpu_host_initialization_from_template( raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); } +// Slacks at and above n_structural fold into their row's bounds: a*x + alpha*s = rhs with +// s in [lo, hi] becomes rhs - max(alpha*lo, alpha*hi) <= a*x <= rhs - min(alpha*lo, alpha*hi). +template +static void eliminate_slacks(const lp_problem_t& problem, + i_t n_structural, + csr_matrix_t& csr_A, + std::vector& row_lower, + std::vector& row_upper) +{ + cuopt_assert(csr_A.m == problem.num_rows, "row count mismatch"); + cuopt_assert(csr_A.n == problem.num_cols, "column count mismatch"); + cuopt_assert(n_structural > 0, "no structural columns"); + cuopt_assert(n_structural < problem.num_cols, "no slacks to eliminate"); + cuopt_assert(problem.num_cols - n_structural <= problem.num_rows, "more slacks than rows"); + + row_lower = problem.rhs; + row_upper = problem.rhs; + + std::vector row_has_slack(problem.num_rows, 0); + for (i_t j = n_structural; j < problem.num_cols; ++j) { + cuopt_assert(problem.A.col_length(j) == 1, "slack column is not a singleton"); + + const i_t entry = problem.A.col_start[j]; + const i_t row = problem.A.i[entry]; + const f_t alpha = problem.A.x[entry]; + cuopt_assert(std::abs(alpha) == f_t{1}, "slack coefficient is not +/-1"); + cuopt_assert(!row_has_slack[row], "row has more than one slack"); + row_has_slack[row] = 1; + + const f_t scaled_lower = alpha * problem.lower[j]; + const f_t scaled_upper = alpha * problem.upper[j]; + row_lower[row] = problem.rhs[row] - std::max(scaled_lower, scaled_upper); + row_upper[row] = problem.rhs[row] - std::min(scaled_lower, scaled_upper); + cuopt_assert(std::isfinite(row_lower[row]) || std::isfinite(row_upper[row]), + "eliminated row is free on both sides"); + cuopt_assert(row_lower[row] <= row_upper[row], "eliminated row has crossed bounds"); + } + + i_t out = 0; + for (i_t row = 0; row < csr_A.m; ++row) { + const i_t row_start = csr_A.row_start[row]; + const i_t row_end = csr_A.row_start[row + 1]; + csr_A.row_start[row] = out; + for (i_t p = row_start; p < row_end; ++p) { + if (csr_A.j[p] >= n_structural) { continue; } + csr_A.j[out] = csr_A.j[p]; + csr_A.x[out] = csr_A.x[p]; + ++out; + } + } + cuopt_assert( + out == csr_A.row_start[csr_A.m] - static_cast(problem.num_cols - n_structural), + "slack elimination removed the wrong number of entries"); + + csr_A.row_start[csr_A.m] = out; + csr_A.j.resize(out); + csr_A.x.resize(out); + csr_A.nz_max = out; + csr_A.n = n_structural; +} + template static std::unique_ptr> init_fj_cpu_from_host_lp( const lp_problem_t& problem, const std::vector& variable_types, + i_t n_structural, const std::vector& seed_assignment, const simplex_solver_settings_t& settings, std::atomic& preemption_flag, @@ -2715,16 +2777,27 @@ static std::unique_ptr> init_fj_cpu_from_host_lp( tolerances.absolute_mip_gap = settings.absolute_mip_gap_tol; tolerances.relative_mip_gap = settings.relative_mip_gap_tol; - const i_t n_variables = problem.num_cols; const i_t n_constraints = problem.num_rows; csr_matrix_t csr_A(problem.num_rows, problem.num_cols, problem.A.nnz()); problem.A.to_compressed_row(csr_A); - std::vector coefficients = csr_A.x; - std::vector variables = csr_A.j; - std::vector offsets = csr_A.row_start; - std::vector constraint_lower_bounds = problem.rhs; - std::vector constraint_upper_bounds = problem.rhs; + + std::vector constraint_lower_bounds; + std::vector constraint_upper_bounds; + i_t n_variables; + if (n_structural > 0 && n_structural < problem.num_cols) { + eliminate_slacks(problem, n_structural, csr_A, constraint_lower_bounds, constraint_upper_bounds); + n_variables = n_structural; + } else { + n_variables = problem.num_cols; + // Standard form: every row is an equality. + constraint_lower_bounds = problem.rhs; + constraint_upper_bounds = problem.rhs; + } + + std::vector coefficients = csr_A.x; + std::vector variables = csr_A.j; + std::vector offsets = csr_A.row_start; std::vector variable_bounds(n_variables); std::vector cpufj_variable_types(n_variables); std::vector is_binary_variable(n_variables, 0); @@ -2781,8 +2854,9 @@ static std::unique_ptr> init_fj_cpu_from_host_lp( fj_cpu->h_coefficients = std::move(coefficients); fj_cpu->h_offsets = std::move(offsets); fj_cpu->h_variables = std::move(variables); - fj_cpu->h_obj_coeffs = problem.objective; - fj_cpu->h_var_bounds = std::move(variable_bounds); + fj_cpu->h_obj_coeffs = + std::vector(problem.objective.begin(), problem.objective.begin() + n_variables); + fj_cpu->h_var_bounds = std::move(variable_bounds); fj_cpu->h_cstr_lb = std::move(constraint_lower_bounds); fj_cpu->h_cstr_ub = std::move(constraint_upper_bounds); fj_cpu->h_var_types = std::move(cpufj_variable_types); @@ -3784,23 +3858,33 @@ void fj_cpu_worker_t::fj_cpu_deleter_t::operator()(fj_cpu_climber_t +std::shared_ptr> make_fj_cpu_shared_incumbent() +{ + return std::make_shared>(); +} + template void fj_cpu_worker_t::create_worker( const lp_problem_t& problem, const std::vector& variable_types, + i_t n_structural, const std::vector& seed_assignment, const simplex_solver_settings_t& settings, std::string log_prefix, - int64_t seed) + int64_t seed, + int lane) { auto new_climber = init_fj_cpu_from_host_lp( - problem, variable_types, seed_assignment, settings, preemption_flag, seed); + problem, variable_types, n_structural, seed_assignment, settings, preemption_flag, seed); fj_cpu.reset(new_climber.release()); fj_cpu->log_prefix = std::move(log_prefix); fj_cpu->improvement_callback = improvement_callback; + fj_cpu->shared_incumbent = shared_incumbent; fj_cpu->halted = false; preemption_flag = false; is_initialized = true; + if (lane >= 0) { apply_lane_diversification(*fj_cpu, lane, fj_cpu->settings.seed); } } template @@ -3847,6 +3931,8 @@ void fj_cpu_worker_t::send_stop_signal() #if MIP_INSTANTIATE_FLOAT template class fj_t; template struct fj_cpu_worker_t; +template std::shared_ptr> +make_fj_cpu_shared_incumbent(); template void cpufj_solve(fj_cpu_climber_t* fj_cpu, float in_time_limit, double work_unit_limit); @@ -3876,6 +3962,8 @@ template void finalize_fj_cpu_host_initialization( #if MIP_INSTANTIATE_DOUBLE template class fj_t; template struct fj_cpu_worker_t; +template std::shared_ptr> +make_fj_cpu_shared_incumbent(); template void cpufj_solve(fj_cpu_climber_t* fj_cpu, double in_time_limit, double work_unit_limit); diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh index bb2c69f81c..b30081059b 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh @@ -24,6 +24,13 @@ namespace cuopt::mathematical_optimization::mip { template struct fj_cpu_climber_t; +template +struct fj_cpu_shared_incumbent_t; + +// Defined in fj_cpu.cu, where the type is complete. +template +std::shared_ptr> make_fj_cpu_shared_incumbent(); + template struct fj_cpu_worker_t { // Custom deleter to avoid pulling the entire fj_cpu_climber_t class here. @@ -35,19 +42,26 @@ struct fj_cpu_worker_t { std::atomic preemption_flag{false}; std::unique_ptr, fj_cpu_deleter_t> fj_cpu; std::function&, double)> improvement_callback; + // Set before create_worker to join a portfolio; left null when the climber runs alone. + std::shared_ptr> shared_incumbent; ~fj_cpu_worker_t() { stop(); } + // `n_structural` is where `problem`'s slack block starts; those columns fold into two-sided row + // bounds, so the climber and the assignment it reports span only the ones below. -1 keeps them. // `seed` selects the FJ RNG seed: pass a non-negative value for a deterministic seed, // or -1 to draw from the global cuopt::seed_generator (the historical behavior). // In deterministic mode the caller MUST pass an explicit seed, otherwise the underlying // seed_generator::get_seed() racing with concurrent callers breaks reproducibility. + // `lane` >= 0 applies that lane's persona from the portfolio diversification ladder. void create_worker(const simplex::lp_problem_t& problem, const std::vector& variable_types, + i_t n_structural, const std::vector& seed_assignment, const simplex::simplex_solver_settings_t& settings, std::string log_prefix, - int64_t seed = -1); + int64_t seed = -1, + int lane = -1); // Run the worker asynchronously (i.e., launch an openmp task and then continue the // execution). Call `stop()` for stopping the worker diff --git a/cpp/src/mip_heuristics/mip_constants.hpp b/cpp/src/mip_heuristics/mip_constants.hpp index d09fa710c9..ab2451f817 100644 --- a/cpp/src/mip_heuristics/mip_constants.hpp +++ b/cpp/src/mip_heuristics/mip_constants.hpp @@ -25,6 +25,10 @@ * the problem and occupies an OMP task for the whole of presolve. */ #define CUOPT_MIP_EARLY_CPUFJ_MAX_CLIMBERS 8 +/* @brief Upper bound on the persistent root CPUFJ lane set. Every lane holds its own host copy of + * the root LP and occupies an OMP task for the whole of the cut loop. */ +#define CUOPT_MIP_ROOT_CPUFJ_MAX_LANES 4 + // MIP-only gate: skip the concurrent barrier when fewer threads are available than this // (1 PDLP + 1 dual simplex + 1 barrier). Stand-alone LP always runs all three. #define CUOPT_CONCURRENT_LP_BARRIER_REQUIRED_THREAD_COUNT 3 diff --git a/cpp/src/mip_heuristics/root_heuristics.hpp b/cpp/src/mip_heuristics/root_heuristics.hpp index 1f29b25eef..17a7a73e51 100644 --- a/cpp/src/mip_heuristics/root_heuristics.hpp +++ b/cpp/src/mip_heuristics/root_heuristics.hpp @@ -9,8 +9,15 @@ #include #include +#include #include "feasibility_jump/fj_cpu_worker.cuh" +#include +#include +#include +#include +#include + namespace cuopt::mathematical_optimization::mip { template @@ -91,19 +98,63 @@ struct root_heuristics_t { std::shared_ptr> worker_count_; i_t max_workers_; + // CPU FJ lanes that outlive a single cut pass. + std::vector>> persistent_lanes_; + // Shared by every CPU FJ lane of the root phase, persistent and per-cut-pass alike. + std::shared_ptr> shared_incumbent_; + root_heuristics_t(i_t max_workers) - : worker_count_(std::make_shared>(0)), max_workers_(max_workers) + : worker_count_(std::make_shared>(0)), + max_workers_(max_workers), + shared_incumbent_(make_fj_cpu_shared_incumbent()) { } ~root_heuristics_t() { stop_and_sync(); } + // Must be called from the same task region as stop_and_sync: run_async's task dependence is + // matched only by a taskwait in the encountering region. + void start_persistent_lanes(const simplex::lp_problem_t& lp, + const std::vector& var_types, + i_t n_structural, + const std::vector& seed_assignment, + const simplex::simplex_solver_settings_t& settings, + i_t n_lanes, + f_t time_limit, + int64_t base_seed, + std::function&, double)> callback) + { + persistent_lanes_.reserve(n_lanes); + for (i_t k = 0; k < n_lanes; ++k) { + auto lane = std::make_unique>(); + lane->improvement_callback = callback; + lane->shared_incumbent = shared_incumbent_; + lane->create_worker(lp, + var_types, + n_structural, + seed_assignment, + settings, + "[Root FJ lane " + std::to_string(k) + "] ", + base_seed + k, + k); + lane->run_async(time_limit); + persistent_lanes_.push_back(std::move(lane)); + } + } + void stop_and_sync() { + for (auto& lane : persistent_lanes_) { + lane->send_stop_signal(); + } for (auto& heuristic : cut_passes_heuristics_) { heuristic->send_stop_signal(); } + for (auto& lane : persistent_lanes_) { + lane->stop(); + } + persistent_lanes_.clear(); for (auto& heuristic : cut_passes_heuristics_) { heuristic->stop_and_sync(); } @@ -127,8 +178,12 @@ struct root_heuristics_t { cut_passes_heuristics_.erase(cut_passes_heuristics_.begin()); } - return cut_passes_heuristics_.emplace_back(std::make_shared>( - Arow, var_types, root_solution, root_edge_norm)); + auto& heuristic = cut_passes_heuristics_.emplace_back( + std::make_shared>( + Arow, var_types, root_solution, root_edge_norm)); + // Read by create_worker, so it has to be in place before the caller builds the climber. + heuristic->fj_cpu_worker_.shared_incumbent = shared_incumbent_; + return heuristic; } }; From 7eda27d71fae6ab01724a392573ee4430e487e51 Mon Sep 17 00:00:00 2001 From: yboucher Date: Tue, 25 Aug 2026 05:31:55 -0700 Subject: [PATCH 61/61] many lanes during early heuristics --- .../mip_heuristics/diversity/diversity_manager.cu | 11 +++++++++++ .../mip_heuristics/feasibility_jump/early_cpufj.cu | 4 ++-- .../mip_heuristics/feasibility_jump/early_cpufj.cuh | 6 +++++- cpp/src/mip_heuristics/mip_constants.hpp | 6 +++--- cpp/src/mip_heuristics/solve.cu | 12 ++++++++---- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cpp/src/mip_heuristics/diversity/diversity_manager.cu b/cpp/src/mip_heuristics/diversity/diversity_manager.cu index ec82c4b423..e69022c42d 100644 --- a/cpp/src/mip_heuristics/diversity/diversity_manager.cu +++ b/cpp/src/mip_heuristics/diversity/diversity_manager.cu @@ -8,6 +8,7 @@ #include "cuda_profiler_api.h" #include "diversity_manager.cuh" +#include #include #include @@ -22,6 +23,9 @@ #include #include +#include + +#include #include #include #include @@ -314,6 +318,13 @@ bool diversity_manager_t::run_presolve(f_t time_limit, timer_t global_ if (run_probing_cache && !global_timer.check_time_limit() && !presolve_timer.check_time_limit()) { log_presolve_budget("PROBING", probing_features, probing_budget); + // The early CPUFJ lanes hold their threads for the whole of presolve, and probing's default + // task count assumes the whole team. Its pools are sized per task, so this bounds host memory + // as well as concurrency. + const i_t held_by_cpufj = + context.early_cpufj_ptr != nullptr ? (i_t)context.early_cpufj_ptr->lane_count() : 0; + ls.constraint_prop.bounds_update.settings.num_tasks = + std::max(1, omp_get_num_threads() - 1 - held_by_cpufj); f_t time_for_probing_cache = std::min(time_limit, (f_t)global_timer.remaining_time()); timer_t probing_timer{time_for_probing_cache}; [[maybe_unused]] const auto probing_t0 = std::chrono::steady_clock::now(); diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu index 8b1444d8a1..5f9a68ac99 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu @@ -33,7 +33,7 @@ early_cpufj_t::~early_cpufj_t() } template -void early_cpufj_t::start() +void early_cpufj_t::start(int n_lanes) { // 1: presolve, 1: early GPU FJ, 1: early CPU FJ if (!climbers_.empty() || omp_get_num_threads() < CUOPT_MIP_EARLY_CPUFJ_REQUIRED_THREAD_COUNT) { @@ -45,7 +45,7 @@ void early_cpufj_t::start() // Tasks are not preempted, so a lane posted beyond the team size would sit in the queue for the // whole of presolve without running an iteration. - const int n_lanes = std::min(CUOPT_MIP_EARLY_CPUFJ_MAX_CLIMBERS, omp_get_num_threads()); + n_lanes = std::clamp(n_lanes, 1, omp_get_num_threads()); const int64_t base_seed = cuopt::seed_generator::get_seed(); climbers_.resize(n_lanes); diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh index ede2ffe168..3bae5ed63b 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh @@ -28,9 +28,13 @@ class early_cpufj_t : public early_heuristic_t static constexpr const char* name() { return "CPUFJ"; } - void start(); + // Lanes are OMP tasks that never yield, so n_lanes threads are unavailable to anything else + // until stop(). Callers sharing the team with other work size it accordingly. + void start(int n_lanes); void stop(); + int lane_count() const { return (int)climbers_.size(); } + private: friend class early_heuristic_t>; diff --git a/cpp/src/mip_heuristics/mip_constants.hpp b/cpp/src/mip_heuristics/mip_constants.hpp index ab2451f817..e5c85a65fa 100644 --- a/cpp/src/mip_heuristics/mip_constants.hpp +++ b/cpp/src/mip_heuristics/mip_constants.hpp @@ -21,9 +21,9 @@ #define CUOPT_MIP_BATCH_PDLP_REQUIRED_THREAD_COUNT 3 #define CUOPT_MIP_CLIQUE_CUTS_REQUIRED_THREAD_COUNT 3 -/* @brief Upper bound on the early CPUFJ climber portfolio. Every lane holds its own host copy of - * the problem and occupies an OMP task for the whole of presolve. */ -#define CUOPT_MIP_EARLY_CPUFJ_MAX_CLIMBERS 8 +/* @brief Threads the early CPUFJ portfolio leaves to the rest of the team. Every lane holds its + * own host copy of the problem and occupies an OMP task for the whole of presolve. */ +#define CUOPT_MIP_EARLY_CPUFJ_RESERVED_THREADS 4 /* @brief Upper bound on the persistent root CPUFJ lane set. Every lane holds its own host copy of * the root LP and occupies an OMP task for the whole of the cut loop. */ diff --git a/cpp/src/mip_heuristics/solve.cu b/cpp/src/mip_heuristics/solve.cu index 5371a87026..c42bac4365 100644 --- a/cpp/src/mip_heuristics/solve.cu +++ b/cpp/src/mip_heuristics/solve.cu @@ -296,9 +296,11 @@ mip_solution_t run_mip_solver( if (std::isfinite(initial_upper_bound)) { early_cpufj->set_best_objective(problem.get_solver_obj_from_user_obj(initial_upper_bound)); } - early_cpufj->start(); + early_cpufj->start(omp_get_num_threads() - CUOPT_MIP_EARLY_CPUFJ_RESERVED_THREADS); solver.context.early_cpufj_ptr = early_cpufj.get(); - CUOPT_LOG_DEBUG("Started early CPUFJ on papilo-presolved problem during cuOpt presolve"); + CUOPT_LOG_DEBUG( + "Started early CPUFJ on papilo-presolved problem during cuOpt presolve with %d lanes", + early_cpufj->lane_count()); } auto presolved_sol = solver.run_solver(); @@ -512,8 +514,10 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p // Start early CPUFJ on original problem (will restart on presolved problem after Papilo) early_cpufj = std::make_unique>( op_problem, settings.get_tolerances(), early_fj_callback); - early_cpufj->start(); - CUOPT_LOG_DEBUG("Started early CPUFJ on original problem"); + // Papilo runs on its own threads, so the team is otherwise idle here. + early_cpufj->start(omp_get_num_threads() - CUOPT_MIP_EARLY_CPUFJ_RESERVED_THREADS); + CUOPT_LOG_DEBUG("Started early CPUFJ on original problem with %d lanes", + early_cpufj->lane_count()); } auto early_cpufj_guard = cuopt::scope_guard([&]() {