From dae0ec221959e18f699dacc59ff36dd15dddaf3a Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Fri, 23 Feb 2024 10:20:28 +0100 Subject: [PATCH] math_opt: export from google3 --- cmake/python.cmake | 1 + ortools/math_opt/cpp/BUILD.bazel | 13 ++++ ortools/math_opt/cpp/parameters.cc | 2 +- ortools/math_opt/cpp/solver_resources.cc | 66 +++++++++++++++++ ortools/math_opt/cpp/solver_resources.h | 73 +++++++++++++++++++ ortools/math_opt/python/BUILD.bazel | 7 ++ ortools/math_opt/python/mathopt.py | 1 + ortools/math_opt/python/mathopt_test.py | 2 + ortools/math_opt/python/solver_resources.py | 66 +++++++++++++++++ .../math_opt/python/solver_resources_test.py | 38 ++++++++++ ortools/math_opt/rpc.proto | 6 +- ortools/math_opt/samples/cpp/tsp.cc | 2 +- .../math_opt/solver_tests/callback_tests.cc | 5 +- .../solver_tests/ip_parameter_tests.cc | 5 +- ortools/math_opt/solver_tests/status_tests.cc | 5 +- ortools/math_opt/storage/objective_storage.cc | 2 +- ortools/math_opt/storage/objective_storage.h | 2 +- 17 files changed, 280 insertions(+), 16 deletions(-) create mode 100644 ortools/math_opt/cpp/solver_resources.cc create mode 100644 ortools/math_opt/cpp/solver_resources.h create mode 100644 ortools/math_opt/python/solver_resources.py create mode 100644 ortools/math_opt/python/solver_resources_test.py diff --git a/cmake/python.cmake b/cmake/python.cmake index 287f06c81b..2099134b44 100644 --- a/cmake/python.cmake +++ b/cmake/python.cmake @@ -370,6 +370,7 @@ if(BUILD_MATH_OPT) ortools/math_opt/python/result.py ortools/math_opt/python/solution.py ortools/math_opt/python/solve.py + ortools/math_opt/python/solver_resources.py ortools/math_opt/python/sparse_containers.py ortools/math_opt/python/statistics.py DESTINATION ${PYTHON_PROJECT_DIR}/math_opt/python) diff --git a/ortools/math_opt/cpp/BUILD.bazel b/ortools/math_opt/cpp/BUILD.bazel index 9d508407e2..5399a3df62 100644 --- a/ortools/math_opt/cpp/BUILD.bazel +++ b/ortools/math_opt/cpp/BUILD.bazel @@ -508,3 +508,16 @@ cc_library( "//ortools/math_opt/core:solve_interrupter", ], ) + +cc_library( + name = "solver_resources", + srcs = ["solver_resources.cc"], + hdrs = ["solver_resources.h"], + deps = [ + "//ortools/math_opt:rpc_cc_proto", + "//ortools/port:proto_utils", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/ortools/math_opt/cpp/parameters.cc b/ortools/math_opt/cpp/parameters.cc index 760693dc5b..562367beae 100644 --- a/ortools/math_opt/cpp/parameters.cc +++ b/ortools/math_opt/cpp/parameters.cc @@ -345,7 +345,7 @@ bool AbslParseFlag(absl::string_view text, SolveParameters* solve_parameters, } std::string AbslUnparseFlag(SolveParameters solve_parameters) { - return ProtobufTextFormatPrintToString(solve_parameters.Proto()); + return ProtobufTextFormatPrintToStringForFlag(solve_parameters.Proto()); } } // namespace math_opt diff --git a/ortools/math_opt/cpp/solver_resources.cc b/ortools/math_opt/cpp/solver_resources.cc new file mode 100644 index 0000000000..eb261e2d6f --- /dev/null +++ b/ortools/math_opt/cpp/solver_resources.cc @@ -0,0 +1,66 @@ +// Copyright 2010-2024 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ortools/math_opt/cpp/solver_resources.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "ortools/port/proto_utils.h" + +namespace operations_research::math_opt { + +SolverResourcesProto SolverResources::Proto() const { + SolverResourcesProto ret; + if (cpu.has_value()) { + ret.set_cpu(cpu.value()); + } + return ret; +} + +absl::StatusOr SolverResources::FromProto( + const SolverResourcesProto& proto) { + SolverResources ret; + if (proto.has_cpu()) { + ret.cpu = proto.cpu(); + } + return ret; +} + +bool AbslParseFlag(const absl::string_view text, + SolverResources* const solver_resources, + std::string* const error) { + SolverResourcesProto proto; + if (!ProtobufParseTextProtoForFlag(text, &proto, error)) { + return false; + } + absl::StatusOr resources = SolverResources::FromProto(proto); + if (!resources.ok()) { + *error = absl::StrCat( + "SolverResourcesProto was invalid and could not convert to " + "SolverResources: ", + resources.status().ToString()); + return false; + } + *solver_resources = *std::move(resources); + return true; +} + +std::string AbslUnparseFlag(const SolverResources& solver_resources) { + return ProtobufTextFormatPrintToStringForFlag(solver_resources.Proto()); +} + +} // namespace operations_research::math_opt diff --git a/ortools/math_opt/cpp/solver_resources.h b/ortools/math_opt/cpp/solver_resources.h new file mode 100644 index 0000000000..9eb23e6248 --- /dev/null +++ b/ortools/math_opt/cpp/solver_resources.h @@ -0,0 +1,73 @@ +// Copyright 2010-2024 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_MATH_OPT_CPP_SOLVER_RESOURCES_H_ +#define OR_TOOLS_MATH_OPT_CPP_SOLVER_RESOURCES_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "ortools/math_opt/rpc.pb.h" + +namespace operations_research::math_opt { + +// The hints on the resources a remote solve is expected to use. These +// parameters are hints and may be ignored by the remote server (in particular +// in case of solve in a local subprocess, for example). +// +// When using RemoteSolve() and RemoteComputeInfeasibleSubsystem(), these hints +// are mostly optional as some defaults will be computed based on the other +// parameters. +// +// When using RemoteStreamingSolve() these hints are used to dimension the +// resources available during the execution of every action; thus it is +// recommended to set them. +// +struct SolverResources { + // The number of solver threads that are expected to actually execute in + // parallel. Must be finite and >0.0. + // + // For example a value of 3.0 means that if the solver has 5 threads that can + // execute we expect at least 3 of these threads to be scheduled in parallel + // for any given time slice of the operating system scheduler. + // + // A fractional value indicates that we don't expect the operating system to + // constantly schedule the solver's work. For example with 0.5 we would expect + // the solver's threads to be scheduled half the time. + // + // This parameter is usually used in conjunction with + // SolveParameters.threads. For some solvers like Gurobi it makes sense to use + // SolverResources.cpu = SolveParameters.threads. For other solvers like + // CP-SAT, it may makes sense to use a value lower than the number of threads + // as not all threads may be ready to be scheduled at the same time. It is + // better to consult each solver documentation to set this parameter. + // + // Note that if the SolveParameters.threads is not set then this parameter + // should also be left unset. + std::optional cpu; + + SolverResourcesProto Proto() const; + static absl::StatusOr FromProto( + const SolverResourcesProto& proto); +}; + +bool AbslParseFlag(absl::string_view text, SolverResources* solver_resources, + std::string* error); + +std::string AbslUnparseFlag(const SolverResources& solver_resources); + +} // namespace operations_research::math_opt + +#endif // OR_TOOLS_MATH_OPT_CPP_SOLVER_RESOURCES_H_ diff --git a/ortools/math_opt/python/BUILD.bazel b/ortools/math_opt/python/BUILD.bazel index 7272a3656c..2219ec5456 100644 --- a/ortools/math_opt/python/BUILD.bazel +++ b/ortools/math_opt/python/BUILD.bazel @@ -35,6 +35,7 @@ py_library( ":result", ":solution", ":solve", + ":solver_resources", ":sparse_containers", ], ) @@ -194,3 +195,9 @@ py_library( ":model", ], ) + +py_library( + name = "solver_resources", + srcs = ["solver_resources.py"], + deps = ["//ortools/math_opt:rpc_py_pb2"], +) diff --git a/ortools/math_opt/python/mathopt.py b/ortools/math_opt/python/mathopt.py index 19830fd6ca..0fb5a1fbe6 100644 --- a/ortools/math_opt/python/mathopt.py +++ b/ortools/math_opt/python/mathopt.py @@ -158,6 +158,7 @@ from ortools.math_opt.python.solve import compute_infeasible_subsystem from ortools.math_opt.python.solve import IncrementalSolver from ortools.math_opt.python.solve import solve from ortools.math_opt.python.solve import SolveCallback +from ortools.math_opt.python.solver_resources import SolverResources from ortools.math_opt.python.sparse_containers import LinearConstraintFilter from ortools.math_opt.python.sparse_containers import parse_linear_constraint_map from ortools.math_opt.python.sparse_containers import parse_variable_map diff --git a/ortools/math_opt/python/mathopt_test.py b/ortools/math_opt/python/mathopt_test.py index d35e28d351..0858500434 100644 --- a/ortools/math_opt/python/mathopt_test.py +++ b/ortools/math_opt/python/mathopt_test.py @@ -31,6 +31,7 @@ from ortools.math_opt.python import parameters from ortools.math_opt.python import result from ortools.math_opt.python import solution from ortools.math_opt.python import solve +from ortools.math_opt.python import solver_resources from ortools.math_opt.python import sparse_containers # This list does not contain some modules intentionally: @@ -56,6 +57,7 @@ _MODULES_TO_CHECK: List[types.ModuleType] = [ sparse_containers, solution, solve, + solver_resources, ] # Some symbols are not meant to be exported; we exclude them here. diff --git a/ortools/math_opt/python/solver_resources.py b/ortools/math_opt/python/solver_resources.py new file mode 100644 index 0000000000..bc4e4aa516 --- /dev/null +++ b/ortools/math_opt/python/solver_resources.py @@ -0,0 +1,66 @@ +# Copyright 2010-2024 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configures solver resources.""" + +import dataclasses +from typing import Optional + +from ortools.math_opt import rpc_pb2 + + +@dataclasses.dataclass +class SolverResources: + """The hints on the resources a remote solve is expected to use. + + These parameters are hints and may be ignored by the remote server (in + particular in case of solve in a local subprocess, for example). + + When using remote_solve() and remote_compute_infeasible_subsystem(), these + hints are mostly optional as some defaults will be computed based on the other + parameters. + + When using remote_streaming_solve() these hints are used to dimension the + resources available during the execution of every action; thus it is + recommended to set them. + + MOE:begin_intracomment_strip + + The go/uoss server will use these parameters to do a bin-packing of all + requests. They are generally used as soft-limits though instead of + hard-limits and a solve may be able to consume more resources than requested. + + MOE:end_intracomment_strip + + Attributes: + cpu: The number of solver threads that are expected to actually execute in + parallel. Must be finite and >0.0. For example a value of 3.0 means that + if the solver has 5 threads that can execute we expect at least 3 of these + threads to be scheduled in parallel for any given time slice of the + operating system scheduler. A fractional value indicates that we don't + expect the operating system to constantly schedule the solver's work. For + example with 0.5 we would expect the solver's threads to be scheduled half + the time. This parameter is usually used in conjunction with + SolveParameters.threads. For some solvers like Gurobi it makes sense to + use SolverResources.cpu = SolveParameters.threads. For other solvers like + CP-SAT, it may makes sense to use a value lower than the number of threads + as not all threads may be ready to be scheduled at the same time. It is + better to consult each solver documentation to set this parameter. Note + that if the SolveParameters.threads is not set then this parameter should + also be left unset. + """ + + cpu: Optional[float] = None + + def to_proto(self) -> rpc_pb2.SolverResourcesProto: + return rpc_pb2.SolverResourcesProto(cpu=self.cpu) diff --git a/ortools/math_opt/python/solver_resources_test.py b/ortools/math_opt/python/solver_resources_test.py new file mode 100644 index 0000000000..455be29d25 --- /dev/null +++ b/ortools/math_opt/python/solver_resources_test.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# Copyright 2010-2024 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for solver_resources.""" + +from absl.testing import absltest +from ortools.math_opt import rpc_pb2 +from ortools.math_opt.python import solver_resources +from ortools.math_opt.python.testing import compare_proto + + +class SolverResourcesTest(compare_proto.MathOptProtoAssertions, absltest.TestCase): + def test_to_proto_empty(self): + self.assert_protos_equiv( + solver_resources.SolverResources().to_proto(), + rpc_pb2.SolverResourcesProto(), + ) + + def test_to_proto_with_cpu(self): + self.assert_protos_equiv( + solver_resources.SolverResources(cpu=3.5).to_proto(), + rpc_pb2.SolverResourcesProto(cpu=3.5), + ) + + +if __name__ == "__main__": + absltest.main() diff --git a/ortools/math_opt/rpc.proto b/ortools/math_opt/rpc.proto index 415efa93cf..42a750ae9d 100644 --- a/ortools/math_opt/rpc.proto +++ b/ortools/math_opt/rpc.proto @@ -28,7 +28,7 @@ option java_multiple_files = true; // This message is used to specify some hints on the resources a remote solve is // expected to use. These parameters are hints and may be ignored by the remote -// server (in particular in case of solve in a local subprocess for example). +// server (in particular in case of solve in a local subprocess, for example). // // When using SolveService.Solve and SolveService.ComputeInfeasibleSubsystem, // these hints are mostly optional as some defaults will be computed based on @@ -43,10 +43,10 @@ message SolverResourcesProto { // parallel. Must be finite and >0.0. // // For example a value of 3.0 means that if the solver has 5 threads that can - // execute we expect at least 3 of these threads to be schedule in parallel + // execute we expect at least 3 of these threads to be scheduled in parallel // for any given time slice of the operating system scheduler. // - // A fractional value indicate that we don't expect the operating system to + // A fractional value indicates that we don't expect the operating system to // constantly schedule the solver's work. For example with 0.5 we would expect // the solver's threads to be scheduled half the time. // diff --git a/ortools/math_opt/samples/cpp/tsp.cc b/ortools/math_opt/samples/cpp/tsp.cc index c83f9485c7..d53ae450cb 100644 --- a/ortools/math_opt/samples/cpp/tsp.cc +++ b/ortools/math_opt/samples/cpp/tsp.cc @@ -247,7 +247,7 @@ math_opt::BoundedLinearExpression CutsetConstraint( // Solves the TSP by returning the ordering of the cities that minimizes travel // distance. absl::StatusOr SolveTsp( - const std::vector>& cities, + absl::Span> cities, const math_opt::SolverType solver) { const int n = static_cast(cities.size()); const std::vector> distance_matrix = diff --git a/ortools/math_opt/solver_tests/callback_tests.cc b/ortools/math_opt/solver_tests/callback_tests.cc index 47dae6ec18..c0904dcf61 100644 --- a/ortools/math_opt/solver_tests/callback_tests.cc +++ b/ortools/math_opt/solver_tests/callback_tests.cc @@ -96,10 +96,9 @@ struct EnumFormatter { absl::StatusOr> LoadMiplibInstance( const absl::string_view name) { ASSIGN_OR_RETURN(const ModelProto model_proto, - ReadMpsFile(devtools_build::GetDataDependencyFilepath( - absl::StrCat("operations_research_data/" + ReadMpsFile(absl::StrCat("operations_research_data/" "MIP_MIPLIB/miplib2017/", - name, ".mps.gz")))); + name, ".mps.gz"))); return Model::FromModelProto(model_proto); } diff --git a/ortools/math_opt/solver_tests/ip_parameter_tests.cc b/ortools/math_opt/solver_tests/ip_parameter_tests.cc index 7e0b7b29ce..3ffd777a38 100644 --- a/ortools/math_opt/solver_tests/ip_parameter_tests.cc +++ b/ortools/math_opt/solver_tests/ip_parameter_tests.cc @@ -1025,10 +1025,9 @@ namespace { absl::StatusOr> LoadMiplibInstance( absl::string_view name) { ASSIGN_OR_RETURN(const ModelProto model_proto, - ReadMpsFile(devtools_build::GetDataDependencyFilepath( - absl::StrCat("operations_research_data/" + ReadMpsFile(absl::StrCat("operations_research_data/" "MIP_MIPLIB/miplib2017/", - name, ".mps.gz")))); + name, ".mps.gz"))); return Model::FromModelProto(model_proto); } diff --git a/ortools/math_opt/solver_tests/status_tests.cc b/ortools/math_opt/solver_tests/status_tests.cc index 6a56fdfc00..1a5ce6aefd 100644 --- a/ortools/math_opt/solver_tests/status_tests.cc +++ b/ortools/math_opt/solver_tests/status_tests.cc @@ -64,10 +64,9 @@ namespace { absl::StatusOr> LoadMiplibInstance( absl::string_view name) { ASSIGN_OR_RETURN(const ModelProto model_proto, - ReadMpsFile(devtools_build::GetDataDependencyFilepath( - absl::StrCat("operations_research_data/" + ReadMpsFile(absl::StrCat("operations_research_data/" "MIP_MIPLIB/miplib2017/", - name, ".mps.gz")))); + name, ".mps.gz"))); return Model::FromModelProto(model_proto); } diff --git a/ortools/math_opt/storage/objective_storage.cc b/ortools/math_opt/storage/objective_storage.cc index b035ff5e2d..fc3d8820d0 100644 --- a/ortools/math_opt/storage/objective_storage.cc +++ b/ortools/math_opt/storage/objective_storage.cc @@ -157,7 +157,7 @@ std::optional ObjectiveStorage::ObjectiveData::Update( std::pair ObjectiveStorage::Update( const Diff& diff, const absl::flat_hash_set& deleted_variables, - const std::vector& new_variables) const { + absl::Span new_variables) const { AuxiliaryObjectivesUpdatesProto auxiliary_result; for (const AuxiliaryObjectiveId id : diff.deleted) { diff --git a/ortools/math_opt/storage/objective_storage.h b/ortools/math_opt/storage/objective_storage.h index 0296dcffff..acaaba5968 100644 --- a/ortools/math_opt/storage/objective_storage.h +++ b/ortools/math_opt/storage/objective_storage.h @@ -192,7 +192,7 @@ class ObjectiveStorage { std::pair Update( const Diff& diff, const absl::flat_hash_set& deleted_variables, - const std::vector& new_variables) const; + absl::Span new_variables) const; // Updates the checkpoint and clears all stored changes in diff. void AdvanceCheckpointInDiff(VariableId variable_checkpoint,