From 2a4fc564b400e49dcde54cd73750952be1000cc3 Mon Sep 17 00:00:00 2001 From: Laurent Perron Date: Thu, 18 Apr 2019 19:18:48 +0200 Subject: [PATCH] add constraint with index in linear solver (API + Proto); rewrite MPS reader/writer; update base library as needed --- ortools/base/BUILD | 12 + ortools/base/canonical_errors.h | 2 + ortools/base/protobuf_util.h | 69 ++ ortools/base/status_macros.h | 65 ++ ortools/base/statusor.h | 8 + ortools/constraint_solver/routing.cc | 276 +++--- ortools/constraint_solver/routing.h | 165 ++-- ortools/constraint_solver/routing_search.cc | 44 +- ortools/glop/basis_representation.cc | 56 +- ortools/glop/basis_representation.h | 7 +- ortools/glop/lu_factorization.cc | 19 +- ortools/glop/lu_factorization.h | 4 +- ortools/glop/revised_simplex.cc | 6 +- ortools/glop/revised_simplex.h | 2 +- ortools/linear_solver/BUILD | 1 + ortools/linear_solver/csharp/linear_solver.i | 20 +- ortools/linear_solver/java/linear_solver.i | 34 +- ortools/linear_solver/linear_solver.cc | 23 +- ortools/linear_solver/model_exporter.cc | 397 ++++++--- ortools/linear_solver/model_exporter.h | 244 ++---- .../model_exporter_swig_helper.h | 39 + ortools/linear_solver/python/linear_solver.i | 24 +- ortools/lp_data/model_reader.cc | 50 +- ortools/lp_data/model_reader.h | 8 +- ortools/lp_data/mps_reader.cc | 503 ++--------- ortools/lp_data/mps_reader.h | 804 ++++++++++++++++-- ortools/lp_data/sparse.cc | 2 +- ortools/lp_data/sparse.h | 8 +- ortools/lp_data/sparse_column.h | 2 +- ortools/sat/lp_utils.cc | 24 + ortools/util/BUILD | 2 + ortools/util/file_util.cc | 8 + ortools/util/file_util.h | 4 + 33 files changed, 1774 insertions(+), 1158 deletions(-) create mode 100644 ortools/base/protobuf_util.h create mode 100644 ortools/base/status_macros.h create mode 100644 ortools/linear_solver/model_exporter_swig_helper.h diff --git a/ortools/base/BUILD b/ortools/base/BUILD index bbffaecea8..2679e03ff2 100644 --- a/ortools/base/BUILD +++ b/ortools/base/BUILD @@ -49,6 +49,18 @@ cc_library( ], ) +cc_library( + name = "status_macros", + hdrs = [ + "status_macros.h", + ], + deps = [ + ":base", + ":status", + ":statusor", + ], +) + cc_library( name = "int128", hdrs = [ diff --git a/ortools/base/canonical_errors.h b/ortools/base/canonical_errors.h index 92d65e0432..b9cb6695ab 100644 --- a/ortools/base/canonical_errors.h +++ b/ortools/base/canonical_errors.h @@ -14,6 +14,8 @@ #ifndef OR_TOOLS_BASE_CANONICAL_ERRORS_H_ #define OR_TOOLS_BASE_CANONICAL_ERRORS_H_ +#include "ortools/base/status.h" + namespace util { inline Status InvalidArgumentError(const std::string& message) { diff --git a/ortools/base/protobuf_util.h b/ortools/base/protobuf_util.h new file mode 100644 index 0000000000..4b9a54861a --- /dev/null +++ b/ortools/base/protobuf_util.h @@ -0,0 +1,69 @@ +// Copyright 2010-2018 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_BASE_PROTOBUF_UTIL_H_ +#define OR_TOOLS_BASE_PROTOBUF_UTIL_H_ + +#include "google/protobuf/repeated_field.h" +#include "ortools/base/logging.h" + +namespace google { +namespace protobuf { +namespace util { +// RepeatedPtrField version. +template +inline void Truncate(RepeatedPtrField* array, int new_size) { + const int size = array->size(); + DCHECK_GE(size, new_size); + array->DeleteSubrange(new_size, size - new_size); +} + +// Removes the elements at the indices specified by 'indices' from 'array' in +// time linear in the size of 'array' (on average, even when 'indices' is a +// singleton) while preserving the relative order of the remaining elements. +// The indices must be a container of ints in strictly increasing order, such +// as vector, set or initializer_list, and in the range [0, N - +// 1] where N is the number of elements in 'array', and RepeatedType must be +// RepeatedField or RepeatedPtrField. +// Returns number of elements erased. +template > +int RemoveAt(RepeatedType* array, const IndexContainer& indices) { + if (indices.size() == 0) { + return 0; + } + const int num_indices = indices.size(); + const int num_elements = array->size(); + DCHECK_LE(num_indices, num_elements); + if (num_indices == num_elements) { + // Assumes that 'indices' consists of [0 ... N-1]. + array->Clear(); + return num_indices; + } + typename IndexContainer::const_iterator remove_iter = indices.begin(); + int write_index = *(remove_iter++); + for (int scan = write_index + 1; scan < num_elements; ++scan) { + if (remove_iter != indices.end() && *remove_iter == scan) { + ++remove_iter; + } else { + array->SwapElements(scan, write_index++); + } + } + DCHECK_EQ(write_index, num_elements - num_indices); + Truncate(array, write_index); + return num_indices; +} +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // OR_TOOLS_BASE_PROTOBUF_UTIL_H_ \ No newline at end of file diff --git a/ortools/base/status_macros.h b/ortools/base/status_macros.h new file mode 100644 index 0000000000..e4a96856a2 --- /dev/null +++ b/ortools/base/status_macros.h @@ -0,0 +1,65 @@ +// Copyright 2010-2018 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_BASE_STATUS_MACROS_H_ +#define OR_TOOLS_BASE_STATUS_MACROS_H_ + +#include "ortools/base/status.h" +#include "ortools/base/statusor.h" + +namespace util { + +// Run a command that returns a util::Status. If the called code returns an +// error status, return that status up out of this method too. +// +// Example: +// RETURN_IF_ERROR(DoThings(4)); +#define RETURN_IF_ERROR(expr) \ + do { \ + /* Using _status below to avoid capture problems if expr is "status". */ \ + const ::util::Status _status = (expr); \ + if (!_status.ok()) return _status; \ + } while (0) + +// Internal helper for concatenating macro values. +#define STATUS_MACROS_CONCAT_NAME_INNER(x, y) x##y +#define STATUS_MACROS_CONCAT_NAME(x, y) STATUS_MACROS_CONCAT_NAME_INNER(x, y) + +template +::util::Status DoAssignOrReturn(T& lhs, ::util::StatusOr result) { // NOLINT + if (result.ok()) { + lhs = result.ValueOrDie(); + } + return result.status(); +} + +#define ASSIGN_OR_RETURN_IMPL(status, lhs, rexpr) \ + ::util::Status status = DoAssignOrReturn(lhs, (rexpr)); \ + if (!status.ok()) return status; + +// Executes an expression that returns a util::StatusOr, extracting its value +// into the variable defined by lhs (or returning on error). +// +// Example: Assigning to an existing value +// ValueType value; +// ASSIGN_OR_RETURN(value, MaybeGetValue(arg)); +// +// WARNING: ASSIGN_OR_RETURN expands into multiple statements; it cannot be used +// in a single statement (e.g. as the body of an if statement without {})! +#define ASSIGN_OR_RETURN(lhs, rexpr) \ + ASSIGN_OR_RETURN_IMPL( \ + STATUS_MACROS_CONCAT_NAME(_status_or_value, __COUNTER__), lhs, rexpr); + +} // namespace util + +#endif // OR_TOOLS_BASE_STATUS_MACROS_H_ diff --git a/ortools/base/statusor.h b/ortools/base/statusor.h index 7c1093bd1d..46121ed181 100644 --- a/ortools/base/statusor.h +++ b/ortools/base/statusor.h @@ -39,6 +39,14 @@ struct StatusOr { Status status() const { return status_; } +template +T value_or(U&& default_value) const& { + if (ok()) { + return value_; + } + return std::forward(default_value); +} + private: T value_; Status status_; diff --git a/ortools/constraint_solver/routing.cc b/ortools/constraint_solver/routing.cc index ac7628abd6..41fc36dc9b 100644 --- a/ortools/constraint_solver/routing.cc +++ b/ortools/constraint_solver/routing.cc @@ -656,6 +656,9 @@ RoutingModel::RoutingModel(const RoutingIndexManager& index_manager, cache_callbacks_(false), vehicle_class_index_of_vehicle_(vehicles_, VehicleClassIndex(-1)), vehicle_pickup_delivery_policy_(vehicles_, PICKUP_AND_DELIVERY_NO_ORDER), + has_hard_type_incompatibilities_(false), + has_temporal_type_incompatibilities_(false), + has_temporal_type_requirements_(false), num_visit_types_(0), starts_(vehicles_), ends_(vehicles_), @@ -1873,9 +1876,9 @@ void RoutingModel::CloseModelWithParameters( solver_->MakeIsDifferentCstCt(vehicle_vars_[i], -1, active_[i])); } - if (HasTemporalTypeIncompatibilities() || HasHardTypeIncompatibilities()) { + if (HasTypeRegulations()) { solver_->AddConstraint( - solver_->RevAlloc(new TypeIncompatibilityConstraint(*this))); + solver_->RevAlloc(new TypeRegulationsConstraint(*this))); } // Associate first and "logical" last nodes @@ -3654,69 +3657,59 @@ int RoutingModel::GetVisitType(int64 index) const { return index_to_visit_type_[index]; } -void RoutingModel::AddTypeIncompatibilityInternal( - int type1, int type2, - std::vector>* incompatible_types_per_type_index) { - // Resizing incompatible_types_per_type_index if necessary (first - // incompatibility or new type). - num_visit_types_ = std::max(num_visit_types_, std::max(type1, type2) + 1); - if (incompatible_types_per_type_index->size() < num_visit_types_) { - incompatible_types_per_type_index->resize(num_visit_types_); - } - (*incompatible_types_per_type_index)[type1].insert(type2); - (*incompatible_types_per_type_index)[type2].insert(type1); +void RoutingModel::CloseVisitTypes() { + hard_incompatible_types_per_type_index_.resize(num_visit_types_); + temporal_incompatible_types_per_type_index_.resize(num_visit_types_); + temporal_required_type_alternatives_per_type_index_.resize(num_visit_types_); } void RoutingModel::AddHardTypeIncompatibility(int type1, int type2) { - AddTypeIncompatibilityInternal(type1, type2, - &hard_incompatible_types_per_type_index_); + DCHECK_LT(std::max(type1, type2), + hard_incompatible_types_per_type_index_.size()); + has_hard_type_incompatibilities_ = true; + + hard_incompatible_types_per_type_index_[type1].insert(type2); + hard_incompatible_types_per_type_index_[type2].insert(type1); } void RoutingModel::AddTemporalTypeIncompatibility(int type1, int type2) { - AddTypeIncompatibilityInternal(type1, type2, - &temporal_incompatible_types_per_type_index_); -} + DCHECK_LT(std::max(type1, type2), + temporal_incompatible_types_per_type_index_.size()); + has_temporal_type_incompatibilities_ = true; -const absl::flat_hash_set& -RoutingModel::GetTypeIncompatibilitiesOfTypeInternal( - int type, const std::vector>& - incompatible_types_per_type_index) const { - CHECK_GE(type, 0); - if (type >= incompatible_types_per_type_index.size()) { - // No incompatibilities added for this type. - return empty_incompatibilities_; - } - return incompatible_types_per_type_index[type]; + temporal_incompatible_types_per_type_index_[type1].insert(type2); + temporal_incompatible_types_per_type_index_[type2].insert(type1); } const absl::flat_hash_set& RoutingModel::GetHardTypeIncompatibilitiesOfType(int type) const { - return GetTypeIncompatibilitiesOfTypeInternal( - type, hard_incompatible_types_per_type_index_); + DCHECK_GE(type, 0); + DCHECK_LT(type, hard_incompatible_types_per_type_index_.size()); + return hard_incompatible_types_per_type_index_[type]; } const absl::flat_hash_set& RoutingModel::GetTemporalTypeIncompatibilitiesOfType(int type) const { - return GetTypeIncompatibilitiesOfTypeInternal( - type, temporal_incompatible_types_per_type_index_); + DCHECK_GE(type, 0); + DCHECK_LT(type, temporal_incompatible_types_per_type_index_.size()); + return temporal_incompatible_types_per_type_index_[type]; } -const absl::flat_hash_set& -RoutingModel::GetHardTypeIncompatibilitiesOfNode(int64 index) const { - const int type = GetVisitType(index); - return type == kUnassigned - ? empty_incompatibilities_ - : GetTypeIncompatibilitiesOfTypeInternal( - type, hard_incompatible_types_per_type_index_); +void RoutingModel::AddTemporalRequiredTypeAlternatives( + int dependent_type, absl::flat_hash_set required_type_alternatives) { + DCHECK_LT(dependent_type, + temporal_required_type_alternatives_per_type_index_.size()); + has_temporal_type_requirements_ = true; + + temporal_required_type_alternatives_per_type_index_[dependent_type].push_back( + std::move(required_type_alternatives)); } -const absl::flat_hash_set& -RoutingModel::GetTemporalTypeIncompatibilitiesOfNode(int64 index) const { - const int type = GetVisitType(index); - return type == kUnassigned - ? empty_incompatibilities_ - : GetTypeIncompatibilitiesOfTypeInternal( - type, temporal_incompatible_types_per_type_index_); +const std::vector>& +RoutingModel::GetTemporalRequiredTypeAlternativesOfType(int type) const { + DCHECK_GE(type, 0); + DCHECK_LT(type, temporal_required_type_alternatives_per_type_index_.size()); + return temporal_required_type_alternatives_per_type_index_[type]; } int64 RoutingModel::UnperformedPenalty(int64 var_index) const { @@ -4218,8 +4211,8 @@ RoutingModel::GetOrCreateLocalSearchFilters() { filters_.push_back(MakePickupDeliveryFilter( *this, pickup_delivery_pairs_, vehicle_pickup_delivery_policy_)); } - if (HasTemporalTypeIncompatibilities() || HasHardTypeIncompatibilities()) { - filters_.push_back(MakeTypeIncompatibilityFilter(*this)); + if (HasTypeRegulations()) { + filters_.push_back(MakeTypeRegulationsFilter(*this)); } filters_.push_back(MakeVehicleVarFilter(*this)); @@ -4250,8 +4243,8 @@ RoutingModel::GetOrCreateFeasibilityFilters() { feasibility_filters_.push_back(MakePickupDeliveryFilter( *this, pickup_delivery_pairs_, vehicle_pickup_delivery_policy_)); } - if (HasTemporalTypeIncompatibilities() || HasHardTypeIncompatibilities()) { - feasibility_filters_.push_back(MakeTypeIncompatibilityFilter(*this)); + if (HasTypeRegulations()) { + feasibility_filters_.push_back(MakeTypeRegulationsFilter(*this)); } feasibility_filters_.push_back(MakeVehicleVarFilter(*this)); @@ -5920,14 +5913,13 @@ bool DisjunctivePropagator::ForbiddenIntervals(Tasks* tasks) { return true; } -TypeIncompatibilityChecker::TypeIncompatibilityChecker( - const RoutingModel& model) - : model_(model) { - if (!model.HasTemporalTypeIncompatibilities() && - !model.HasHardTypeIncompatibilities()) { +TypeRegulationsChecker::TypeRegulationsChecker(const RoutingModel& model) + : model_(model), + pickup_delivery_status_of_node_(model.Size()), + counts_of_type_(model.GetNumberOfVisitTypes()) { + if (!model.HasTypeRegulations()) { return; } - pickup_delivery_status_of_node_.resize(model.Size()); for (int node_index = 0; node_index < model.Size(); node_index++) { const std::vector>& pickup_index_pairs = model.GetPickupIndexPairs(node_index); @@ -5950,50 +5942,25 @@ TypeIncompatibilityChecker::TypeIncompatibilityChecker( } } -bool TypeIncompatibilityChecker::TemporalIncompatibilitiesRespectedOnVehicle( - int vehicle, const std::function& next_accessor) const { - return IncompatibilitiesRespectedOnVehicle( - vehicle, next_accessor, /*check_hard_incompatibilities=*/true); -} - -bool TypeIncompatibilityChecker::AllIncompatibilitiesRespectedOnVehicle( - int vehicle, const std::function& next_accessor) const { - return IncompatibilitiesRespectedOnVehicle( - vehicle, next_accessor, /*check_hard_incompatibilities=*/true); -} - -// TODO(user): Remove the check_hard_incompatibilities boolean and always -// check both incompatibilities to simplify the code. -bool TypeIncompatibilityChecker::IncompatibilitiesRespectedOnVehicle( - int vehicle, const std::function& next_accessor, - bool check_hard_incompatibilities) const { - if (!model_.HasTemporalTypeIncompatibilities() && - (!check_hard_incompatibilities || - !model_.HasHardTypeIncompatibilities())) { +bool TypeRegulationsChecker::CheckVehicle( + int vehicle, const std::function& next_accessor) { + if (!HasRegulationsToCheck()) { return true; } - struct NodeCount { - int non_pickup_delivery = 0; - int pickup = 0; - int delivery = 0; - }; + // Accumulates the count of types before the current node. - std::vector counts_of_type(model_.GetNumberOfVisitTypes()); + counts_of_type_.assign(model_.GetNumberOfVisitTypes(), NodeCount()); for (int64 current = model_.Start(vehicle); !model_.IsEnd(current); current = next_accessor(current)) { - if (model_.GetTemporalTypeIncompatibilitiesOfNode(current).empty() && - (!check_hard_incompatibilities || - model_.GetHardTypeIncompatibilitiesOfNode(current).empty())) { + const int type = model_.GetVisitType(current); + if (type < 0) { continue; } - const int type = model_.GetVisitType(current); - // If the node had no type, its incompatibilities would have been empty. - DCHECK_GE(type, 0); - DCHECK_LT(type, counts_of_type.size()); + DCHECK_LT(type, counts_of_type_.size()); const PickupDeliveryStatus pickup_delivery_status = pickup_delivery_status_of_node_[current]; - NodeCount& counts = counts_of_type[type]; + NodeCount& counts = counts_of_type_[type]; if (pickup_delivery_status == DELIVERY) { // The node is a delivery. if (counts.pickup <= counts.delivery) { @@ -6005,28 +5972,8 @@ bool TypeIncompatibilityChecker::IncompatibilitiesRespectedOnVehicle( continue; } // The node is either a pickup or a "fixed" (non-pickup/delivery) node. - // Verify incompatibilities. - - for (int incompatible_type : - model_.GetTemporalTypeIncompatibilitiesOfNode(current)) { - const NodeCount& incompatible_counts = counts_of_type[incompatible_type]; - const int non_delivered_count = - incompatible_counts.pickup - incompatible_counts.delivery; - if (non_delivered_count + incompatible_counts.non_pickup_delivery > 0) { - return false; - } - } - if (check_hard_incompatibilities) { - for (int incompatible_type : - model_.GetHardTypeIncompatibilitiesOfNode(current)) { - const NodeCount& incompatible_counts = - counts_of_type[incompatible_type]; - if (incompatible_counts.non_pickup_delivery + - incompatible_counts.pickup > - 0) { - return false; - } - } + if (!CheckTypeRegulations(type)) { + return false; } // Update count of type based on whether it is a pickup or not. int& count = pickup_delivery_status == NONE ? counts.non_pickup_delivery @@ -6036,14 +5983,75 @@ bool TypeIncompatibilityChecker::IncompatibilitiesRespectedOnVehicle( return true; } -TypeIncompatibilityConstraint::TypeIncompatibilityConstraint( - const RoutingModel& model) +int TypeRegulationsChecker::GetNonDeliveryCount(int type) const { + const NodeCount& counts = counts_of_type_[type]; + return counts.non_pickup_delivery + counts.pickup; +} + +int TypeRegulationsChecker::GetNonDeliveredCount(int type) const { + return GetNonDeliveryCount(type) - counts_of_type_[type].delivery; +} + +TypeIncompatibilityChecker::TypeIncompatibilityChecker( + const RoutingModel& model, bool check_hard_incompatibilities) + : TypeRegulationsChecker(model), + check_hard_incompatibilities_(check_hard_incompatibilities) {} + +bool TypeIncompatibilityChecker::HasRegulationsToCheck() const { + return model_.HasTemporalTypeIncompatibilities() || + (check_hard_incompatibilities_ && + model_.HasHardTypeIncompatibilities()); +} + +// TODO(user): Remove the check_hard_incompatibilities_ boolean and always +// check both incompatibilities to simplify the code? +bool TypeIncompatibilityChecker::CheckTypeRegulations(int type) const { + for (int incompatible_type : + model_.GetTemporalTypeIncompatibilitiesOfType(type)) { + if (GetNonDeliveredCount(incompatible_type) > 0) { + return false; + } + } + if (check_hard_incompatibilities_) { + for (int incompatible_type : + model_.GetHardTypeIncompatibilitiesOfType(type)) { + if (GetNonDeliveryCount(incompatible_type) > 0) { + return false; + } + } + } + return true; +} + +bool TypeRequirementChecker::HasRegulationsToCheck() const { + return model_.HasTemporalTypeRequirements(); +} + +bool TypeRequirementChecker::CheckTypeRegulations(int type) const { + for (const absl::flat_hash_set& requirement_alternatives : + model_.GetTemporalRequiredTypeAlternativesOfType(type)) { + bool has_one_of_alternatives = false; + for (const int type_alternative : requirement_alternatives) { + if (GetNonDeliveredCount(type_alternative) > 0) { + has_one_of_alternatives = true; + break; + } + } + if (!has_one_of_alternatives) { + return false; + } + } + return true; +} + +TypeRegulationsConstraint::TypeRegulationsConstraint(const RoutingModel& model) : Constraint(model.solver()), model_(model), - incompatibility_checker_(model), + incompatibility_checker_(model, /*check_hard_incompatibilities*/ true), + requirement_checker_(model), vehicle_demons_(model.vehicles()) {} -void TypeIncompatibilityConstraint::PropagateNodeIncompatibilities(int node) { +void TypeRegulationsConstraint::PropagateNodeRegulations(int node) { DCHECK_LT(node, model_.Size()); if (!model_.VehicleVar(node)->Bound() || !model_.NextVar(node)->Bound()) { // Vehicle var or Next var not bound. @@ -6055,40 +6063,38 @@ void TypeIncompatibilityConstraint::PropagateNodeIncompatibilities(int node) { EnqueueDelayedDemon(vehicle_demons_[vehicle]); } -void TypeIncompatibilityConstraint::CheckIncompatibilitiesOnVehicle( - int vehicle) { - if (!incompatibility_checker_.AllIncompatibilitiesRespectedOnVehicle( - vehicle, /*next_accessor*/ [this, vehicle](int64 node) { - if (model_.NextVar(node)->Bound()) { - return model_.NextVar(node)->Value(); - } - // Node not bound, skip to the end of the vehicle. - return model_.End(vehicle); - })) { +void TypeRegulationsConstraint::CheckRegulationsOnVehicle(int vehicle) { + const auto next_accessor = [this, vehicle](int64 node) { + if (model_.NextVar(node)->Bound()) { + return model_.NextVar(node)->Value(); + } + // Node not bound, skip to the end of the vehicle. + return model_.End(vehicle); + }; + if (!incompatibility_checker_.CheckVehicle(vehicle, next_accessor) || + !requirement_checker_.CheckVehicle(vehicle, next_accessor)) { model_.solver()->Fail(); } } -void TypeIncompatibilityConstraint::Post() { +void TypeRegulationsConstraint::Post() { for (int vehicle = 0; vehicle < model_.vehicles(); vehicle++) { vehicle_demons_[vehicle] = MakeDelayedConstraintDemon1( - solver(), this, - &TypeIncompatibilityConstraint::CheckIncompatibilitiesOnVehicle, - "CheckIncompatibilitiesOnVehicle", vehicle); + solver(), this, &TypeRegulationsConstraint::CheckRegulationsOnVehicle, + "CheckRegulationsOnVehicle", vehicle); } for (int node = 0; node < model_.Size(); node++) { Demon* node_demon = MakeConstraintDemon1( - solver(), this, - &TypeIncompatibilityConstraint::PropagateNodeIncompatibilities, - "PropagateNodeIncompatibilities", node); + solver(), this, &TypeRegulationsConstraint::PropagateNodeRegulations, + "PropagateNodeRegulations", node); model_.NextVar(node)->WhenBound(node_demon); model_.VehicleVar(node)->WhenBound(node_demon); } } -void TypeIncompatibilityConstraint::InitialPropagate() { +void TypeRegulationsConstraint::InitialPropagate() { for (int vehicle = 0; vehicle < model_.vehicles(); vehicle++) { - CheckIncompatibilitiesOnVehicle(vehicle); + CheckRegulationsOnVehicle(vehicle); } } diff --git a/ortools/constraint_solver/routing.h b/ortools/constraint_solver/routing.h index 10f5331866..8855fe3aa9 100644 --- a/ortools/constraint_solver/routing.h +++ b/ortools/constraint_solver/routing.h @@ -680,18 +680,28 @@ class RoutingModel { return pickup_delivery_disjunctions_; } #endif // SWIG - // Set the node visit types and incompatibilities between the types. - // Two nodes with "hard" incompatible types cannot share the same route at - // all, while with a "temporal" incompatibility they can't be on the same - // route at the same time. + // Set the node visit types and incompatibilities/requirements between the + // types (see below). // NOTE: The visit type of a node must be positive, and all nodes belonging to // the same pickup/delivery pair must have the same type (or no type at all). - // NOTE: These incompatibilities are only handled when each node index appears - // in at most one pickup/delivery pair, i.e. when the same node isn't a pickup - // and/or delivery in multiple pickup/delivery pairs. + // NOTE: Before adding any incompatibilities and/or requirements on types: + // 1) All corresponding node types must have been set. + // 2) CloseVisitTypes() must be called so all containers are resized + // accordingly. + // NOTE: These incompatibilities and requirements are only handled when each + // node index appears in at most one pickup/delivery pair, i.e. when the same + // node isn't a pickup and/or delivery in multiple pickup/delivery pairs. // TODO(user): Support multiple visit types per node? void SetVisitType(int64 index, int type); int GetVisitType(int64 index) const; + // This function should be called once all node visit types have been set and + // prior to adding any incompatibilities/requirements. + void CloseVisitTypes(); + int GetNumberOfVisitTypes() const { return num_visit_types_; } + // Incompatibilities: + // Two nodes with "hard" incompatible types cannot share the same route at + // all, while with a "temporal" incompatibility they can't be on the same + // route at the same time. void AddHardTypeIncompatibility(int type1, int type2); void AddTemporalTypeIncompatibility(int type1, int type2); // Returns visit types incompatible with a given type. @@ -699,20 +709,42 @@ class RoutingModel { int type) const; const absl::flat_hash_set& GetTemporalTypeIncompatibilitiesOfType( int type) const; - // Returns types incompatible with a given node index. - const absl::flat_hash_set& GetHardTypeIncompatibilitiesOfNode( - int64 index) const; - const absl::flat_hash_set& GetTemporalTypeIncompatibilitiesOfNode( - int64 index) const; // Returns true iff any hard (resp. temporal) type incompatibilities have been // added to the model. bool HasHardTypeIncompatibilities() const { - return !hard_incompatible_types_per_type_index_.empty(); + return has_hard_type_incompatibilities_; } bool HasTemporalTypeIncompatibilities() const { - return !temporal_incompatible_types_per_type_index_.empty(); + return has_temporal_type_incompatibilities_; } - int GetNumberOfVisitTypes() const { return num_visit_types_; } + // Requirements: + // If type_D temporally depends on type_R, any non-delivery node_D of type_D + // requires at least one non-delivered node of type_R on its vehicle at the + // time node_D is visited. + // NOTE: As of 2019-04, cycles in the requirement graph are not supported, + // and lead to the dependent nodes being skipped if possible (otherwise + // the model is considered infeasible). + // The following function specifies that "dependent_type" requires at least + // one of the types in "required_type_alternatives". + void AddTemporalRequiredTypeAlternatives( + int dependent_type, absl::flat_hash_set required_type_alternatives); + // clang-format off + // Returns all sets of requirement alternatives for the given type. + const std::vector >& + GetTemporalRequiredTypeAlternativesOfType(int type) const; + // clang-format on + // Returns true iff any type requirements have been added to the model. + bool HasTemporalTypeRequirements() const { + return has_temporal_type_requirements_; + } + + // Returns true iff the model has any incompatibilities or requirements set + // on node types. + bool HasTypeRegulations() const { + return HasTemporalTypeIncompatibilities() || + HasHardTypeIncompatibilities() || HasTemporalTypeRequirements(); + } + // Get the "unperformed" penalty of a node. This is only well defined if the // node is only part of a single Disjunction involving only itself, and that // disjunction has a penalty. In all other cases, including forced active @@ -1320,15 +1352,6 @@ class RoutingModel { // Sets up pickup and delivery sets. void AddPickupAndDeliverySetsInternal(const std::vector& pickups, const std::vector& deliveries); - // Setup/access type incompatibilities. - // clang-format off - void AddTypeIncompatibilityInternal(int type1, int type2, - std::vector >* - incompatible_types_per_type_index); - const absl::flat_hash_set& GetTypeIncompatibilitiesOfTypeInternal( - int type, const std::vector >& - incompatible_types_per_type_index) const; - // clang-format on // Returns the cost variable related to the soft same vehicle constraint of // index 'vehicle_index'. IntVar* CreateSameVehicleCost(int vehicle_index); @@ -1513,12 +1536,15 @@ class RoutingModel { // clang-format off std::vector > hard_incompatible_types_per_type_index_; + bool has_hard_type_incompatibilities_; std::vector > temporal_incompatible_types_per_type_index_; + bool has_temporal_type_incompatibilities_; + + std::vector > > + temporal_required_type_alternatives_per_type_index_; + bool has_temporal_type_requirements_; // clang-format on - // Empty set used in Get[Hard|Temporal]TypeIncompatibilities() when the given - // type has no incompatibilities. - const absl::flat_hash_set empty_incompatibilities_; int num_visit_types_; // Two indices are equivalent if they correspond to the same node (as given to // the constructors taking a RoutingIndexManager). @@ -1756,32 +1782,68 @@ class GlobalVehicleBreaksConstraint : public Constraint { std::vector fixed_transits_; }; -class TypeIncompatibilityChecker { +class TypeRegulationsChecker { public: - explicit TypeIncompatibilityChecker(const RoutingModel& model); + explicit TypeRegulationsChecker(const RoutingModel& model); + virtual ~TypeRegulationsChecker() {} - bool TemporalIncompatibilitiesRespectedOnVehicle( - int vehicle, const std::function& next_accessor) const; + bool CheckVehicle(int vehicle, + const std::function& next_accessor); - bool AllIncompatibilitiesRespectedOnVehicle( - int vehicle, const std::function& next_accessor) const; - - private: + protected: enum PickupDeliveryStatus { PICKUP, DELIVERY, NONE }; + struct NodeCount { + int non_pickup_delivery = 0; + int pickup = 0; + int delivery = 0; + }; - // NOTE(user): As temporal incompatibilities are always verified when - // calling this function, we only pass 1 boolean indicating whether or not - // hard incompatibilities are also respected. - bool IncompatibilitiesRespectedOnVehicle( - int vehicle, const std::function& next_accessor, - bool check_hard_incompatibilities) const; + // Returns the number of pickups and fixed nodes from counts_of_type_["type"]. + int GetNonDeliveryCount(int type) const; + // Same as above, but substracting the number of deliveries of "type". + int GetNonDeliveredCount(int type) const; + + virtual bool HasRegulationsToCheck() const = 0; + virtual bool CheckTypeRegulations(int type) const = 0; const RoutingModel& model_; + + private: std::vector pickup_delivery_status_of_node_; + std::vector counts_of_type_; }; -// The following constraint ensures that incompatibilities between types are -// respected. +// Checker for type incompatibilities. +class TypeIncompatibilityChecker : public TypeRegulationsChecker { + public: + TypeIncompatibilityChecker(const RoutingModel& model, + bool check_hard_incompatibilities); + ~TypeIncompatibilityChecker() override {} + + private: + bool HasRegulationsToCheck() const override; + bool CheckTypeRegulations(int type) const override; + // NOTE(user): As temporal incompatibilities are always verified with this + // checker, we only store 1 boolean indicating whether or not hard + // incompatibilities are also verified. + bool check_hard_incompatibilities_; +}; + +// Checker for type requirements. +class TypeRequirementChecker : public TypeRegulationsChecker { + public: + explicit TypeRequirementChecker(const RoutingModel& model) + : TypeRegulationsChecker(model) {} + ~TypeRequirementChecker() override {} + + private: + bool HasRegulationsToCheck() const override; + bool CheckTypeRegulations(int type) const override; +}; + +// The following constraint ensures that incompatibilities and requirements +// between types are respected. +// // It verifies both "hard" and "temporal" incompatibilities. // Two nodes with hard incompatible types cannot be served by the same vehicle // at all, while with a temporal incompatibility they can't be on the same route @@ -1791,19 +1853,24 @@ class TypeIncompatibilityChecker { // non-pickup/delivery node n of type T3, the configuration // p1 --> d1 --> n --> p2 --> d2 is acceptable, whereas any configurations // with p1 --> p2 --> d1 --> ..., or p1 --> n --> d1 --> ... is not feasible. -class TypeIncompatibilityConstraint : public Constraint { +// +// It also verifies temporal type requirements. +// In the above example, if T1 is a requirement for T2, p2 must be visited +// between p1 and d1. +class TypeRegulationsConstraint : public Constraint { public: - explicit TypeIncompatibilityConstraint(const RoutingModel& model); + explicit TypeRegulationsConstraint(const RoutingModel& model); void Post() override; void InitialPropagate() override; private: - void PropagateNodeIncompatibilities(int node); - void CheckIncompatibilitiesOnVehicle(int vehicle); + void PropagateNodeRegulations(int node); + void CheckRegulationsOnVehicle(int vehicle); const RoutingModel& model_; - const TypeIncompatibilityChecker incompatibility_checker_; + TypeIncompatibilityChecker incompatibility_checker_; + TypeRequirementChecker requirement_checker_; std::vector vehicle_demons_; }; @@ -3028,7 +3095,7 @@ IntVarLocalSearchFilter* MakeNodeDisjunctionFilter( IntVarLocalSearchFilter* MakeVehicleAmortizedCostFilter( const RoutingModel& routing_model, Solver::ObjectiveWatcher objective_callback); -IntVarLocalSearchFilter* MakeTypeIncompatibilityFilter( +IntVarLocalSearchFilter* MakeTypeRegulationsFilter( const RoutingModel& routing_model); std::vector MakeCumulFilters( const RoutingDimension& dimension, diff --git a/ortools/constraint_solver/routing_search.cc b/ortools/constraint_solver/routing_search.cc index 8f2ecbabec..8d6b8e9472 100644 --- a/ortools/constraint_solver/routing_search.cc +++ b/ortools/constraint_solver/routing_search.cc @@ -698,13 +698,11 @@ IntVarLocalSearchFilter* MakeVehicleAmortizedCostFilter( namespace { -class TypeIncompatibilityFilter : public BasePathFilter { +class TypeRegulationsFilter : public BasePathFilter { public: - explicit TypeIncompatibilityFilter(const RoutingModel& model); - ~TypeIncompatibilityFilter() override {} - std::string DebugString() const override { - return "TypeIncompatibilityFilter"; - } + explicit TypeRegulationsFilter(const RoutingModel& model); + ~TypeRegulationsFilter() override {} + std::string DebugString() const override { return "TypeRegulationsFilter"; } private: void OnSynchronizePathFromStart(int64 start) override; @@ -720,14 +718,17 @@ class TypeIncompatibilityFilter : public BasePathFilter { // incompatibilities. std::vector> hard_incompatibility_type_counts_per_vehicle_; // Used to verify the temporal incompatibilities. - const TypeIncompatibilityChecker temporal_incompatibility_checker_; + TypeIncompatibilityChecker temporal_incompatibility_checker_; + TypeRequirementChecker requirement_checker_; }; -TypeIncompatibilityFilter::TypeIncompatibilityFilter(const RoutingModel& model) +TypeRegulationsFilter::TypeRegulationsFilter(const RoutingModel& model) : BasePathFilter(model.Nexts(), model.Size() + model.vehicles(), nullptr), routing_model_(model), start_to_vehicle_(model.Size(), -1), - temporal_incompatibility_checker_(model) { + temporal_incompatibility_checker_(model, + /*check_hard_incompatibilities*/ false), + requirement_checker_(model) { const int num_vehicles = model.vehicles(); const bool has_hard_type_incompatibilities = model.HasHardTypeIncompatibilities(); @@ -745,7 +746,7 @@ TypeIncompatibilityFilter::TypeIncompatibilityFilter(const RoutingModel& model) } } -void TypeIncompatibilityFilter::OnSynchronizePathFromStart(int64 start) { +void TypeRegulationsFilter::OnSynchronizePathFromStart(int64 start) { if (!routing_model_.HasHardTypeIncompatibilities()) return; const int vehicle = start_to_vehicle_[start]; @@ -767,8 +768,9 @@ void TypeIncompatibilityFilter::OnSynchronizePathFromStart(int64 start) { } } -bool TypeIncompatibilityFilter::HardIncompatibilitiesRespected( - int vehicle, int64 chain_start, int64 chain_end) { +bool TypeRegulationsFilter::HardIncompatibilitiesRespected(int vehicle, + int64 chain_start, + int64 chain_end) { if (!routing_model_.HasHardTypeIncompatibilities()) return true; std::vector new_type_counts = @@ -817,25 +819,23 @@ bool TypeIncompatibilityFilter::HardIncompatibilitiesRespected( return true; } -bool TypeIncompatibilityFilter::AcceptPath(int64 path_start, int64 chain_start, - int64 chain_end) { +bool TypeRegulationsFilter::AcceptPath(int64 path_start, int64 chain_start, + int64 chain_end) { const int vehicle = start_to_vehicle_[path_start]; CHECK_GE(vehicle, 0); + const auto next_accessor = [this](int64 node) { return GetNext(node); }; return HardIncompatibilitiesRespected(vehicle, chain_start, chain_end) && - temporal_incompatibility_checker_ - .TemporalIncompatibilitiesRespectedOnVehicle( - vehicle, - /*next_accessor*/ [this](int64 node) { - return GetNext(node); - }); + temporal_incompatibility_checker_.CheckVehicle(vehicle, + next_accessor) && + requirement_checker_.CheckVehicle(vehicle, next_accessor); } } // namespace -IntVarLocalSearchFilter* MakeTypeIncompatibilityFilter( +IntVarLocalSearchFilter* MakeTypeRegulationsFilter( const RoutingModel& routing_model) { return routing_model.solver()->RevAlloc( - new TypeIncompatibilityFilter(routing_model)); + new TypeRegulationsFilter(routing_model)); } namespace { diff --git a/ortools/glop/basis_representation.cc b/ortools/glop/basis_representation.cc index c93df20903..379082301e 100644 --- a/ortools/glop/basis_representation.cc +++ b/ortools/glop/basis_representation.cc @@ -174,10 +174,12 @@ void EtaFactorization::RightSolve(DenseColumn* d) const { // -------------------------------------------------------- // BasisFactorization // -------------------------------------------------------- -BasisFactorization::BasisFactorization(const MatrixView& matrix, - const RowToColMapping& basis) +BasisFactorization::BasisFactorization( + const MatrixView& matrix, const CompactSparseMatrix& compact_matrix, + const RowToColMapping& basis) : stats_(), matrix_(matrix), + compact_matrix_(compact_matrix), basis_(basis), tau_is_computed_(false), max_num_updates_(0), @@ -197,10 +199,10 @@ void BasisFactorization::Clear() { eta_factorization_.Clear(); lu_factorization_.Clear(); rank_one_factorization_.Clear(); - storage_.Reset(matrix_.num_rows()); - right_storage_.Reset(matrix_.num_rows()); - left_pool_mapping_.assign(matrix_.num_cols(), kInvalidCol); - right_pool_mapping_.assign(matrix_.num_cols(), kInvalidCol); + storage_.Reset(compact_matrix_.num_rows()); + right_storage_.Reset(compact_matrix_.num_rows()); + left_pool_mapping_.assign(compact_matrix_.num_cols(), kInvalidCol); + right_pool_mapping_.assign(compact_matrix_.num_cols(), kInvalidCol); } Status BasisFactorization::Initialize() { @@ -308,7 +310,7 @@ Status BasisFactorization::Update(ColIndex entering_col, void BasisFactorization::LeftSolve(ScatteredRow* y) const { SCOPED_TIME_STAT(&stats_); RETURN_IF_NULL(y); - BumpDeterministicTimeForSolve(matrix_.num_rows().value()); + BumpDeterministicTimeForSolve(compact_matrix_.num_rows().value()); if (use_middle_product_form_update_) { lu_factorization_.LeftSolveUWithNonZeros(y); rank_one_factorization_.LeftSolveWithNonZeros(y); @@ -340,7 +342,7 @@ void BasisFactorization::RightSolve(ScatteredColumn* d) const { const DenseColumn& BasisFactorization::RightSolveForTau( const ScatteredColumn& a) const { SCOPED_TIME_STAT(&stats_); - BumpDeterministicTimeForSolve(matrix_.num_rows().value()); + BumpDeterministicTimeForSolve(compact_matrix_.num_rows().value()); if (use_middle_product_form_update_) { if (tau_computation_can_be_optimized_) { // Once used, the intermediate result is overridden, so RightSolveForTau() @@ -349,7 +351,7 @@ const DenseColumn& BasisFactorization::RightSolveForTau( lu_factorization_.RightSolveLWithPermutedInput(a.values, &tau_.values); tau_.non_zeros.clear(); } else { - ClearAndResizeVectorWithNonZeros(matrix_.num_rows(), &tau_); + ClearAndResizeVectorWithNonZeros(compact_matrix_.num_rows(), &tau_); lu_factorization_.RightSolveLForScatteredColumn(a, &tau_); } rank_one_factorization_.RightSolveWithNonZeros(&tau_); @@ -369,7 +371,8 @@ void BasisFactorization::LeftSolveForUnitRow(ColIndex j, SCOPED_TIME_STAT(&stats_); RETURN_IF_NULL(y); BumpDeterministicTimeForSolve(1); - ClearAndResizeVectorWithNonZeros(RowToColIndex(matrix_.num_rows()), y); + ClearAndResizeVectorWithNonZeros(RowToColIndex(compact_matrix_.num_rows()), + y); if (!use_middle_product_form_update_) { (*y)[j] = 1.0; y->non_zeros.push_back(j); @@ -421,7 +424,8 @@ void BasisFactorization::TemporaryLeftSolveForUnitRow(ColIndex j, SCOPED_TIME_STAT(&stats_); RETURN_IF_NULL(y); BumpDeterministicTimeForSolve(1); - ClearAndResizeVectorWithNonZeros(RowToColIndex(matrix_.num_rows()), y); + ClearAndResizeVectorWithNonZeros(RowToColIndex(compact_matrix_.num_rows()), + y); lu_factorization_.LeftSolveUForUnitRow(j, y); lu_factorization_.LeftSolveLWithNonZeros(y, nullptr); y->SortNonZerosIfNeeded(); @@ -431,11 +435,12 @@ void BasisFactorization::RightSolveForProblemColumn(ColIndex col, ScatteredColumn* d) const { SCOPED_TIME_STAT(&stats_); RETURN_IF_NULL(d); - BumpDeterministicTimeForSolve(matrix_.column(col).num_entries().value()); - ClearAndResizeVectorWithNonZeros(matrix_.num_rows(), d); + BumpDeterministicTimeForSolve( + compact_matrix_.column(col).num_entries().value()); + ClearAndResizeVectorWithNonZeros(compact_matrix_.num_rows(), d); if (!use_middle_product_form_update_) { - matrix_.column(col).CopyToDenseVector(matrix_.num_rows(), &d->values); + compact_matrix_.ColumnCopyToClearedDenseColumn(col, &d->values); lu_factorization_.RightSolve(&d->values); eta_factorization_.RightSolve(&d->values); return; @@ -443,7 +448,7 @@ void BasisFactorization::RightSolveForProblemColumn(ColIndex col, // TODO(user): if right_pool_mapping_[col] != kInvalidCol, we can reuse it and // just apply the last rank one update since it was computed. - lu_factorization_.RightSolveLForSparseColumn(matrix_.column(col), d); + lu_factorization_.RightSolveLForColumnView(compact_matrix_.column(col), d); rank_one_factorization_.RightSolveWithNonZeros(d); if (col >= right_pool_mapping_.size()) { // This is needed because when we do an incremental solve with only new @@ -480,12 +485,12 @@ Fractional BasisFactorization::DualEdgeSquaredNorm(RowIndex row) const { } bool BasisFactorization::IsIdentityBasis() const { - const RowIndex num_rows = matrix_.num_rows(); + const RowIndex num_rows = compact_matrix_.num_rows(); for (RowIndex row(0); row < num_rows; ++row) { const ColIndex col = basis_[row]; - if (matrix_.column(col).num_entries().value() != 1) return false; - const Fractional coeff = matrix_.column(col).GetFirstCoefficient(); - const RowIndex entry_row = matrix_.column(col).GetFirstRow(); + if (compact_matrix_.column(col).num_entries().value() != 1) return false; + const Fractional coeff = compact_matrix_.column(col).GetFirstCoefficient(); + const RowIndex entry_row = compact_matrix_.column(col).GetFirstRow(); if (entry_row != row || coeff != 1.0) return false; } return true; @@ -510,7 +515,7 @@ Fractional BasisFactorization::ComputeInfinityNorm() const { Fractional BasisFactorization::ComputeInverseOneNorm() const { if (IsIdentityBasis()) return 1.0; - const RowIndex num_rows = matrix_.num_rows(); + const RowIndex num_rows = compact_matrix_.num_rows(); const ColIndex num_cols = RowToColIndex(num_rows); Fractional norm = 0.0; for (ColIndex col(0); col < num_cols; ++col) { @@ -532,7 +537,7 @@ Fractional BasisFactorization::ComputeInverseOneNorm() const { Fractional BasisFactorization::ComputeInverseInfinityNorm() const { if (IsIdentityBasis()) return 1.0; - const RowIndex num_rows = matrix_.num_rows(); + const RowIndex num_rows = compact_matrix_.num_rows(); const ColIndex num_cols = RowToColIndex(num_rows); DenseColumn row_sum(num_rows, 0.0); for (ColIndex col(0); col < num_cols; ++col) { @@ -567,7 +572,7 @@ Fractional BasisFactorization::ComputeInfinityNormConditionNumber() const { Fractional BasisFactorization::ComputeInfinityNormConditionNumberUpperBound() const { if (IsIdentityBasis()) return 1.0; - BumpDeterministicTimeForSolve(matrix_.num_rows().value()); + BumpDeterministicTimeForSolve(compact_matrix_.num_rows().value()); return ComputeInfinityNorm() * lu_factorization_.ComputeInverseInfinityNormUpperBound(); } @@ -578,9 +583,10 @@ double BasisFactorization::DeterministicTime() const { void BasisFactorization::BumpDeterministicTimeForSolve(int num_entries) const { // TODO(user): Spend more time finding a good approximation here. - if (matrix_.num_rows().value() == 0) return; - const double density = static_cast(num_entries) / - static_cast(matrix_.num_rows().value()); + if (compact_matrix_.num_rows().value() == 0) return; + const double density = + static_cast(num_entries) / + static_cast(compact_matrix_.num_rows().value()); deterministic_time_ += (1.0 + density) * DeterministicTimeForFpOperations( lu_factorization_.NumberOfEntries().value()) + diff --git a/ortools/glop/basis_representation.h b/ortools/glop/basis_representation.h index 3334f7ab8d..4b38e239aa 100644 --- a/ortools/glop/basis_representation.h +++ b/ortools/glop/basis_representation.h @@ -145,7 +145,9 @@ class EtaFactorization { // every 'refactorization_period' updates. class BasisFactorization { public: - BasisFactorization(const MatrixView& matrix, const RowToColMapping& basis); + BasisFactorization(const MatrixView& matrix, + const CompactSparseMatrix& compact_matrix, + const RowToColMapping& basis); virtual ~BasisFactorization(); // Sets the parameters for this component. @@ -183,7 +185,7 @@ class BasisFactorization { ABSL_MUST_USE_RESULT Status Initialize(); // Return the number of rows in the basis. - RowIndex GetNumberOfRows() const { return matrix_.num_rows(); } + RowIndex GetNumberOfRows() const { return compact_matrix_.num_rows(); } // Clears eta factorization and refactorizes LU. // Nothing happens if this is called on an already refactorized basis. @@ -305,6 +307,7 @@ class BasisFactorization { // References to the basis subpart of the linear program matrix. const MatrixView& matrix_; + const CompactSparseMatrix& compact_matrix_; const RowToColMapping& basis_; // Middle form product update factorization and scratchpad_ used to construct diff --git a/ortools/glop/lu_factorization.cc b/ortools/glop/lu_factorization.cc index a622206f8f..f53d6a797d 100644 --- a/ortools/glop/lu_factorization.cc +++ b/ortools/glop/lu_factorization.cc @@ -13,6 +13,7 @@ #include "ortools/glop/lu_factorization.h" +#include "ortools/lp_data/lp_types.h" #include "ortools/lp_data/lp_utils.h" namespace operations_research { @@ -186,15 +187,16 @@ void LuFactorization::RightSolveLWithPermutedInput(const DenseColumn& a, } } -void LuFactorization::RightSolveLForSparseColumn(const SparseColumn& b, - ScatteredColumn* x) const { +void LuFactorization::RightSolveLForColumnView( + const CompactSparseMatrix::ColumnView& b, ScatteredColumn* x) const { SCOPED_TIME_STAT(&stats_); DCHECK(IsAllZero(x->values)); x->non_zeros.clear(); if (is_identity_factorization_) { - for (const SparseColumn::Entry e : b) { - (*x)[e.row()] = e.coefficient(); - x->non_zeros.push_back(e.row()); + const EntryIndex num_entries = b.num_entries(); + for (EntryIndex i(0); i < num_entries; ++i) { + (*x)[b.EntryRow(i)] = b.EntryCoefficient(i); + x->non_zeros.push_back(b.EntryRow(i)); } return; } @@ -206,9 +208,10 @@ void LuFactorization::RightSolveLForSparseColumn(const SparseColumn& b, // of b. ColIndex first_column_to_consider(RowToColIndex(x->values.size())); const ColIndex limit = lower_.GetFirstNonIdentityColumn(); - for (const SparseColumn::Entry e : b) { - const RowIndex permuted_row = row_perm_[e.row()]; - (*x)[permuted_row] = e.coefficient(); + const EntryIndex num_entries = b.num_entries(); + for (EntryIndex i(0); i < num_entries; ++i) { + const RowIndex permuted_row = row_perm_[b.EntryRow(i)]; + (*x)[permuted_row] = b.EntryCoefficient(i); x->non_zeros.push_back(permuted_row); // The second condition only works because the elements on the diagonal of diff --git a/ortools/glop/lu_factorization.h b/ortools/glop/lu_factorization.h index ae5f863b78..359c6e079a 100644 --- a/ortools/glop/lu_factorization.h +++ b/ortools/glop/lu_factorization.h @@ -103,8 +103,8 @@ class LuFactorization { // or a ScatteredColumn as input. non_zeros will either be cleared or set to // the non zeros of the result. Important: the output x must be of the correct // size and all zero. - void RightSolveLForSparseColumn(const SparseColumn& b, - ScatteredColumn* x) const; + void RightSolveLForColumnView(const CompactSparseMatrix::ColumnView& b, + ScatteredColumn* x) const; void RightSolveLForScatteredColumn(const ScatteredColumn& b, ScatteredColumn* x) const; diff --git a/ortools/glop/revised_simplex.cc b/ortools/glop/revised_simplex.cc index b730edd1b7..799f53eb31 100644 --- a/ortools/glop/revised_simplex.cc +++ b/ortools/glop/revised_simplex.cc @@ -85,7 +85,7 @@ RevisedSimplex::RevisedSimplex() variable_name_(), direction_(), error_(), - basis_factorization_(matrix_with_slack_, basis_), + basis_factorization_(matrix_with_slack_, compact_matrix_, basis_), variables_info_(compact_matrix_, lower_bound_, upper_bound_), variable_values_(compact_matrix_, basis_, variables_info_, basis_factorization_), @@ -284,7 +284,7 @@ Status RevisedSimplex::Solve(const LinearProgram& lp, TimeLimit* time_limit) { // TODO(user): We should also confirm the PRIMAL_UNBOUNDED or DUAL_UNBOUNDED // status by checking with the other phase I that the problem is really - // DUAL_INFEASIBLE or PRIMAL_INFEASIBLE. For instace we currently report + // DUAL_INFEASIBLE or PRIMAL_INFEASIBLE. For instance we currently report // PRIMAL_UNBOUNDED with the primal on the problem l30.mps instead of // OPTIMAL and the dual does not have issues on this problem. if (problem_status_ == ProblemStatus::DUAL_UNBOUNDED) { @@ -1383,7 +1383,7 @@ void RevisedSimplex::CorrectErrorsOnVariableValues() { // problem by extending each basic variable bound with a random value. See how // bound_perturbation_ is used in ComputeHarrisRatioAndLeavingCandidates(). // - // Note that the perturbation is currenlty only reset to zero at the end of + // Note that the perturbation is currently only reset to zero at the end of // the algorithm. // // TODO(user): This is currently disabled because the improvement is unclear. diff --git a/ortools/glop/revised_simplex.h b/ortools/glop/revised_simplex.h index 5bff10683f..fff73670d7 100644 --- a/ortools/glop/revised_simplex.h +++ b/ortools/glop/revised_simplex.h @@ -595,7 +595,7 @@ class RevisedSimplex { // The compact version of matrix_with_slack_. CompactSparseMatrix compact_matrix_; - // The tranpose of compact_matrix_, it may be empty if it is not needed. + // The transpose of compact_matrix_, it may be empty if it is not needed. CompactSparseMatrix transposed_matrix_; // Stop the algorithm and report feasibility if: diff --git a/ortools/linear_solver/BUILD b/ortools/linear_solver/BUILD index 7ac54e0b3b..be550f423a 100644 --- a/ortools/linear_solver/BUILD +++ b/ortools/linear_solver/BUILD @@ -106,6 +106,7 @@ cc_library( "@com_google_absl//absl/synchronization", "@com_google_absl//absl/types:optional", "//ortools/base:status", + "//ortools/base:status_macros", "//ortools/base:stl_util", "@com_google_absl//absl/strings", "//ortools/base:timer", diff --git a/ortools/linear_solver/csharp/linear_solver.i b/ortools/linear_solver/csharp/linear_solver.i index 64d934e22c..1e1c36b531 100644 --- a/ortools/linear_solver/csharp/linear_solver.i +++ b/ortools/linear_solver/csharp/linear_solver.i @@ -38,6 +38,7 @@ %{ #include "ortools/linear_solver/linear_solver.h" #include "ortools/linear_solver/linear_solver.pb.h" +#include "ortools/linear_solver/model_exporter.h" %} @@ -147,17 +148,19 @@ VECTOR_AS_CSHARP_ARRAY(double, double, double, MpDoubleVector); %unignore operations_research::MPSolver::ExportModelAsMpsFormat(bool, bool); %extend operations_research::MPSolver { std::string ExportModelAsLpFormat(bool obfuscated) { - std::string output; - if (!$self->ExportModelAsLpFormat(obfuscated, &output)) return ""; - return output; + operations_research::MPModelExportOptions options; + options.obfuscate = obfuscated; + operations_research::MPModelProto model; + $self->ExportModelToProto(&model); + return ExportModelAsLpFormat(model, options).value_or(""); } std::string ExportModelAsMpsFormat(bool fixed_format, bool obfuscated) { - std::string output; - if (!$self->ExportModelAsMpsFormat(fixed_format, obfuscated, &output)) { - return ""; - } - return output; + operations_research::MPModelExportOptions options; + options.obfuscate = obfuscated; + operations_research::MPModelProto model; + $self->ExportModelToProto(&model); + return ExportModelAsMpsFormat(model, options).value_or(""); } } @@ -258,5 +261,6 @@ VECTOR_AS_CSHARP_ARRAY(double, double, double, MpDoubleVector); %unignore operations_research::MPSolverParameters::SCALING_ON; // no test %include "ortools/linear_solver/linear_solver.h" +%include "ortools/linear_solver/model_exporter.h" %unignoreall diff --git a/ortools/linear_solver/java/linear_solver.i b/ortools/linear_solver/java/linear_solver.i index edcd192f51..64b8ba80a8 100644 --- a/ortools/linear_solver/java/linear_solver.i +++ b/ortools/linear_solver/java/linear_solver.i @@ -50,6 +50,7 @@ typedef uint64_t uint64; %{ #include "ortools/linear_solver/linear_solver.h" +#include "ortools/linear_solver/model_exporter.h" %} %typemap(javaimports) SWIGTYPE %{ @@ -58,18 +59,20 @@ import java.lang.reflect.*; %extend operations_research::MPSolver { - std::string exportModelAsLpFormat(bool obfuscated) { - std::string output; - if (!$self->ExportModelAsLpFormat(obfuscated, &output)) return ""; - return output; + std::string exportModelAsLpFormat( + const operations_research::MPModelExportOptions& options = + operations_research::MPModelExportOptions()) { + operations_research::MPModelProto model; + $self->ExportModelToProto(&model); + return ExportModelAsLpFormat(model, options).value_or(""); } - std::string exportModelAsMpsFormat(bool fixed_format, bool obfuscated) { - std::string output; - if (!$self->ExportModelAsMpsFormat(fixed_format, obfuscated, &output)) { - return ""; - } - return output; + std::string exportModelAsMpsFormat( + const operations_research::MPModelExportOptions& options = + operations_research::MPModelExportOptions()) { + operations_research::MPModelProto model; + $self->ExportModelToProto(&model); + return ExportModelAsMpsFormat(model, options).value_or(""); } void setHint(const std::vector& variables, @@ -315,6 +318,17 @@ import java.lang.reflect.*; %unignore operations_research::MPSolverParameters::SCALING_OFF; // no test %unignore operations_research::MPSolverParameters::SCALING_ON; // no test +// Expose the model exporters. +%unignore operations_research::MPModelExportOptions; +%unignore operations_research::MPModelExportOptions::MPModelExportOptions; +%typemap(javaclassmodifiers) operations_research::MPModelExportOptions + "public final class"; +%rename (Obfuscate) operations_research::MPModelExportOptions::obfuscate; +%rename (LogInvalidNames) operations_research::MPModelExportOptions::log_invalid_names; +%rename (ShowUnusedVariables) operations_research::MPModelExportOptions::show_unused_variables; +%rename (MaxLineLength) operations_research::MPModelExportOptions::max_line_length; + %include "ortools/linear_solver/linear_solver.h" +%include "ortools/linear_solver/model_exporter.h" %unignoreall diff --git a/ortools/linear_solver/linear_solver.cc b/ortools/linear_solver/linear_solver.cc index 5da8ab374c..96311da3b9 100644 --- a/ortools/linear_solver/linear_solver.cc +++ b/ortools/linear_solver/linear_solver.cc @@ -32,7 +32,7 @@ #include "ortools/base/integral_types.h" #include "ortools/base/logging.h" #include "ortools/base/map_util.h" -//#include "ortools/base/status_macros.h" +#include "ortools/base/status_macros.h" #include "ortools/base/stl_util.h" #include "ortools/linear_solver/linear_solver.pb.h" #include "ortools/linear_solver/model_exporter.h" @@ -1368,16 +1368,29 @@ bool MPSolver::ExportModelAsLpFormat(bool obfuscate, std::string* model_str) const { MPModelProto proto; ExportModelToProto(&proto); - MPModelProtoExporter exporter(proto); - return exporter.ExportModelAsLpFormat(obfuscate, model_str); + MPModelExportOptions options; + options.obfuscate = obfuscate; + const auto status_or = + operations_research::ExportModelAsLpFormat(proto, options); + *model_str = status_or.value_or(""); + return status_or.ok(); } bool MPSolver::ExportModelAsMpsFormat(bool fixed_format, bool obfuscate, std::string* model_str) const { +// if (fixed_format) { +// LOG_EVERY_N_SEC(WARNING, 10) +// << "Fixed format is deprecated. Using free format instead."; +// + MPModelProto proto; ExportModelToProto(&proto); - MPModelProtoExporter exporter(proto); - return exporter.ExportModelAsMpsFormat(fixed_format, obfuscate, model_str); + MPModelExportOptions options; + options.obfuscate = obfuscate; + const auto status_or = + operations_research::ExportModelAsMpsFormat(proto, options); + *model_str = status_or.value_or(""); + return status_or.ok(); } void MPSolver::SetHint( diff --git a/ortools/linear_solver/model_exporter.cc b/ortools/linear_solver/model_exporter.cc index 644df20b96..9072a8719f 100644 --- a/ortools/linear_solver/model_exporter.cc +++ b/ortools/linear_solver/model_exporter.cc @@ -13,12 +13,15 @@ #include "ortools/linear_solver/model_exporter.h" +#include #include #include #include "absl/container/flat_hash_set.h" +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "ortools/base/canonical_errors.h" #include "ortools/base/commandlineflags.h" #include "ortools/base/integral_types.h" #include "ortools/base/logging.h" @@ -26,27 +29,162 @@ #include "ortools/linear_solver/linear_solver.pb.h" #include "ortools/util/fp_utils.h" -DEFINE_bool(lp_shows_unused_variables, false, - "Decides whether variable unused in the objective and constraints" - " are shown when exported to a file using the lp format."); - -DEFINE_int32(lp_max_line_length, 10000, - "Maximum line length in exported .lp files. The default was chosen" - " so that SCIP can read the files."); - -DEFINE_bool(lp_log_invalid_name, false, - "Whether to log invalid variable and contraint names."); +DEFINE_bool(lp_log_invalid_name, false, "DEPRECATED."); namespace operations_research { +namespace { -MPModelProtoExporter::MPModelProtoExporter(const MPModelProto& proto) - : proto_(proto), +constexpr double kInfinity = std::numeric_limits::infinity(); + +class MPModelProtoExporter { + public: + explicit MPModelProtoExporter(const MPModelProto& model); + bool ExportModelAsLpFormat(const MPModelExportOptions& options, + std::string* output); + bool ExportModelAsMpsFormat(const MPModelExportOptions& options, + std::string* output); + + private: + // Computes the number of continuous, integer and binary variables. + // Called by ExportModelAsLpFormat() and ExportModelAsMpsFormat(). + void Setup(); + + // Computes smart column widths for free MPS format. + void ComputeMpsSmartColumnWidths(bool obfuscated); + + // Processes all the proto.name() fields and returns the result in a vector. + // + // If 'obfuscate' is true, none of names are actually used, and this just + // returns a vector of 'prefix' + proto index (1-based). + // + // If it is false, this tries to keep the original names, but: + // - if the first character is forbidden, '_' is added at the beginning of + // name. + // - all the other forbidden characters are replaced by '_'. + // To avoid name conflicts, a '_' followed by an integer is appended to the + // result. + // + // If a name is longer than the maximum allowed name length, the obfuscated + // name is used. + // + // Therefore, a name "$20<=40" for proto #3 could be "_$20__40_1". + template + std::vector ExtractAndProcessNames( + const ListOfProtosWithNameFields& proto, const std::string& prefix, + bool obfuscate, bool log_invalid_names, + const std::string& forbidden_first_chars, + const std::string& forbidden_chars); + + // Appends a general "Comment" section with useful metadata about the model + // to "output". + // Note(user): there may be less variables in output than in the original + // model, as unused variables are not shown by default. Similarly, there + // may be more constraints in a .lp file as in the original model as + // a constraint lhs <= term <= rhs will be output as the two constraints + // term >= lhs and term <= rhs. + void AppendComments(const std::string& separator, std::string* output) const; + + // Clears "output" and writes a term to it, in "Lp" format. Returns false on + // error (for example, var_index is out of range). + bool WriteLpTerm(int var_index, double coefficient, + std::string* output) const; + + // Appends a pair name, value to "output", formatted to comply with the MPS + // standard. + void AppendMpsPair(const std::string& name, double value, + std::string* output) const; + + // Appends the head of a line, consisting of an id and a name to output. + void AppendMpsLineHeader(const std::string& id, const std::string& name, + std::string* output) const; + + // Same as AppendMpsLineHeader. Appends an extra new-line at the end the + // std::string pointed to by output. + void AppendMpsLineHeaderWithNewLine(const std::string& id, + const std::string& name, + std::string* output) const; + + // Appends an MPS term in various contexts. The term consists of a head name, + // a name, and a value. If the line is not empty, then only the pair + // (name, value) is appended. The number of columns, limited to 2 by the MPS + // format is also taken care of. + void AppendMpsTermWithContext(const std::string& head_name, + const std::string& name, double value, + std::string* output); + + // Appends a new-line if two columns are already present on the MPS line. + // Used by and in complement to AppendMpsTermWithContext. + void AppendNewLineIfTwoColumns(std::string* output); + + // When 'integrality' is true, appends columns corresponding to integer + // variables. Appends the columns for non-integer variables otherwise. + // The sparse matrix must be passed as a vector of columns ('transpose'). + void AppendMpsColumns( + bool integrality, + const std::vector>>& transpose, + std::string* output); + + // Appends a line describing the bound of a variablenew-line if two columns + // are already present on the MPS line. + // Used by and in complement to AppendMpsTermWithContext. + void AppendMpsBound(const std::string& bound_type, const std::string& name, + double value, std::string* output) const; + + const MPModelProto& proto_; + + // Vector of variable names as they will be exported. + std::vector exported_variable_names_; + + // Vector of constraint names as they will be exported. + std::vector exported_constraint_names_; + + // Number of integer variables in proto_. + int num_integer_variables_; + + // Number of binary variables in proto_. + int num_binary_variables_; + + // Number of continuous variables in proto_. + int num_continuous_variables_; + + // Current MPS file column number. + int current_mps_column_; + + // Format for MPS file lines. + std::unique_ptr> mps_header_format_; + std::unique_ptr> mps_format_; + + DISALLOW_COPY_AND_ASSIGN(MPModelProtoExporter); +}; +} // namespace + +util::StatusOr ExportModelAsLpFormat( + const MPModelProto& model, const MPModelExportOptions& options) { + MPModelProtoExporter exporter(model); + std::string output; + if (!exporter.ExportModelAsLpFormat(options, &output)) { + return util::InvalidArgumentError("Unable to export model."); + } + return output; +} + +util::StatusOr ExportModelAsMpsFormat( + const MPModelProto& model, const MPModelExportOptions& options) { + MPModelProtoExporter exporter(model); + std::string output; + if (!exporter.ExportModelAsMpsFormat(options, &output)) { + return util::InvalidArgumentError("Unable to export model."); + } + return output; +} + +namespace { +MPModelProtoExporter::MPModelProtoExporter(const MPModelProto& model) + : proto_(model), num_integer_variables_(0), num_binary_variables_(0), num_continuous_variables_(0), - current_mps_column_(0), - use_fixed_mps_format_(false), - use_obfuscated_names_(false) {} + current_mps_column_(0) {} namespace { class NameManager { @@ -74,18 +212,18 @@ std::string NameManager::MakeUniqueName(const std::string& name) { } std::string MakeExportableName(const std::string& name, + const std::string& forbidden_first_chars, + const std::string& forbidden_chars, bool* found_forbidden_char) { // Prepend with "_" all the names starting with a forbidden character. - const std::string kForbiddenFirstChars = "$.0123456789"; *found_forbidden_char = - kForbiddenFirstChars.find(name[0]) != std::string::npos; + forbidden_first_chars.find(name[0]) != std::string::npos; std::string exportable_name = *found_forbidden_char ? absl::StrCat("_", name) : name; // Replace all the other forbidden characters with "_". - const std::string kForbiddenChars = " +-*/<>=:\\"; for (char& c : exportable_name) { - if (kForbiddenChars.find(c) != std::string::npos) { + if (forbidden_chars.find(c) != std::string::npos) { c = '_'; *found_forbidden_char = true; } @@ -97,7 +235,9 @@ std::string MakeExportableName(const std::string& name, template std::vector MPModelProtoExporter::ExtractAndProcessNames( const ListOfProtosWithNameFields& proto, const std::string& prefix, - bool obfuscate) { + bool obfuscate, bool log_invalid_names, + const std::string& forbidden_first_chars, + const std::string& forbidden_chars) { const int num_items = proto.size(); std::vector result(num_items); NameManager namer; @@ -108,14 +248,15 @@ std::vector MPModelProtoExporter::ExtractAndProcessNames( absl::StrFormat("%s%0*d", prefix, num_digits, i); if (obfuscate || !item.has_name()) { result[i] = namer.MakeUniqueName(obfuscated_name); - LOG_IF(WARNING, FLAGS_lp_log_invalid_name && !item.has_name()) + LOG_IF(WARNING, log_invalid_names && !item.has_name()) << "Empty name detected, created new name: " << result[i]; } else { bool found_forbidden_char = false; const std::string exportable_name = - MakeExportableName(item.name(), &found_forbidden_char); + MakeExportableName(item.name(), forbidden_first_chars, + forbidden_chars, &found_forbidden_char); result[i] = namer.MakeUniqueName(exportable_name); - LOG_IF(WARNING, FLAGS_lp_log_invalid_name && found_forbidden_char) + LOG_IF(WARNING, log_invalid_names && found_forbidden_char) << "Invalid character detected in " << item.name() << ". Changed to " << result[i]; // If the name is too long, use the obfuscated name that is guaranteed @@ -128,15 +269,10 @@ std::vector MPModelProtoExporter::ExtractAndProcessNames( if (result[i].size() > kMaxNameLength - kMargin) { const std::string old_name = std::move(result[i]); result[i] = namer.MakeUniqueName(obfuscated_name); - LOG_IF(WARNING, FLAGS_lp_log_invalid_name) - << "Name is too long: " << old_name - << " exported as: " << result[i]; + LOG_IF(WARNING, log_invalid_names) << "Name is too long: " << old_name + << " exported as: " << result[i]; } } - // Update whether we can use the names in a fixed-format MPS file. - const int kFixedMpsFieldSize = 8; - use_fixed_mps_format_ = - use_fixed_mps_format_ && (result[i].size() <= kFixedMpsFieldSize); // Prepare for the next round. ++i; @@ -150,8 +286,7 @@ void MPModelProtoExporter::AppendComments(const std::string& separator, absl::StrAppendFormat(output, "%s Generated by MPModelProtoExporter\n", sep); absl::StrAppendFormat(output, "%s %-16s : %s\n", sep, "Name", proto_.has_name() ? proto_.name().c_str() : "NoName"); - absl::StrAppendFormat(output, "%s %-16s : %s\n", sep, "Format", - use_fixed_mps_format_ ? "Fixed" : "Free"); + absl::StrAppendFormat(output, "%s %-16s : %s\n", sep, "Format", "Free"); absl::StrAppendFormat(output, "%s %-16s : %d\n", sep, "Constraints", proto_.constraint_size()); absl::StrAppendFormat(output, "%s %-16s : %d\n", sep, "Variables", @@ -162,9 +297,6 @@ void MPModelProtoExporter::AppendComments(const std::string& separator, num_integer_variables_); absl::StrAppendFormat(output, "%s %-14s : %d\n", sep, "Continuous", num_continuous_variables_); - if (FLAGS_lp_shows_unused_variables) { - absl::StrAppendFormat(output, "%s Unused variables are shown\n", sep); - } } namespace { @@ -233,9 +365,21 @@ bool IsBoolean(const MPVariableProto& var) { return var.is_integer() && ceil(var.lower_bound()) == 0.0 && floor(var.upper_bound()) == 1.0; } + +void UpdateMaxSize(const std::string& new_string, int* size) { + if (new_string.size() > *size) *size = new_string.size(); +} + +void UpdateMaxSize(double new_number, int* size) { + UpdateMaxSize(DoubleToString(new_number), size); +} } // namespace void MPModelProtoExporter::Setup() { + if (FLAGS_lp_log_invalid_name) { + LOG(WARNING) << "The \"lp_log_invalid_name\" flag is deprecated. Use " + "MPModelProtoExportOptions instead."; + } num_binary_variables_ = 0; num_integer_variables_ = 0; for (const MPVariableProto& var : proto_.variable()) { @@ -251,28 +395,78 @@ void MPModelProtoExporter::Setup() { proto_.variable_size() - num_binary_variables_ - num_integer_variables_; } -bool MPModelProtoExporter::ExportModelAsLpFormat(bool obfuscated, - std::string* output) { +void MPModelProtoExporter::ComputeMpsSmartColumnWidths(bool obfuscated) { + // Minimum values for aesthetics (if columns are too narrow, MPS files are + // difficult to read). + int string_field_size = 6; + int number_field_size = 6; + + for (const MPVariableProto& var : proto_.variable()) { + UpdateMaxSize(var.name(), &string_field_size); + UpdateMaxSize(var.objective_coefficient(), &number_field_size); + UpdateMaxSize(var.lower_bound(), &number_field_size); + UpdateMaxSize(var.upper_bound(), &number_field_size); + } + + for (const MPConstraintProto& cst : proto_.constraint()) { + UpdateMaxSize(cst.name(), &string_field_size); + UpdateMaxSize(cst.lower_bound(), &number_field_size); + UpdateMaxSize(cst.upper_bound(), &number_field_size); + for (const double coeff : cst.coefficient()) { + UpdateMaxSize(coeff, &number_field_size); + } + } + + // Maximum values for aesthetics. These are also the values used by other + // solvers. + string_field_size = std::min(string_field_size, 255); + number_field_size = std::min(number_field_size, 255); + + // If the model is obfuscated, all names will have the same size, which we + // compute here. + if (obfuscated) { + int max_digits = + absl::StrCat( + std::max(proto_.variable_size(), proto_.constraint_size()) - 1) + .size(); + string_field_size = std::max(6, max_digits + 1); + } + + mps_header_format_ = absl::ParsedFormat<'s', 's'>::New( + absl::StrCat(" %-2s %-", string_field_size, "s")); + mps_format_ = absl::ParsedFormat<'s', 's'>::New( + absl::StrCat(" %-", string_field_size, "s %", number_field_size, "s")); +} + +bool MPModelProtoExporter::ExportModelAsLpFormat( + const MPModelExportOptions& options, std::string* output) { output->clear(); Setup(); - exported_constraint_names_ = - ExtractAndProcessNames(proto_.constraint(), "C", obfuscated); - exported_variable_names_ = - ExtractAndProcessNames(proto_.variable(), "V", obfuscated); + const std::string kForbiddenFirstChars = "$.0123456789"; + const std::string kForbiddenChars = " +-*/<>=:\\"; + exported_constraint_names_ = ExtractAndProcessNames( + proto_.constraint(), "C", options.obfuscate, options.log_invalid_names, + kForbiddenFirstChars, kForbiddenChars); + exported_variable_names_ = ExtractAndProcessNames( + proto_.variable(), "V", options.obfuscate, options.log_invalid_names, + kForbiddenFirstChars, kForbiddenChars); // Comments section. AppendComments("\\", output); + if (options.show_unused_variables) { + absl::StrAppendFormat(output, "\\ Unused variables are shown\n"); + } // Objective absl::StrAppend(output, proto_.maximize() ? "Maximize\n" : "Minimize\n"); - LineBreaker obj_line_breaker(FLAGS_lp_max_line_length); + LineBreaker obj_line_breaker(options.max_line_length); obj_line_breaker.Append(" Obj: "); if (proto_.objective_offset() != 0.0) { obj_line_breaker.Append(absl::StrCat( DoubleToStringWithForcedSign(proto_.objective_offset()), " Constant ")); } std::vector show_variable(proto_.variable_size(), - FLAGS_lp_shows_unused_variables); + options.show_unused_variables); for (int var_index = 0; var_index < proto_.variable_size(); ++var_index) { const double coeff = proto_.variable(var_index).objective_coefficient(); std::string term; @@ -280,14 +474,14 @@ bool MPModelProtoExporter::ExportModelAsLpFormat(bool obfuscated, return false; } obj_line_breaker.Append(term); - show_variable[var_index] = coeff != 0.0 || FLAGS_lp_shows_unused_variables; + show_variable[var_index] = coeff != 0.0 || options.show_unused_variables; } // Constraints absl::StrAppend(output, obj_line_breaker.GetOutput(), "\nSubject to\n"); for (int cst_index = 0; cst_index < proto_.constraint_size(); ++cst_index) { const MPConstraintProto& ct_proto = proto_.constraint(cst_index); const std::string& name = exported_constraint_names_[cst_index]; - LineBreaker line_breaker(FLAGS_lp_max_line_length); + LineBreaker line_breaker(options.max_line_length); const int kNumFormattingChars = 10; // Overevaluated. // Account for the size of the constraint name + possibly "_rhs" + // the formatting characters here. @@ -300,8 +494,7 @@ bool MPModelProtoExporter::ExportModelAsLpFormat(bool obfuscated, return false; } line_breaker.Append(term); - show_variable[var_index] = - coeff != 0.0 || FLAGS_lp_shows_unused_variables; + show_variable[var_index] = coeff != 0.0 || options.show_unused_variables; } const double lb = ct_proto.lower_bound(); const double ub = ct_proto.upper_bound(); @@ -309,9 +502,9 @@ bool MPModelProtoExporter::ExportModelAsLpFormat(bool obfuscated, line_breaker.Append(absl::StrCat(" = ", DoubleToString(ub), "\n")); absl::StrAppend(output, " ", name, ": ", line_breaker.GetOutput()); } else { - if (ub != +std::numeric_limits::infinity()) { + if (ub != +kInfinity) { std::string rhs_name = name; - if (lb != -std::numeric_limits::infinity()) { + if (lb != -kInfinity) { absl::StrAppend(&rhs_name, "_rhs"); } absl::StrAppend(output, " ", rhs_name, ": ", line_breaker.GetOutput()); @@ -322,9 +515,9 @@ bool MPModelProtoExporter::ExportModelAsLpFormat(bool obfuscated, if (!line_breaker.WillFit(relation)) absl::StrAppend(output, "\n "); absl::StrAppend(output, relation); } - if (lb != -std::numeric_limits::infinity()) { + if (lb != -kInfinity) { std::string lhs_name = name; - if (ub != +std::numeric_limits::infinity()) { + if (ub != +kInfinity) { absl::StrAppend(&lhs_name, "_lhs"); } absl::StrAppend(output, " ", lhs_name, ": ", line_breaker.GetOutput()); @@ -351,15 +544,14 @@ bool MPModelProtoExporter::ExportModelAsLpFormat(bool obfuscated, exported_variable_names_[var_index], ub); } else { absl::StrAppend(output, " "); - if (lb == -std::numeric_limits::infinity() && - ub == std::numeric_limits::infinity()) { + if (lb == -kInfinity && ub == kInfinity) { absl::StrAppend(output, exported_variable_names_[var_index], " free"); } else { - if (lb != -std::numeric_limits::infinity()) { + if (lb != -kInfinity) { absl::StrAppend(output, DoubleToString(lb), " <= "); } absl::StrAppend(output, exported_variable_names_[var_index]); - if (ub != std::numeric_limits::infinity()) { + if (ub != kInfinity) { absl::StrAppend(output, " <= ", DoubleToString(ub)); } } @@ -397,36 +589,19 @@ bool MPModelProtoExporter::ExportModelAsLpFormat(bool obfuscated, void MPModelProtoExporter::AppendMpsPair(const std::string& name, double value, std::string* output) const { - const int kFixedMpsDoubleWidth = 12; - if (use_fixed_mps_format_) { - int precision = kFixedMpsDoubleWidth; - std::string value_str = absl::StrFormat("%.*G", precision, value); - // Use the largest precision that can fit into the field witdh. - while (value_str.size() > kFixedMpsDoubleWidth) { - --precision; - value_str = absl::StrFormat("%.*g", precision, value); - } - absl::StrAppendFormat(output, " %-8s %*s ", name, kFixedMpsDoubleWidth, - value_str); - } else { - absl::StrAppendFormat(output, " %-16s %21s ", name, - DoubleToString(value)); - } + absl::StrAppendFormat(output, *mps_format_, name, DoubleToString(value)); } void MPModelProtoExporter::AppendMpsLineHeader(const std::string& id, const std::string& name, std::string* output) const { - if (use_fixed_mps_format_) { - absl::StrAppendFormat(output, " %-2s %-8s", id, name); - } else { - absl::StrAppendFormat(output, " %-2s %-16s", id, name); - } + absl::StrAppendFormat(output, *mps_header_format_, id, name); } void MPModelProtoExporter::AppendMpsLineHeaderWithNewLine( const std::string& id, const std::string& name, std::string* output) const { AppendMpsLineHeader(id, name, output); + absl::StripTrailingAsciiWhitespace(output); absl::StrAppend(output, "\n"); } @@ -445,12 +620,14 @@ void MPModelProtoExporter::AppendMpsBound(const std::string& bound_type, std::string* output) const { AppendMpsLineHeader(bound_type, "BOUND", output); AppendMpsPair(name, value, output); + absl::StripTrailingAsciiWhitespace(output); absl::StrAppend(output, "\n"); } void MPModelProtoExporter::AppendNewLineIfTwoColumns(std::string* output) { ++current_mps_column_; if (current_mps_column_ == 2) { + absl::StripTrailingAsciiWhitespace(output); absl::StrAppend(output, "\n"); current_mps_column_ = 0; } @@ -481,20 +658,20 @@ void MPModelProtoExporter::AppendMpsColumns( } } -bool MPModelProtoExporter::ExportModelAsMpsFormat(bool fixed_format, - bool obfuscated, - std::string* output) { +bool MPModelProtoExporter::ExportModelAsMpsFormat( + const MPModelExportOptions& options, std::string* output) { output->clear(); Setup(); - use_fixed_mps_format_ = fixed_format; - exported_constraint_names_ = - ExtractAndProcessNames(proto_.constraint(), "C", obfuscated); - exported_variable_names_ = - ExtractAndProcessNames(proto_.variable(), "V", obfuscated); + ComputeMpsSmartColumnWidths(options.obfuscate); + const std::string kForbiddenFirstChars = ""; + const std::string kForbiddenChars = " "; + exported_constraint_names_ = ExtractAndProcessNames( + proto_.constraint(), "C", options.obfuscate, options.log_invalid_names, + kForbiddenFirstChars, kForbiddenChars); + exported_variable_names_ = ExtractAndProcessNames( + proto_.variable(), "V", options.obfuscate, options.log_invalid_names, + kForbiddenFirstChars, kForbiddenChars); - // use_fixed_mps_format_ was possibly modified by ExtractAndProcessNames(). - LOG_IF(WARNING, fixed_format && !use_fixed_mps_format_) - << "Cannot use fixed format. Falling back to free format"; if (proto_.maximize()) { LOG(DFATAL) << "MPS cannot represent maximization objectives."; return false; @@ -514,13 +691,13 @@ bool MPModelProtoExporter::ExportModelAsMpsFormat(bool fixed_format, const double lb = ct_proto.lower_bound(); const double ub = ct_proto.upper_bound(); const std::string& cst_name = exported_constraint_names_[cst_index]; - if (lb == ub) { + if (lb == -kInfinity && ub == kInfinity) { + AppendMpsLineHeaderWithNewLine("N", cst_name, &rows_section); + } else if (lb == ub) { AppendMpsLineHeaderWithNewLine("E", cst_name, &rows_section); - } else if (lb == -std::numeric_limits::infinity()) { - DCHECK_NE(std::numeric_limits::infinity(), ub); + } else if (lb == -kInfinity) { AppendMpsLineHeaderWithNewLine("L", cst_name, &rows_section); } else { - DCHECK_NE(-std::numeric_limits::infinity(), lb); AppendMpsLineHeaderWithNewLine("G", cst_name, &rows_section); } } @@ -554,7 +731,7 @@ bool MPModelProtoExporter::ExportModelAsMpsFormat(bool fixed_format, std::string columns_section; AppendMpsColumns(/*integrality=*/true, transpose, &columns_section); if (!columns_section.empty()) { - constexpr const char kIntMarkerFormat[] = " %-10s%-36s%-10s\n"; + constexpr const char kIntMarkerFormat[] = " %-10s%-36s%-8s\n"; columns_section = absl::StrFormat(kIntMarkerFormat, "INTSTART", "'MARKER'", "'INTORG'") + columns_section; @@ -574,9 +751,9 @@ bool MPModelProtoExporter::ExportModelAsMpsFormat(bool fixed_format, const double lb = ct_proto.lower_bound(); const double ub = ct_proto.upper_bound(); const std::string& cst_name = exported_constraint_names_[cst_index]; - if (lb != -std::numeric_limits::infinity()) { + if (lb != -kInfinity) { AppendMpsTermWithContext("RHS", cst_name, lb, &rhs_section); - } else if (ub != +std::numeric_limits::infinity()) { + } else if (ub != +kInfinity) { AppendMpsTermWithContext("RHS", cst_name, ub, &rhs_section); } } @@ -591,7 +768,7 @@ bool MPModelProtoExporter::ExportModelAsMpsFormat(bool fixed_format, for (int cst_index = 0; cst_index < proto_.constraint_size(); ++cst_index) { const MPConstraintProto& ct_proto = proto_.constraint(cst_index); const double range = fabs(ct_proto.upper_bound() - ct_proto.lower_bound()); - if (range != 0.0 && range != +std::numeric_limits::infinity()) { + if (range != 0.0 && range != +kInfinity) { const std::string& cst_name = exported_constraint_names_[cst_index]; AppendMpsTermWithContext("RANGE", cst_name, range, &ranges_section); } @@ -609,33 +786,46 @@ bool MPModelProtoExporter::ExportModelAsMpsFormat(bool fixed_format, const double lb = var_proto.lower_bound(); const double ub = var_proto.upper_bound(); const std::string& var_name = exported_variable_names_[var_index]; + + if (lb == -kInfinity && ub == +kInfinity) { + AppendMpsLineHeader("FR", "BOUND", &bounds_section); + absl::StrAppendFormat(&bounds_section, " %s\n", var_name); + continue; + } + if (var_proto.is_integer()) { if (IsBoolean(var_proto)) { AppendMpsLineHeader("BV", "BOUND", &bounds_section); absl::StrAppendFormat(&bounds_section, " %s\n", var_name); } else { - if (lb != 0.0) { + if (lb == -kInfinity && ub > 0) { + // Non-standard MPS use seen on miplib2017/ns1456591 and adopted. + // "MI" (indicating [-inf, 0] bounds) is supposed to be used only for + // continuous variables, but solvers seem to read it as expected. + AppendMpsLineHeader("MI", "BOUND", &bounds_section); + absl::StrAppendFormat(&bounds_section, " %s\n", var_name); + } + // "LI" can be skipped if it's -inf, or if it's 0. + // There is one exception to that rule: if UI=+inf, we can't skip LI=0 + // or the variable will be parsed as binary. + if (lb != -kInfinity && (lb != 0.0 || ub == kInfinity)) { AppendMpsBound("LI", var_name, lb, &bounds_section); } - if (ub != +std::numeric_limits::infinity()) { + if (ub != kInfinity) { AppendMpsBound("UI", var_name, ub, &bounds_section); } } } else { - if (lb == -std::numeric_limits::infinity() && - ub == +std::numeric_limits::infinity()) { - AppendMpsLineHeader("FR", "BOUND", &bounds_section); - absl::StrAppendFormat(&bounds_section, " %s\n", var_name); - } else if (lb == ub) { + if (lb == ub) { AppendMpsBound("FX", var_name, lb, &bounds_section); } else { if (lb != 0.0) { AppendMpsBound("LO", var_name, lb, &bounds_section); - } else if (ub == +std::numeric_limits::infinity()) { + } else if (ub == +kInfinity) { AppendMpsLineHeader("PL", "BOUND", &bounds_section); absl::StrAppendFormat(&bounds_section, " %s\n", var_name); } - if (ub != +std::numeric_limits::infinity()) { + if (ub != +kInfinity) { AppendMpsBound("UP", var_name, ub, &bounds_section); } } @@ -649,4 +839,5 @@ bool MPModelProtoExporter::ExportModelAsMpsFormat(bool fixed_format, return true; } +} // namespace } // namespace operations_research diff --git a/ortools/linear_solver/model_exporter.h b/ortools/linear_solver/model_exporter.h index ec85b49856..02dd90b024 100644 --- a/ortools/linear_solver/model_exporter.h +++ b/ortools/linear_solver/model_exporter.h @@ -17,194 +17,80 @@ #include #include +#include "absl/strings/str_format.h" #include "ortools/base/hash.h" #include "ortools/base/macros.h" +#include "ortools/base/statusor.h" +#include "ortools/linear_solver/linear_solver.pb.h" namespace operations_research { -class MPConstraint; -class MPObjective; -class MPVariable; +struct MPModelExportOptions { + MPModelExportOptions() {} -class MPModelProto; + // Obfuscates variable and constraint names. + bool obfuscate = false; + // Whether to log invalid variable and constraint names. + bool log_invalid_names = false; -class MPModelProtoExporter { - public: - // The argument must live as long as this class is active. - explicit MPModelProtoExporter(const MPModelProto& proto); - - // Outputs the current model (variables, constraints, objective) as a - // std::string encoded in the so-called "CPLEX LP file format" as generated by - // SCIP. The LP file format is easily readable by a human. - // - // Returns false if some error has occurred during execution. - // The validity of names is automatically checked. If a variable name or a - // constraint name is invalid or non-existent, a new valid name is - // automatically generated. - // - // If 'obfuscated' is true, the variable and constraint names of proto_ - // are not used. Variable and constraint names of the form "V12345" - // and "C12345" are used instead. - // - // For more information about the different LP file formats: - // http://lpsolve.sourceforge.net/5.5/lp-format.htm - // The following give a reasonable idea of the CPLEX LP file format: - // http://lpsolve.sourceforge.net/5.5/CPLEX-format.htm - // http://tinyurl.com/cplex-lp-format - // http://www.gurobi.com/documentation/5.1/reference-manual/node871 - bool ExportModelAsLpFormat(bool obfuscated, std::string* output); - - // Outputs the current model (variables, constraints, objective) as a - // std::string encoded in MPS file format, using the "fixed" MPS format if - // possible, and the "free" MPS format otherwise. - // - // Returns false if some error has occurred during execution. Models with - // maximization objectives trigger an error, because MPS can encode only - // minimization problems. - // - // If fixed_format is true, the method tries to use the MPS fixed format (the - // use of which is discouraged as coefficients are printed with less - // precision). If it is not possible to use the fixed format, the method falls - // back to the so-called "free format". - // - // The validity of names is automatically checked. If a variable name or a - // constraint name is invalid or non-existent, a new valid name is - // automatically generated. - // - // Name validity and obfuscation works exactly as in ExportModelAsLpFormat(). - // - // For more information about the MPS format: - // http://en.wikipedia.org/wiki/MPS_(format) - // A close-to-original description coming from OSL: - // http://tinyurl.com/mps-format-by-osl - // A recent description from CPLEX: - // http://tinyurl.com/mps-format-by-cplex - // CPLEX extensions: - // http://tinyurl.com/mps-extensions-by-cplex - // Gurobi's description: - // http://www.gurobi.com/documentation/5.1/reference-manual/node869 - bool ExportModelAsMpsFormat(bool fixed_format, bool obfuscated, - std::string* output); - - private: - // Computes the number of continuous, integer and binary variables. - // Called by ExportModelAsLpFormat() and ExportModelAsMpsFormat(). - void Setup(); - - // Processes all the proto.name() fields and returns the result in a vector. - // - // If 'obfuscate' is true, none of names are actually used, and this just - // returns a vector of 'prefix' + proto index (1-based). - // - // If it is false, this tries to keep the original names, but: - // - if the first character is forbidden, '_' is added at the beginning of - // name. - // - all the other forbidden characters are replaced by '_'. - // To avoid name conflicts, a '_' followed by an integer is appended to the - // result. - // - // If a name is longer than the maximum allowed name length, the obfuscated - // name is used. - // - // This method also sets use_fixed_mps_format_ to false if one name is too - // long. - // - // Therefore, a name "$20<=40" for proto #3 could be "_$20__40_1". - template - std::vector ExtractAndProcessNames( - const ListOfProtosWithNameFields& proto, const std::string& prefix, - bool obfuscate); - - // Returns true when the fixed MPS format can be used. - // The fixed format is used when the variable and constraint names do not - // exceed 8 characters. In the case of an obfuscated file, this means that - // the maximum number of digits for constraints and variables is limited to 7. - bool CanUseFixedMpsFormat() const; - - // Appends a general "Comment" section with useful metadata about the model - // to "output". - // Note(user): there may be less variables in output than in the original - // model, as unused variables are not shown by default. Similarly, there - // may be more constraints in a .lp file as in the original model as - // a constraint lhs <= term <= rhs will be output as the two constraints - // term >= lhs and term <= rhs. - void AppendComments(const std::string& separator, std::string* output) const; - - // Clears "output" and writes a term to it, in "Lp" format. Returns false on - // error (for example, var_index is out of range). - bool WriteLpTerm(int var_index, double coefficient, - std::string* output) const; - - // Appends a pair name, value to "output", formatted to comply with the MPS - // standard. - void AppendMpsPair(const std::string& name, double value, - std::string* output) const; - - // Appends the head of a line, consisting of an id and a name to output. - void AppendMpsLineHeader(const std::string& id, const std::string& name, - std::string* output) const; - - // Same as AppendMpsLineHeader. Appends an extra new-line at the end the - // std::string pointed to by output. - void AppendMpsLineHeaderWithNewLine(const std::string& id, - const std::string& name, - std::string* output) const; - - // Appends an MPS term in various contexts. The term consists of a head name, - // a name, and a value. If the line is not empty, then only the pair - // (name, value) is appended. The number of columns, limited to 2 by the MPS - // format is also taken care of. - void AppendMpsTermWithContext(const std::string& head_name, - const std::string& name, double value, - std::string* output); - - // Appends a new-line if two columns are already present on the MPS line. - // Used by and in complement to AppendMpsTermWithContext. - void AppendNewLineIfTwoColumns(std::string* output); - - // When 'integrality' is true, appends columns corresponding to integer - // variables. Appends the columns for non-integer variables otherwise. - // The sparse matrix must be passed as a vector of columns ('transpose'). - void AppendMpsColumns( - bool integrality, - const std::vector>>& transpose, - std::string* output); - - // Appends a line describing the bound of a variablenew-line if two columns - // are already present on the MPS line. - // Used by and in complement to AppendMpsTermWithContext. - void AppendMpsBound(const std::string& bound_type, const std::string& name, - double value, std::string* output) const; - - const MPModelProto& proto_; - - // Vector of variable names as they will be exported. - std::vector exported_variable_names_; - - // Vector of constraint names as they will be exported. - std::vector exported_constraint_names_; - - // Number of integer variables in proto_. - int num_integer_variables_; - - // Number of binary variables in proto_. - int num_binary_variables_; - - // Number of continuous variables in proto_. - int num_continuous_variables_; - - // Current MPS file column number. - int current_mps_column_; - - // True is the fixed MPS format shall be used. - bool use_fixed_mps_format_; - - // True if the variable and constraint names will be obfuscated. - bool use_obfuscated_names_; - - DISALLOW_COPY_AND_ASSIGN(MPModelProtoExporter); + // For .lp files only. Decides whether variables unused in the objective and + // constraints are shown when exported to a file. + bool show_unused_variables = false; + // For .lp files only. Maximum line length in exported files. The default + // was chosen so that SCIP can read the files. + int max_line_length = 10000; }; +// Outputs the current model (variables, constraints, objective) as a +// std::string encoded in the so-called "CPLEX LP file format" as generated by +// SCIP. The LP file format is easily readable by a human. +// +// Returns false if some error has occurred during execution. +// The validity of names is automatically checked. If a variable name or a +// constraint name is invalid or non-existent, a new valid name is +// automatically generated. +// +// If 'obfuscated' is true, the variable and constraint names of proto_ +// are not used. Variable and constraint names of the form "V12345" +// and "C12345" are used instead. +// +// For more information about the different LP file formats: +// http://lpsolve.sourceforge.net/5.5/lp-format.htm +// The following give a reasonable idea of the CPLEX LP file format: +// http://lpsolve.sourceforge.net/5.5/CPLEX-format.htm +// http://tinyurl.com/cplex-lp-format +// http://www.gurobi.com/documentation/5.1/reference-manual/node871 +util::StatusOr ExportModelAsLpFormat( + const MPModelProto& model, + const MPModelExportOptions& options = MPModelExportOptions()); + +// Outputs the current model (variables, constraints, objective) as a +// std::string encoded in MPS file format, using the "free" MPS format. +// +// Returns false if some error has occurred during execution. Models with +// maximization objectives trigger an error, because MPS can encode only +// minimization problems. +// +// The validity of names is automatically checked. If a variable name or a +// constraint name is invalid or non-existent, a new valid name is +// automatically generated. +// +// Name validity and obfuscation works exactly as in ExportModelAsLpFormat(). +// +// For more information about the MPS format: +// http://en.wikipedia.org/wiki/MPS_(format) +// A close-to-original description coming from OSL: +// http://tinyurl.com/mps-format-by-osl +// A recent description from CPLEX: +// http://tinyurl.com/mps-format-by-cplex +// CPLEX extensions: +// http://tinyurl.com/mps-extensions-by-cplex +// Gurobi's description: +// http://www.gurobi.com/documentation/5.1/reference-manual/node869 +util::StatusOr ExportModelAsMpsFormat( + const MPModelProto& model, + const MPModelExportOptions& options = MPModelExportOptions()); + } // namespace operations_research #endif // OR_TOOLS_LINEAR_SOLVER_MODEL_EXPORTER_H_ diff --git a/ortools/linear_solver/model_exporter_swig_helper.h b/ortools/linear_solver/model_exporter_swig_helper.h new file mode 100644 index 0000000000..36353ce0e2 --- /dev/null +++ b/ortools/linear_solver/model_exporter_swig_helper.h @@ -0,0 +1,39 @@ +// Copyright 2010-2018 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_LINEAR_SOLVER_MODEL_EXPORTER_SWIG_HELPER_H_ +#define OR_TOOLS_LINEAR_SOLVER_MODEL_EXPORTER_SWIG_HELPER_H_ + +#include + +#include "ortools/linear_solver/linear_solver.pb.h" +#include "ortools/linear_solver/model_exporter.h" + +namespace operations_research { + +std::string ExportModelAsLpFormatReturnString( + const MPModelProto& input_model, + const MPModelExportOptions& options = MPModelExportOptions()) { + return operations_research::ExportModelAsLpFormat(input_model, options) + .value_or(""); +} + +std::string ExportModelAsMpsFormatReturnString( + const MPModelProto& input_model, + const MPModelExportOptions& options = MPModelExportOptions()) { + return operations_research::ExportModelAsMpsFormat(input_model, options) + .value_or(""); +} +} // namespace operations_research + +#endif // OR_TOOLS_LINEAR_SOLVER_MODEL_EXPORTER_SWIG_HELPER_H_ diff --git a/ortools/linear_solver/python/linear_solver.i b/ortools/linear_solver/python/linear_solver.i index 6d092fc047..82105e9207 100644 --- a/ortools/linear_solver/python/linear_solver.i +++ b/ortools/linear_solver/python/linear_solver.i @@ -46,6 +46,8 @@ class MPSolutionResponse; %{ #include "ortools/linear_solver/linear_solver.h" +#include "ortools/linear_solver/model_exporter.h" +#include "ortools/linear_solver/model_exporter_swig_helper.h" %} typedef int64_t int64; @@ -81,20 +83,6 @@ from ortools.linear_solver.linear_solver_natural_api import VariableExpr } %extend MPSolver { - // Change a (bool, std::string*) outputs to a python std::string (empty if bool=false). - std::string ExportModelAsLpFormat(bool obfuscated) { - std::string output; - if (!$self->ExportModelAsLpFormat(obfuscated, &output)) return ""; - return output; - } - std::string ExportModelAsMpsFormat(bool fixed_format, bool obfuscated) { - std::string output; - if (!$self->ExportModelAsMpsFormat(fixed_format, obfuscated, &output)) { - return ""; - } - return output; - } - // Change the API of LoadModelFromProto() to simply return the error message: // it will always be empty iff the model was valid. std::string LoadModelFromProto(const MPModelProto& input_model) { @@ -365,7 +353,15 @@ from ortools.linear_solver.linear_solver_natural_api import VariableExpr %unignore operations_research::MPSolverParameters::SCALING_OFF; %unignore operations_research::MPSolverParameters::SCALING_ON; +// Expose the model exporters. +%rename (ModelExportOptions) operations_research::MPModelExportOptions; +%rename (ModelExportOptions) operations_research::MPModelExportOptions::MPModelExportOptions; +%rename (ExportModelAsLpFormat) operations_research::ExportModelAsLpFormatReturnString; +%rename (ExportModelAsMpsFormat) operations_research::ExportModelAsMpsFormatReturnString; + %include "ortools/linear_solver/linear_solver.h" +%include "ortools/linear_solver/model_exporter.h" +%include "ortools/linear_solver/model_exporter_swig_helper.h" %unignoreall diff --git a/ortools/lp_data/model_reader.cc b/ortools/lp_data/model_reader.cc index 271f3d94dd..069f4ecdff 100644 --- a/ortools/lp_data/model_reader.cc +++ b/ortools/lp_data/model_reader.cc @@ -14,62 +14,14 @@ #include "ortools/lp_data/model_reader.h" #include "ortools/base/file.h" -#include "ortools/lp_data/mps_reader.h" +#include "ortools/linear_solver/linear_solver.pb.h" #include "ortools/lp_data/proto_utils.h" #include "ortools/util/file_util.h" namespace operations_research { namespace glop { -bool LoadLinearProgramFromMps(const std::string& input_file_path, - const std::string& forced_mps_format, - LinearProgram* linear_program) { - LinearProgram linear_program_fixed; - LinearProgram linear_program_free; - MPSReader mps_reader; - mps_reader.set_log_errors(forced_mps_format == "free" || - forced_mps_format == "fixed"); - bool fixed_read = forced_mps_format != "free" && - mps_reader.LoadFileWithMode(input_file_path, false, - &linear_program_fixed); - const bool free_read = - forced_mps_format != "fixed" && - mps_reader.LoadFileWithMode(input_file_path, true, &linear_program_free); - if (!fixed_read && !free_read) { - LOG(ERROR) << "Error while parsing the mps file '" << input_file_path - << "' Use the --forced_mps_format flags to see the errors."; - return false; - } - if (fixed_read && free_read) { - if (linear_program_fixed.name() != linear_program_free.name()) { - VLOG(1) << "Name of the model differs between fixed and free forms. " - << "Fallbacking to free form."; - fixed_read = false; - } - } - if (!fixed_read) { - VLOG(1) << "Read file in free format."; - linear_program->PopulateFromLinearProgram(linear_program_free); - } else { - VLOG(1) << "Read file in fixed format."; - linear_program->PopulateFromLinearProgram(linear_program_fixed); - if (free_read) { - // TODO(user): Dump() take ages on large program, so we need an efficient - // comparison function between two linear programs. Using - // GetProblemStats() for now. - if (linear_program_free.GetProblemStats() != - linear_program_fixed.GetProblemStats()) { - LOG(ERROR) << "Could not decide if '" << input_file_path - << "' is in fixed or free format."; - return false; - } - } - } - return true; -} - bool LoadLinearProgramFromModelOrRequest(const std::string& input_file_path, - LinearProgram* linear_program) { MPModelProto model_proto; MPModelRequest request_proto; diff --git a/ortools/lp_data/model_reader.h b/ortools/lp_data/model_reader.h index 9a7059336b..9f5d720826 100644 --- a/ortools/lp_data/model_reader.h +++ b/ortools/lp_data/model_reader.h @@ -14,17 +14,13 @@ #ifndef OR_TOOLS_LP_DATA_MODEL_READER_H_ #define OR_TOOLS_LP_DATA_MODEL_READER_H_ -#include "ortools/linear_solver/linear_solver.pb.h" +#include + #include "ortools/lp_data/lp_data.h" namespace operations_research { namespace glop { -// Helper function to read data from mps files into LinearProgram. -bool LoadLinearProgramFromMps(const std::string& input_file_path, - const std::string& forced_mps_format, - LinearProgram* linear_program); - // Helper function to read data from model files into LinearProgram. bool LoadLinearProgramFromModelOrRequest(const std::string& input_file_path, LinearProgram* linear_program); diff --git a/ortools/lp_data/mps_reader.cc b/ortools/lp_data/mps_reader.cc index c01913cbac..a964f7ebc0 100644 --- a/ortools/lp_data/mps_reader.cc +++ b/ortools/lp_data/mps_reader.cc @@ -13,26 +13,10 @@ #include "ortools/lp_data/mps_reader.h" -#include -#include -#include -#include -#include - #include "absl/strings/match.h" -#include "absl/strings/numbers.h" #include "absl/strings/str_split.h" -#include "ortools/base/commandlineflags.h" -#include "ortools/base/file.h" -#include "ortools/base/filelineiter.h" -#include "ortools/base/logging.h" -#include "ortools/base/map_util.h" // for FindOrNull, FindWithDefault #include "ortools/base/status.h" -#include "ortools/lp_data/lp_print_utils.h" - -ABSL_FLAG(bool, mps_free_form, false, "Read MPS files in free form."); -ABSL_FLAG(bool, mps_stop_after_first_error, true, - "Stop after the first error."); +//#include "ortools/base/status_builder.h" namespace operations_research { namespace glop { @@ -40,12 +24,11 @@ namespace glop { const int MPSReader::kNumFields = 6; const int MPSReader::kFieldStartPos[kNumFields] = {1, 4, 14, 24, 39, 49}; const int MPSReader::kFieldLength[kNumFields] = {2, 8, 8, 12, 8, 12}; +const int MPSReader::kSpacePos[12] = {12, 13, 22, 23, 36, 37, + 38, 47, 48, 61, 62, 63}; MPSReader::MPSReader() - : free_form_(absl::GetFlag(FLAGS_mps_free_form)), - data_(nullptr), - problem_name_(""), - parse_success_(true), + : free_form_(true), fields_(kNumFields), section_(UNKNOWN_SECTION), section_name_to_id_map_(), @@ -54,10 +37,8 @@ MPSReader::MPSReader() integer_type_names_set_(), line_num_(0), line_(), - has_lazy_constraints_(false), in_integer_section_(false), - num_unconstrained_rows_(0), - log_errors_(true) { + num_unconstrained_rows_(0) { section_name_to_id_map_["*"] = COMMENT; section_name_to_id_map_["NAME"] = NAME; section_name_to_id_map_["ROWS"] = ROWS; @@ -66,6 +47,7 @@ MPSReader::MPSReader() section_name_to_id_map_["RHS"] = RHS; section_name_to_id_map_["RANGES"] = RANGES; section_name_to_id_map_["BOUNDS"] = BOUNDS; + section_name_to_id_map_["INDICATORS"] = INDICATORS; section_name_to_id_map_["ENDATA"] = ENDATA; row_name_to_id_map_["E"] = EQUALITY; row_name_to_id_map_["L"] = LESS_THAN; @@ -87,10 +69,7 @@ MPSReader::MPSReader() void MPSReader::Reset() { fields_.resize(kNumFields); - parse_success_ = true; - problem_name_.clear(); line_num_ = 0; - has_lazy_constraints_ = false; in_integer_section_ = false; num_unconstrained_rows_ = 0; objective_name_.clear(); @@ -98,18 +77,35 @@ void MPSReader::Reset() { void MPSReader::DisplaySummary() { if (num_unconstrained_rows_ > 0) { - LOG(INFO) << "There are " << num_unconstrained_rows_ + 1 - << " unconstrained rows. The first of them (" << objective_name_ - << ") was used as the objective."; + VLOG(1) << "There are " << num_unconstrained_rows_ + 1 + << " unconstrained rows. The first of them (" << objective_name_ + << ") was used as the objective."; } } -void MPSReader::SplitLineIntoFields() { +bool MPSReader::IsFixedFormat() { + for (const int i : kSpacePos) { + if (i >= line_.length()) break; + if (line_[i] != ' ') return false; + } + return true; +} + +util::Status MPSReader::SplitLineIntoFields() { if (free_form_) { fields_ = absl::StrSplit(line_, absl::ByAnyChar(" \t"), absl::SkipEmpty()); - DCHECK_GE(kNumFields, fields_.size()); + if (fields_.size() > kNumFields) { + return InvalidArgumentError("Found too many fields."); + } } else { - int length = line_.length(); + // Note: the name should also comply with the fixed format guidelines + // (maximum 8 characters) but in practice there are many problem files in + // our netlib archive that are in fixed format and have a long name. We + // choose to ignore these cases and treat them as fixed format anyway. + if (section_ != NAME && !IsFixedFormat()) { + return InvalidArgumentError("Line is not in fixed format."); + } + const int length = line_.length(); for (int i = 0; i < kNumFields; ++i) { if (kFieldStartPos[i] < length) { fields_[i] = line_.substr(kFieldStartPos[i], kFieldLength[i]); @@ -119,6 +115,7 @@ void MPSReader::SplitLineIntoFields() { } } } + return util::OkStatus(); } std::string MPSReader::GetFirstWord() const { @@ -126,55 +123,10 @@ std::string MPSReader::GetFirstWord() const { return std::string(""); } const int first_space_pos = line_.find(' '); - std::string first_word = line_.substr(0, first_space_pos); + const std::string first_word = line_.substr(0, first_space_pos); return first_word; } -bool MPSReader::LoadFile(const std::string& file_name, LinearProgram* data) { - if (data == nullptr) { - LOG(ERROR) << "Serious programming error: NULL LinearProgram pointer " - << "passed as argument."; - return false; - } - Reset(); - data_ = data; - data_->Clear(); - for (const std::string& line : - FileLines(file_name, FileLineIterator::REMOVE_INLINE_CR)) { - ProcessLine(line); - } - data->CleanUp(); - DisplaySummary(); - return parse_success_; -} - -// TODO(user): Ideally have a method to compare instances of LinearProgram -// and have method which reads in both modes, compares the programs and checks -// that either both modes succeeded and led to the same program, or one mode -// failed or both modes failed (cf. what is done in linear_solver/solve.cc -// using protos). -bool MPSReader::LoadFileWithMode(const std::string& file_name, bool free_form, - LinearProgram* data) { - free_form_ = free_form; - if (LoadFile(file_name, data)) { - free_form_ = absl::GetFlag(FLAGS_mps_free_form); - return true; - } - free_form_ = absl::GetFlag(FLAGS_mps_free_form); - return false; -} - -bool MPSReader::LoadFileAndTryFreeFormOnFail(const std::string& file_name, - LinearProgram* data) { - if (!LoadFileWithMode(file_name, false, data)) { - LOG(INFO) << "Trying to read as an MPS free-format file."; - return LoadFileWithMode(file_name, true, data); - } - return true; -} - -std::string MPSReader::GetProblemName() const { return problem_name_; } - bool MPSReader::IsCommentOrBlank() const { const char* line = line_.c_str(); if (*line == '*') { @@ -188,389 +140,42 @@ bool MPSReader::IsCommentOrBlank() const { return true; } -void MPSReader::ProcessLine(const std::string& line) { - ++line_num_; - if (!parse_success_ && absl::GetFlag(FLAGS_mps_stop_after_first_error)) - return; - line_ = line; - if (IsCommentOrBlank()) { - return; // Skip blank lines and comments. - } - if (!free_form_ && line_.find('\t') != std::string::npos) { - if (log_errors_) { - LOG(ERROR) << "Line " << line_num_ << ": contains tab " - << "(Line contents: " << line_ << ")."; - } - parse_success_ = false; - } - std::string section; - if (line[0] != '\0' && line[0] != ' ') { - section = GetFirstWord(); - section_ = - gtl::FindWithDefault(section_name_to_id_map_, section, UNKNOWN_SECTION); - if (section_ == UNKNOWN_SECTION) { - if (log_errors_) { - LOG(ERROR) << "At line " << line_num_ - << ": Unknown section: " << section - << ". (Line contents: " << line_ << ")."; - } - parse_success_ = false; - return; - } - if (section_ == COMMENT) { - return; - } - if (section_ == NAME) { - SplitLineIntoFields(); - if (free_form_) { - if (fields_.size() >= 2) { - problem_name_ = fields_[1]; - } - } else { - if (fields_.size() >= 3) { - problem_name_ = fields_[2]; - } - } - // NOTE(user): The name may differ between fixed and free forms. In - // fixed form, the name has at most 8 characters, and starts at a specific - // position in the NAME line. For MIPLIB2010 problems (eg, air04, glass4), - // the name in fixed form ends up being preceded with a whitespace. - // TODO(user,user): Return an error for fixed form if the problem name - // does not fit. - data_->SetName(problem_name_); - } - return; - } - SplitLineIntoFields(); - switch (section_) { - case NAME: - if (log_errors_) { - LOG(ERROR) << "At line " << line_num_ << ": Second NAME field" - << ". (Line contents: " << line_ << ")."; - } - parse_success_ = false; - break; - case ROWS: - ProcessRowsSection(); - break; - case LAZYCONS: - if (!has_lazy_constraints_) { - LOG(WARNING) << "LAZYCONS section detected. It will be handled as an " - "extension of the ROWS section."; - has_lazy_constraints_ = true; - } - ProcessRowsSection(); - break; - case COLUMNS: - ProcessColumnsSection(); - break; - case RHS: - ProcessRhsSection(); - break; - case RANGES: - ProcessRangesSection(); - break; - case BOUNDS: - ProcessBoundsSection(); - break; - case SOS: - ProcessSosSection(); - break; - case ENDATA: // Do nothing. - break; - default: - if (log_errors_) { - LOG(ERROR) << "At line " << line_num_ - << ": Unknown section: " << section - << ". (Line contents: " << line_ << ")."; - } - parse_success_ = false; - break; - } -} - -double MPSReader::GetDoubleFromString(const std::string& param) { +util::StatusOr MPSReader::GetDoubleFromString(const std::string& str) { double result; - if (!absl::SimpleAtod(param, &result)) { - if (log_errors_) { - LOG(ERROR) << "At line " << line_num_ - << ": Failed to convert std::string to double. String = " - << param << ". (Line contents = '" << line_ << "')." - << " free_form_ = " << free_form_; - } - parse_success_ = false; + if (!absl::SimpleAtod(str, &result)) { + return InvalidArgumentError( + absl::StrCat("Failed to convert \"", str, "\" to double.")); + } + if (std::isnan(result)) { + return InvalidArgumentError("Found NaN value."); } return result; } -void MPSReader::ProcessRowsSection() { - std::string row_type_name = fields_[0]; - std::string row_name = fields_[1]; - MPSRowType row_type = gtl::FindWithDefault(row_name_to_id_map_, row_type_name, - UNKNOWN_ROW_TYPE); - if (row_type == UNKNOWN_ROW_TYPE) { - if (log_errors_) { - LOG(ERROR) << "At line " << line_num_ << ": Unknown row type " - << row_type_name << ". (Line contents = " << line_ << ")."; - } - parse_success_ = false; - return; +util::StatusOr MPSReader::GetBoolFromString(const std::string& str) { + int result; + if (!absl::SimpleAtoi(str, &result) || result < 0 || result > 1) { + return InvalidArgumentError( + absl::StrCat("Failed to convert \"", str, "\" to bool.")); } - - // The first NONE constraint is used as the objective. - if (objective_name_.empty() && row_type == NONE) { - row_type = OBJECTIVE; - objective_name_ = row_name; - } else { - if (row_type == NONE) { - ++num_unconstrained_rows_; - } - RowIndex row = data_->FindOrCreateConstraint(row_name); - - // The initial row range is [0, 0]. We encode the type in the range by - // setting one of the bound to +/- infinity. - switch (row_type) { - case LESS_THAN: - data_->SetConstraintBounds(row, -kInfinity, - data_->constraint_upper_bounds()[row]); - break; - case GREATER_THAN: - data_->SetConstraintBounds(row, data_->constraint_lower_bounds()[row], - kInfinity); - break; - case NONE: - data_->SetConstraintBounds(row, -kInfinity, kInfinity); - break; - case EQUALITY: - default: - break; - } + if (std::isnan(result)) { + return InvalidArgumentError("Found NaN value."); } + return result; } -void MPSReader::ProcessColumnsSection() { - // Take into account the INTORG and INTEND markers. - if (line_.find("'MARKER'") != std::string::npos) { - if (line_.find("'INTORG'") != std::string::npos) { - VLOG(2) << "Entering integer marker.\n" << line_; - CHECK(!in_integer_section_); - in_integer_section_ = true; - } else if (line_.find("'INTEND'") != std::string::npos) { - VLOG(2) << "Leaving integer marker.\n" << line_; - CHECK(in_integer_section_); - in_integer_section_ = false; - } - return; - } - const int start_index = free_form_ ? 0 : 1; - const std::string& column_name = GetField(start_index, 0); - const std::string& row1_name = GetField(start_index, 1); - const std::string& row1_value = GetField(start_index, 2); - const ColIndex col = data_->FindOrCreateVariable(column_name); - is_binary_by_default_.resize(col + 1, false); - if (in_integer_section_) { - data_->SetVariableType(col, LinearProgram::VariableType::INTEGER); - // The default bounds for integer variables are [0, 1]. - data_->SetVariableBounds(col, 0.0, 1.0); - is_binary_by_default_[col] = true; - } else { - data_->SetVariableBounds(col, 0.0, kInfinity); - } - StoreCoefficient(col, row1_name, row1_value); - if (fields_.size() - start_index >= 4) { - const std::string& row2_name = GetField(start_index, 3); - const std::string& row2_value = GetField(start_index, 4); - StoreCoefficient(col, row2_name, row2_value); - } +util::Status MPSReader::ProcessSosSection() { + return InvalidArgumentError("Section SOS currently not supported."); } -void MPSReader::ProcessRhsSection() { - const int start_index = free_form_ ? 0 : 2; - const int offset = start_index + GetFieldOffset(); - // const std::string& rhs_name = fields_[0]; is not used - const std::string& row1_name = GetField(offset, 0); - const std::string& row1_value = GetField(offset, 1); - StoreRightHandSide(row1_name, row1_value); - if (fields_.size() - start_index >= 4) { - const std::string& row2_name = GetField(offset, 2); - const std::string& row2_value = GetField(offset, 3); - StoreRightHandSide(row2_name, row2_value); - } +util::Status MPSReader::InvalidArgumentError(const std::string& error_message) { + return util::InvalidArgumentError(error_message); } -void MPSReader::ProcessRangesSection() { - const int start_index = free_form_ ? 0 : 2; - const int offset = start_index + GetFieldOffset(); - // const std::string& range_name = fields_[0]; is not used - const std::string& row1_name = GetField(offset, 0); - const std::string& row1_value = GetField(offset, 1); - StoreRange(row1_name, row1_value); - if (fields_.size() - start_index >= 4) { - const std::string& row2_name = GetField(offset, 2); - const std::string& row2_value = GetField(offset, 3); - StoreRange(row2_name, row2_value); - } -} - -void MPSReader::ProcessBoundsSection() { - std::string bound_type_mnemonic = fields_[0]; - std::string bound_row_name = fields_[1]; - std::string column_name = fields_[2]; - std::string bound_value; - if (fields_.size() >= 4) { - bound_value = fields_[3]; - } - StoreBound(bound_type_mnemonic, column_name, bound_value); -} - -void MPSReader::ProcessSosSection() { - LOG(ERROR) << "At line " << line_num_ - << "Section SOS currently not supported." - << ". (Line contents: " << line_ << ")."; - parse_success_ = false; -} - -void MPSReader::StoreCoefficient(ColIndex col, const std::string& row_name, - const std::string& row_value) { - if (row_name.empty() || row_name == "$") { - return; - } - const Fractional value(GetDoubleFromString(row_value)); - if (value == 0.0) return; - if (row_name == objective_name_) { - data_->SetObjectiveCoefficient(col, value); - } else { - const RowIndex row = data_->FindOrCreateConstraint(row_name); - data_->SetCoefficient(row, col, value); - } -} - -void MPSReader::StoreRightHandSide(const std::string& row_name, - const std::string& row_value) { - if (row_name.empty()) { - return; - } - if (row_name != objective_name_) { - RowIndex row = data_->FindOrCreateConstraint(row_name); - const Fractional value = GetDoubleFromString(row_value); - - // The row type is encoded in the bounds, so at this point we have either - // (-kInfinity, 0.0], [0.0, 0.0] or [0.0, kInfinity). We use the right - // hand side to change any finite bound. - const Fractional lower_bound = - (data_->constraint_lower_bounds()[row] == -kInfinity) ? -kInfinity - : value; - const Fractional upper_bound = - (data_->constraint_upper_bounds()[row] == kInfinity) ? kInfinity - : value; - data_->SetConstraintBounds(row, lower_bound, upper_bound); - } -} - -void MPSReader::StoreRange(const std::string& row_name, - const std::string& range_value) { - if (row_name.empty()) { - return; - } - const RowIndex row = data_->FindOrCreateConstraint(row_name); - const Fractional range(GetDoubleFromString(range_value)); - - Fractional lower_bound = data_->constraint_lower_bounds()[row]; - Fractional upper_bound = data_->constraint_upper_bounds()[row]; - if (lower_bound == upper_bound) { - if (range < 0.0) { - lower_bound += range; - } else { - upper_bound += range; - } - } - if (lower_bound == -kInfinity) { - lower_bound = upper_bound - fabs(range); - } - if (upper_bound == kInfinity) { - upper_bound = lower_bound + fabs(range); - } - data_->SetConstraintBounds(row, lower_bound, upper_bound); -} - -void MPSReader::StoreBound(const std::string& bound_type_mnemonic, - const std::string& column_name, - const std::string& bound_value) { - const BoundTypeId bound_type_id = gtl::FindWithDefault( - bound_name_to_id_map_, bound_type_mnemonic, UNKNOWN_BOUND_TYPE); - if (bound_type_id == UNKNOWN_BOUND_TYPE) { - parse_success_ = false; - if (log_errors_) { - LOG(ERROR) << "At line " << line_num_ << ": Unknown bound type " - << bound_type_mnemonic << ". (Line contents = " << line_ - << ")."; - } - return; - } - const ColIndex col = data_->FindOrCreateVariable(column_name); - if (integer_type_names_set_.count(bound_type_mnemonic) != 0) { - data_->SetVariableType(col, LinearProgram::VariableType::INTEGER); - } - // Resize the is_binary_by_default_ in case it is the first time this column - // is encountered. - is_binary_by_default_.resize(col + 1, false); - // Check that "binary by default" implies "integer". - DCHECK(!is_binary_by_default_[col] || data_->IsVariableInteger(col)); - Fractional lower_bound = data_->variable_lower_bounds()[col]; - Fractional upper_bound = data_->variable_upper_bounds()[col]; - // If a variable is binary by default, its status is reset if any bound - // is set on it. We take care to restore the default bounds for general - // integer variables. - if (is_binary_by_default_[col]) { - lower_bound = Fractional(0.0); - upper_bound = kInfinity; - } - switch (bound_type_id) { - case LOWER_BOUND: - lower_bound = Fractional(GetDoubleFromString(bound_value)); - // LI with the value 0.0 specifies general integers with no upper bound. - if (bound_type_mnemonic == "LI" && lower_bound == 0.0) { - upper_bound = kInfinity; - } - break; - case UPPER_BOUND: - upper_bound = Fractional(GetDoubleFromString(bound_value)); - break; - case FIXED_VARIABLE: { - const Fractional value(GetDoubleFromString(bound_value)); - lower_bound = value; - upper_bound = value; - break; - } - case FREE_VARIABLE: - lower_bound = -kInfinity; - upper_bound = +kInfinity; - break; - case NEGATIVE: - lower_bound = -kInfinity; - upper_bound = Fractional(0.0); - break; - case POSITIVE: - lower_bound = Fractional(0.0); - upper_bound = +kInfinity; - break; - case BINARY: - lower_bound = Fractional(0.0); - upper_bound = Fractional(1.0); - break; - case UNKNOWN_BOUND_TYPE: - default: - if (log_errors_) { - LOG(ERROR) << "At line " << line_num_ - << "Serious error: unknown bound type " << column_name << " " - << bound_type_mnemonic << " " << bound_value - << ". (Line contents: " << line_ << ")."; - } - parse_success_ = false; - } - is_binary_by_default_[col] = false; - data_->SetVariableBounds(col, lower_bound, upper_bound); -} +// util::Status MPSReader::AppendLineToError(const util::Status& status) { +// return util::StatusBuilder(status, GTL_LOC).SetAppend() +// << " Line " << line_num_ << ": \"" << line_ << "\"."; +// } } // namespace glop } // namespace operations_research diff --git a/ortools/lp_data/mps_reader.h b/ortools/lp_data/mps_reader.h index aa5aed2234..1cb88869b5 100644 --- a/ortools/lp_data/mps_reader.h +++ b/ortools/lp_data/mps_reader.h @@ -18,78 +18,73 @@ // MPS stands for Mathematical Programming System. // // The format was invented by IBM in the 60's, and has become the de facto -// standard. We developed this reader to be able to read benchmark data -// files. Using the MPS file format for new models is discouraged. +// standard. We developed this reader to be able to read benchmark data files. +// Using the MPS file format for new models is discouraged. #ifndef OR_TOOLS_LP_DATA_MPS_READER_H_ #define OR_TOOLS_LP_DATA_MPS_READER_H_ -#include // for max -#include // for std::string -#include // for vector -#include "absl/container/flat_hash_map.h" +#include +#include +#include +#include +#include +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "ortools/base/protobuf_util.h" +#include "ortools/base/canonical_errors.h" #include "ortools/base/commandlineflags.h" +#include "ortools/base/filelineiter.h" #include "ortools/base/hash.h" #include "ortools/base/int_type.h" #include "ortools/base/int_type_indexed_vector.h" -#include "ortools/base/macros.h" // for DISALLOW_COPY_AND_ASSIGN, NULL -#include "ortools/base/map_util.h" // for FindOrNull, FindWithDefault +#include "ortools/base/logging.h" +#include "ortools/base/macros.h" // for DISALLOW_COPY_AND_ASSIGN, NULL +#include "ortools/base/map_util.h" +#include "ortools/base/status.h" +#include "ortools/base/status_macros.h" +#include "ortools/base/statusor.h" +#include "ortools/linear_solver/linear_solver.pb.h" #include "ortools/lp_data/lp_data.h" #include "ortools/lp_data/lp_types.h" -ABSL_DECLARE_FLAG(bool, mps_free_form); -ABSL_DECLARE_FLAG(bool, mps_stop_after_first_error); - namespace operations_research { namespace glop { -// Different types of constraints for a given row. -typedef enum { - UNKNOWN_ROW_TYPE, - EQUALITY, - LESS_THAN, - GREATER_THAN, - OBJECTIVE, - NONE -} MPSRowType; - // Reads a linear program in the mps format. // -// All Load() methods clear the previously loaded instance and store the result -// in the given LinearProgram. They return false in case of failure to read the +// All Parse() methods clear the previously parsed instance and store the result +// in the given Data class. They return false in case of failure to read the // instance. class MPSReader { public: + enum Form { AUTO_DETECT, FREE, FIXED }; + MPSReader(); - // Loads instance from a file. - bool LoadFile(const std::string& file_name, LinearProgram* data); - - // Loads instance from a file, specifying if free or fixed format is used. - bool LoadFileWithMode(const std::string& file_name, bool free_form, - LinearProgram* data); - - // Same as load file, but try free form mps on fail. - bool LoadFileAndTryFreeFormOnFail(const std::string& file_name, - LinearProgram* data); - - // Returns the name of the last loaded problem as defined in the NAME line. - std::string GetProblemName() const; - - // See log_errors_ (the default is true). - void set_log_errors(bool v) { log_errors_ = v; } + // Parses instance from a file. We currently support LinearProgram and + // MpModelProto for the Data type, but it should be easy to add more. + template + util::Status ParseFile(const std::string& file_name, Data* data, + Form form = AUTO_DETECT); private: - // Number of fields in one line of MPS file + // Number of fields in one line of MPS file. static const int kNumFields; - // Starting positions of each of the fields. + // Starting positions of each of the fields for fixed format. static const int kFieldStartPos[]; - // Lengths of each of the fields. + // Lengths of each of the fields for fixed format. static const int kFieldLength[]; + // Positions where there should be spaces for fixed format. + static const int kSpacePos[]; + // Resets the object to its initial value before reading a new file. void Reset(); @@ -97,7 +92,10 @@ class MPSReader { void DisplaySummary(); // Get each field for a given line. - void SplitLineIntoFields(); + util::Status SplitLineIntoFields(); + + // Returns true if the line matches the fixed format. + bool IsFixedFormat(); // Get the first word in a line. std::string GetFirstWord() const; @@ -119,36 +117,44 @@ class MPSReader { int GetFieldOffset() const { return free_form_ ? fields_.size() & 1 : 0; } // Line processor. - void ProcessLine(const std::string& line); - - // Process section NAME in the MPS file. - void ProcessNameSection(); + template + util::Status ProcessLine(const std::string& line, DataWrapper* data); // Process section ROWS in the MPS file. - void ProcessRowsSection(); + template + util::Status ProcessRowsSection(bool is_lazy, DataWrapper* data); // Process section COLUMNS in the MPS file. - void ProcessColumnsSection(); + template + util::Status ProcessColumnsSection(DataWrapper* data); // Process section RHS in the MPS file. - void ProcessRhsSection(); + template + util::Status ProcessRhsSection(DataWrapper* data); // Process section RANGES in the MPS file. - void ProcessRangesSection(); + template + util::Status ProcessRangesSection(DataWrapper* data); // Process section BOUNDS in the MPS file. - void ProcessBoundsSection(); + template + util::Status ProcessBoundsSection(DataWrapper* data); + + // Process section INDICATORS in the MPS file. + template + util::Status ProcessIndicatorsSection(DataWrapper* data); // Process section SOS in the MPS file. - void ProcessSosSection(); + util::Status ProcessSosSection(); - // Safely converts a std::string to a double. Possibly sets parse_success_ to - // false if the std::string passed as parameter is ill-formed. - double GetDoubleFromString(const std::string& param); + // Safely converts a std::string to a numerical type. Returns an error if the + // std::string passed as parameter is ill-formed. + util::StatusOr GetDoubleFromString(const std::string& str); + util::StatusOr GetBoolFromString(const std::string& str); // Different types of variables, as defined in the MPS file specification. // Note these are more precise than the ones in PrimalSimplex. - typedef enum { + enum BoundTypeId { UNKNOWN_BOUND_TYPE, LOWER_BOUND, UPPER_BOUND, @@ -157,35 +163,52 @@ class MPSReader { NEGATIVE, POSITIVE, BINARY - } BoundTypeId; + }; + + // Different types of constraints for a given row. + enum RowTypeId { + UNKNOWN_ROW_TYPE, + EQUALITY, + LESS_THAN, + GREATER_THAN, + OBJECTIVE, + NONE + }; // Stores a bound value of a given type, for a given column name. - void StoreBound(const std::string& bound_type_mnemonic, - const std::string& column_name, - const std::string& bound_value); + template + util::Status StoreBound(const std::string& bound_type_mnemonic, + const std::string& column_name, + const std::string& bound_value, DataWrapper* data); // Stores a coefficient value for a column number and a row name. - void StoreCoefficient(ColIndex col, const std::string& row_name, - const std::string& row_value); + template + util::Status StoreCoefficient(int col, const std::string& row_name, + const std::string& row_value, + DataWrapper* data); // Stores a right-hand-side value for a row name. - void StoreRightHandSide(const std::string& row_name, - const std::string& row_value); + template + util::Status StoreRightHandSide(const std::string& row_name, + const std::string& row_value, + DataWrapper* data); // Stores a range constraint of value row_value for a row name. - void StoreRange(const std::string& row_name, const std::string& range_value); + template + util::Status StoreRange(const std::string& row_name, + const std::string& range_value, DataWrapper* data); + + // Returns an InvalidArgumentError with the given error message, postfixed by + // the current line of the .mps file (number and contents). + util::Status InvalidArgumentError(const std::string& error_message); + + // Appends the current line of the .mps file (number and contents) to the + // status if it's an error message. + // util::Status AppendLineToError(const util::Status& status); // Boolean set to true if the reader expects a free-form MPS file. bool free_form_; - LinearProgram* data_; - - // The name of the problem as defined on the NAME line in the MPS file. - std::string problem_name_; - - // True if the parsing was successful. - bool parse_success_; - // Storage of the fields for a line of the MPS file. std::vector fields_; @@ -203,6 +226,7 @@ class MPSReader { RHS, RANGES, BOUNDS, + INDICATORS, SOS, ENDATA } SectionId; @@ -214,7 +238,7 @@ class MPSReader { absl::flat_hash_map section_name_to_id_map_; // Maps row type mnemonic --> row type id. - absl::flat_hash_map row_name_to_id_map_; + absl::flat_hash_map row_name_to_id_map_; // Maps bound type mnemonic --> bound type id. absl::flat_hash_map bound_name_to_id_map_; @@ -230,10 +254,7 @@ class MPSReader { // A row of Booleans. is_binary_by_default_[col] is true if col // appeared within a scope started by INTORG and ended with INTEND markers. - DenseBooleanRow is_binary_by_default_; - - // True if the problem contains lazy constraints (LAZYCONS). - bool has_lazy_constraints_; + std::vector is_binary_by_default_; // True if the next variable has to be interpreted as an integer variable. // This is used to support the marker INTORG that starts an integer section @@ -245,14 +266,629 @@ class MPSReader { // be removed in the preprocessor). int num_unconstrained_rows_; - // Whether to log errors to LOG(ERROR) or not. Sometimes we just want to use - // this class to detect a valid .mps file, and encountering errors is - // expected. - bool log_errors_; - DISALLOW_COPY_AND_ASSIGN(MPSReader); }; +// Data templates. + +template +class DataWrapper {}; + +template <> +class DataWrapper { + public: + explicit DataWrapper(LinearProgram* data) { data_ = data; } + + void SetUp() { + data_->SetDcheckBounds(false); + data_->Clear(); + } + + void SetName(const std::string& name) { data_->SetName(name); } + + int FindOrCreateConstraint(const std::string& name) { + return data_->FindOrCreateConstraint(name).value(); + } + void SetConstraintBounds(int index, double lower_bound, double upper_bound) { + data_->SetConstraintBounds(RowIndex(index), lower_bound, upper_bound); + } + void SetConstraintCoefficient(int row_index, int col_index, + double coefficient) { + data_->SetCoefficient(RowIndex(row_index), ColIndex(col_index), + coefficient); + } + void SetIsLazy(int row_index) { + LOG_FIRST_N(WARNING, 1) + << "LAZYCONS section detected. It will be handled as an extension of " + "the ROWS section."; + } + double ConstraintLowerBound(int row_index) { + return data_->constraint_lower_bounds()[RowIndex(row_index)]; + } + double ConstraintUpperBound(int row_index) { + return data_->constraint_upper_bounds()[RowIndex(row_index)]; + } + + int FindOrCreateVariable(const std::string& name) { + return data_->FindOrCreateVariable(name).value(); + } + void SetVariableTypeToInteger(int index) { + data_->SetVariableType(ColIndex(index), + LinearProgram::VariableType::INTEGER); + } + void SetVariableBounds(int index, double lower_bound, double upper_bound) { + data_->SetVariableBounds(ColIndex(index), lower_bound, upper_bound); + } + void SetObjectiveCoefficient(int index, double coefficient) { + data_->SetObjectiveCoefficient(ColIndex(index), coefficient); + } + bool VariableIsInteger(int index) { + return data_->IsVariableInteger(ColIndex(index)); + } + double VariableLowerBound(int index) { + return data_->variable_lower_bounds()[ColIndex(index)]; + } + double VariableUpperBound(int index) { + return data_->variable_upper_bounds()[ColIndex(index)]; + } + + util::Status CreateIndicatorConstraint(std::string row_name, int col_index, + bool col_value) { + return util::UnimplementedError( + "LinearProgram does not support indicator constraints."); + } + + void CleanUp() { data_->CleanUp(); } + + private: + LinearProgram* data_; +}; + +template <> +class DataWrapper { + public: + explicit DataWrapper(MPModelProto* data) { data_ = data; } + + void SetUp() { data_->Clear(); } + + void SetName(const std::string& name) { data_->set_name(name); } + + int FindOrCreateConstraint(const std::string& name) { + const auto it = constraint_indices_by_name_.find(name); + if (it != constraint_indices_by_name_.end()) return it->second; + + const int index = data_->constraint_size(); + MPConstraintProto* const constraint = data_->add_constraint(); + constraint->set_lower_bound(0.0); + constraint->set_upper_bound(0.0); + constraint->set_name(name); + constraint_indices_by_name_[name] = index; + return index; + } + void SetConstraintBounds(int index, double lower_bound, double upper_bound) { + data_->mutable_constraint(index)->set_lower_bound(lower_bound); + data_->mutable_constraint(index)->set_upper_bound(upper_bound); + } + void SetConstraintCoefficient(int row_index, int col_index, + double coefficient) { + MPConstraintProto* const constraint = data_->mutable_constraint(row_index); + for (int i = 0; i < constraint->var_index_size(); ++i) { + if (constraint->var_index(i) == col_index) { + constraint->set_coefficient(i, coefficient); + return; + } + } + constraint->add_var_index(col_index); + constraint->add_coefficient(coefficient); + } + void SetIsLazy(int row_index) { + data_->mutable_constraint(row_index)->set_is_lazy(true); + } + double ConstraintLowerBound(int row_index) { + return data_->constraint(row_index).lower_bound(); + } + double ConstraintUpperBound(int row_index) { + return data_->constraint(row_index).upper_bound(); + } + + int FindOrCreateVariable(const std::string& name) { + const auto it = variable_indices_by_name_.find(name); + if (it != variable_indices_by_name_.end()) return it->second; + + const int index = data_->variable_size(); + MPVariableProto* const variable = data_->add_variable(); + variable->set_lower_bound(0.0); + variable->set_name(name); + variable_indices_by_name_[name] = index; + return index; + } + void SetVariableTypeToInteger(int index) { + data_->mutable_variable(index)->set_is_integer(true); + } + void SetVariableBounds(int index, double lower_bound, double upper_bound) { + data_->mutable_variable(index)->set_lower_bound(lower_bound); + data_->mutable_variable(index)->set_upper_bound(upper_bound); + } + void SetObjectiveCoefficient(int index, double coefficient) { + data_->mutable_variable(index)->set_objective_coefficient(coefficient); + } + bool VariableIsInteger(int index) { + return data_->variable(index).is_integer(); + } + double VariableLowerBound(int index) { + return data_->variable(index).lower_bound(); + } + double VariableUpperBound(int index) { + return data_->variable(index).upper_bound(); + } + + util::Status CreateIndicatorConstraint(std::string cst_name, int var_index, + bool var_value) { + const auto it = constraint_indices_by_name_.find(cst_name); + if (it == constraint_indices_by_name_.end()) { + return util::InvalidArgumentError( + absl::StrCat("Constraint \"", cst_name, "\" doesn't exist.")); + } + const int cst_index = it->second; + + MPGeneralConstraintProto* const constraint = + data_->add_general_constraint(); + constraint->set_name( + absl::StrCat("ind_", data_->constraint(cst_index).name())); + MPIndicatorConstraint* const indicator = + constraint->mutable_indicator_constraint(); + *indicator->mutable_constraint() = data_->constraint(cst_index); + indicator->set_var_index(var_index); + indicator->set_var_value(var_value); + constraints_to_delete_.insert(cst_index); + + return util::OkStatus(); + } + + void CleanUp() { + google::protobuf::util::RemoveAt(data_->mutable_constraint(), + constraints_to_delete_); + } + + private: + MPModelProto* data_; + + absl::flat_hash_map variable_indices_by_name_; + absl::flat_hash_map constraint_indices_by_name_; + std::set constraints_to_delete_; +}; + +template +util::Status MPSReader::ParseFile(const std::string& file_name, Data* data, + Form form) { + if (data == nullptr) { + return util::InvalidArgumentError("NULL pointer passed as argument."); + } + + if (form == AUTO_DETECT) { + if (ParseFile(file_name, data, FIXED).ok()) return util::OkStatus(); + return ParseFile(file_name, data, FREE); + } + + // TODO(user): Use the form directly. + free_form_ = form == FREE; + Reset(); + DataWrapper data_wrapper(data); + data_wrapper.SetUp(); + for (const std::string& line : + FileLines(file_name, FileLineIterator::REMOVE_INLINE_CR)) { + RETURN_IF_ERROR(ProcessLine(line, &data_wrapper)); + } + data_wrapper.CleanUp(); + DisplaySummary(); + return util::OkStatus(); +} + +template +util::Status MPSReader::ProcessLine(const std::string& line, + DataWrapper* data) { + ++line_num_; + line_ = line; + if (IsCommentOrBlank()) { + return util::OkStatus(); // Skip blank lines and comments. + } + if (!free_form_ && line_.find('\t') != std::string::npos) { + return InvalidArgumentError("File contains tabs."); + } + std::string section; + if (line[0] != '\0' && line[0] != ' ') { + section = GetFirstWord(); + section_ = + gtl::FindWithDefault(section_name_to_id_map_, section, UNKNOWN_SECTION); + if (section_ == UNKNOWN_SECTION) { + return InvalidArgumentError("Unknown section."); + } + if (section_ == COMMENT) { + return util::OkStatus(); + } + if (section_ == NAME) { + RETURN_IF_ERROR(SplitLineIntoFields()); + // NOTE(user): The name may differ between fixed and free forms. In + // fixed form, the name has at most 8 characters, and starts at a specific + // position in the NAME line. For MIPLIB2010 problems (eg, air04, glass4), + // the name in fixed form ends up being preceded with a whitespace. + // TODO(user,user): Return an error for fixed form if the problem name + // does not fit. + if (free_form_) { + if (fields_.size() >= 2) { + data->SetName(fields_[1]); + } + } else { + const std::vector free_fields = + absl::StrSplit(line_, absl::ByAnyChar(" \t"), absl::SkipEmpty()); + const std::string free_name = + free_fields.size() >= 2 ? free_fields[1] : ""; + const std::string fixed_name = fields_.size() >= 3 ? fields_[2] : ""; + if (free_name != fixed_name) { + return InvalidArgumentError( + "Fixed form invalid: name differs between free and fixed " + "forms."); + } + data->SetName(fixed_name); + } + } + return util::OkStatus(); + } + RETURN_IF_ERROR(SplitLineIntoFields()); + switch (section_) { + case NAME: + return InvalidArgumentError("Second NAME field."); + case ROWS: + return ProcessRowsSection(/*is_lazy=*/false, data); + case LAZYCONS: + return ProcessRowsSection(/*is_lazy=*/true, data); + case COLUMNS: + return ProcessColumnsSection(data); + case RHS: + return ProcessRhsSection(data); + case RANGES: + return ProcessRangesSection(data); + case BOUNDS: + return ProcessBoundsSection(data); + case INDICATORS: + return ProcessIndicatorsSection(data); + case SOS: + return ProcessSosSection(); + case ENDATA: // Do nothing. + break; + default: + return InvalidArgumentError("Unknown section."); + } + return util::OkStatus(); +} + +template +util::Status MPSReader::ProcessRowsSection(bool is_lazy, DataWrapper* data) { + if (fields_.size() < 2) { + return InvalidArgumentError("Not enough fields in ROWS section."); + } + const std::string row_type_name = fields_[0]; + const std::string row_name = fields_[1]; + RowTypeId row_type = gtl::FindWithDefault(row_name_to_id_map_, row_type_name, + UNKNOWN_ROW_TYPE); + if (row_type == UNKNOWN_ROW_TYPE) { + return InvalidArgumentError("Unknown row type."); + } + + // The first NONE constraint is used as the objective. + if (objective_name_.empty() && row_type == NONE) { + row_type = OBJECTIVE; + objective_name_ = row_name; + } else { + if (row_type == NONE) { + ++num_unconstrained_rows_; + } + const int row = data->FindOrCreateConstraint(row_name); + if (is_lazy) data->SetIsLazy(row); + + // The initial row range is [0, 0]. We encode the type in the range by + // setting one of the bounds to +/- infinity. + switch (row_type) { + case LESS_THAN: + data->SetConstraintBounds(row, -kInfinity, + data->ConstraintUpperBound(row)); + break; + case GREATER_THAN: + data->SetConstraintBounds(row, data->ConstraintLowerBound(row), + kInfinity); + break; + case NONE: + data->SetConstraintBounds(row, -kInfinity, kInfinity); + break; + case EQUALITY: + default: + break; + } + } + return util::OkStatus(); +} + +template +util::Status MPSReader::ProcessColumnsSection(DataWrapper* data) { + // Take into account the INTORG and INTEND markers. + if (line_.find("'MARKER'") != std::string::npos) { + if (line_.find("'INTORG'") != std::string::npos) { + VLOG(2) << "Entering integer marker.\n" << line_; + if (in_integer_section_) { + return InvalidArgumentError("Found INTORG inside the integer section."); + } + in_integer_section_ = true; + } else if (line_.find("'INTEND'") != std::string::npos) { + VLOG(2) << "Leaving integer marker.\n" << line_; + if (!in_integer_section_) { + return InvalidArgumentError( + "Found INTEND without corresponding INTORG."); + } + in_integer_section_ = false; + } + return util::OkStatus(); + } + const int start_index = free_form_ ? 0 : 1; + if (fields_.size() < start_index + 3) { + return InvalidArgumentError("Not enough fields in COLUMNS section."); + } + const std::string& column_name = GetField(start_index, 0); + const std::string& row1_name = GetField(start_index, 1); + const std::string& row1_value = GetField(start_index, 2); + const int col = data->FindOrCreateVariable(column_name); + is_binary_by_default_.resize(col + 1, false); + if (in_integer_section_) { + data->SetVariableTypeToInteger(col); + // The default bounds for integer variables are [0, 1]. + data->SetVariableBounds(col, 0.0, 1.0); + is_binary_by_default_[col] = true; + } else { + data->SetVariableBounds(col, 0.0, kInfinity); + } + RETURN_IF_ERROR(StoreCoefficient(col, row1_name, row1_value, data)); + if (fields_.size() == start_index + 4) { + return InvalidArgumentError("Unexpected number of fields."); + } + if (fields_.size() - start_index > 4) { + const std::string& row2_name = GetField(start_index, 3); + const std::string& row2_value = GetField(start_index, 4); + RETURN_IF_ERROR(StoreCoefficient(col, row2_name, row2_value, data)); + } + return util::OkStatus(); +} + +template +util::Status MPSReader::ProcessRhsSection(DataWrapper* data) { + const int start_index = free_form_ ? 0 : 2; + const int offset = start_index + GetFieldOffset(); + if (fields_.size() < offset + 2) { + return InvalidArgumentError("Not enough fields in RHS section."); + } + // const std::string& rhs_name = fields_[0]; is not used + const std::string& row1_name = GetField(offset, 0); + const std::string& row1_value = GetField(offset, 1); + RETURN_IF_ERROR(StoreRightHandSide(row1_name, row1_value, data)); + if (fields_.size() - start_index >= 4) { + const std::string& row2_name = GetField(offset, 2); + const std::string& row2_value = GetField(offset, 3); + RETURN_IF_ERROR(StoreRightHandSide(row2_name, row2_value, data)); + } + return util::OkStatus(); +} + +template +util::Status MPSReader::ProcessRangesSection(DataWrapper* data) { + const int start_index = free_form_ ? 0 : 2; + const int offset = start_index + GetFieldOffset(); + if (fields_.size() < offset + 2) { + return InvalidArgumentError("Not enough fields in RHS section."); + } + // const std::string& range_name = fields_[0]; is not used + const std::string& row1_name = GetField(offset, 0); + const std::string& row1_value = GetField(offset, 1); + RETURN_IF_ERROR(StoreRange(row1_name, row1_value, data)); + if (fields_.size() - start_index >= 4) { + const std::string& row2_name = GetField(offset, 2); + const std::string& row2_value = GetField(offset, 3); + RETURN_IF_ERROR(StoreRange(row2_name, row2_value, data)); + } + return util::OkStatus(); +} + +template +util::Status MPSReader::ProcessBoundsSection(DataWrapper* data) { + if (fields_.size() < 3) { + return InvalidArgumentError("Not enough fields in BOUNDS section."); + } + const std::string bound_type_mnemonic = fields_[0]; + const std::string bound_row_name = fields_[1]; + const std::string column_name = fields_[2]; + std::string bound_value; + if (fields_.size() >= 4) { + bound_value = fields_[3]; + } + return StoreBound(bound_type_mnemonic, column_name, bound_value, data); +} + +template +util::Status MPSReader::ProcessIndicatorsSection(DataWrapper* data) { + // TODO(user): Enforce section order. This section must come after + // anything related to constraints, or we'll have partial data inside the + // indicator constraints. + if (fields_.size() < 4) { + return InvalidArgumentError("Not enough fields in INDICATORS section."); + } + + const std::string type = fields_[0]; + if (type != "IF") { + return InvalidArgumentError( + "Indicator constraints must start with \"IF\"."); + } + const std::string row_name = fields_[1]; + const std::string column_name = fields_[2]; + const std::string column_value = fields_[3]; + + bool value; + ASSIGN_OR_RETURN(value, GetBoolFromString(column_value)); + + const int col = data->FindOrCreateVariable(column_name); + // Variables used in indicator constraints become Boolean by default. + data->SetVariableTypeToInteger(col); + data->SetVariableBounds(col, std::max(0.0, data->VariableLowerBound(col)), + std::min(1.0, data->VariableUpperBound(col))); + + RETURN_IF_ERROR(data->CreateIndicatorConstraint(row_name, col, value)); + + // RETURN_IF_ERROR( + // AppendLineToError(data->CreateIndicatorConstraint(row_name, col, value))); + return util::OkStatus(); +} + +template +util::Status MPSReader::StoreCoefficient(int col, const std::string& row_name, + const std::string& row_value, + DataWrapper* data) { + if (row_name.empty() || row_name == "$") { + return util::OkStatus(); + } + double value; + ASSIGN_OR_RETURN(value, GetDoubleFromString(row_value)); + if (value == kInfinity || value == -kInfinity) { + return InvalidArgumentError("Constraint coefficients cannot be infinity."); + } + if (value == 0.0) return util::OkStatus(); + if (row_name == objective_name_) { + data->SetObjectiveCoefficient(col, value); + } else { + const int row = data->FindOrCreateConstraint(row_name); + data->SetConstraintCoefficient(row, col, value); + } + return util::OkStatus(); +} + +template +util::Status MPSReader::StoreRightHandSide(const std::string& row_name, + const std::string& row_value, + DataWrapper* data) { + if (row_name.empty()) return util::OkStatus(); + + if (row_name != objective_name_) { + const int row = data->FindOrCreateConstraint(row_name); + Fractional value; + ASSIGN_OR_RETURN(value, GetDoubleFromString(row_value)); + + // The row type is encoded in the bounds, so at this point we have either + // (-kInfinity, 0.0], [0.0, 0.0] or [0.0, kInfinity). We use the right + // hand side to change any finite bound. + const Fractional lower_bound = + (data->ConstraintLowerBound(row) == -kInfinity) ? -kInfinity : value; + const Fractional upper_bound = + (data->ConstraintUpperBound(row) == kInfinity) ? kInfinity : value; + data->SetConstraintBounds(row, lower_bound, upper_bound); + } + return util::OkStatus(); +} + +template +util::Status MPSReader::StoreRange(const std::string& row_name, + const std::string& range_value, + DataWrapper* data) { + if (row_name.empty()) return util::OkStatus(); + + const int row = data->FindOrCreateConstraint(row_name); + Fractional range; + ASSIGN_OR_RETURN(range, GetDoubleFromString(range_value)); + + Fractional lower_bound = data->ConstraintLowerBound(row); + Fractional upper_bound = data->ConstraintUpperBound(row); + if (lower_bound == upper_bound) { + if (range < 0.0) { + lower_bound += range; + } else { + upper_bound += range; + } + } + if (lower_bound == -kInfinity) { + lower_bound = upper_bound - fabs(range); + } + if (upper_bound == kInfinity) { + upper_bound = lower_bound + fabs(range); + } + data->SetConstraintBounds(row, lower_bound, upper_bound); + return util::OkStatus(); +} + +template +util::Status MPSReader::StoreBound(const std::string& bound_type_mnemonic, + const std::string& column_name, + const std::string& bound_value, + DataWrapper* data) { + const BoundTypeId bound_type_id = gtl::FindWithDefault( + bound_name_to_id_map_, bound_type_mnemonic, UNKNOWN_BOUND_TYPE); + if (bound_type_id == UNKNOWN_BOUND_TYPE) { + return InvalidArgumentError("Unknown bound type."); + } + const int col = data->FindOrCreateVariable(column_name); + if (integer_type_names_set_.count(bound_type_mnemonic) != 0) { + data->SetVariableTypeToInteger(col); + } + // Resize the is_binary_by_default_ in case it is the first time this column + // is encountered. + is_binary_by_default_.resize(col + 1, false); + // Check that "binary by default" implies "integer". + DCHECK(!is_binary_by_default_[col] || data->VariableIsInteger(col)); + Fractional lower_bound = data->VariableLowerBound(col); + Fractional upper_bound = data->VariableUpperBound(col); + // If a variable is binary by default, its status is reset if any bound + // is set on it. We take care to restore the default bounds for general + // integer variables. + if (is_binary_by_default_[col]) { + lower_bound = Fractional(0.0); + upper_bound = kInfinity; + } + switch (bound_type_id) { + case LOWER_BOUND: { + ASSIGN_OR_RETURN(lower_bound, GetDoubleFromString(bound_value)); + // LI with the value 0.0 specifies general integers with no upper bound. + if (bound_type_mnemonic == "LI" && lower_bound == 0.0) { + upper_bound = kInfinity; + } + break; + } + case UPPER_BOUND: { + ASSIGN_OR_RETURN(upper_bound, GetDoubleFromString(bound_value)); + break; + } + case FIXED_VARIABLE: { + ASSIGN_OR_RETURN(lower_bound, GetDoubleFromString(bound_value)); + upper_bound = lower_bound; + break; + } + case FREE_VARIABLE: + lower_bound = -kInfinity; + upper_bound = +kInfinity; + break; + case NEGATIVE: + lower_bound = -kInfinity; + upper_bound = Fractional(0.0); + break; + case POSITIVE: + lower_bound = Fractional(0.0); + upper_bound = +kInfinity; + break; + case BINARY: + lower_bound = Fractional(0.0); + upper_bound = Fractional(1.0); + break; + case UNKNOWN_BOUND_TYPE: + default: + return InvalidArgumentError("Unknown bound type."); + } + is_binary_by_default_[col] = false; + data->SetVariableBounds(col, lower_bound, upper_bound); + return util::OkStatus(); +} + } // namespace glop } // namespace operations_research diff --git a/ortools/lp_data/sparse.cc b/ortools/lp_data/sparse.cc index 8d1581896e..7b573e3de3 100644 --- a/ortools/lp_data/sparse.cc +++ b/ortools/lp_data/sparse.cc @@ -1071,7 +1071,7 @@ void TriangularMatrix::PermutedComputeRowsToConsider( // A few notes: // - By construction, if the matrix can be permuted into a lower triangular // form, there is no cycle. This code does nothing to test for cycles, but - // there is a DCHECK() to detect them during debuging. + // there is a DCHECK() to detect them during debugging. // - This version uses sentinels (kInvalidRow) on nodes_to_explore_ to know // when a node has been explored (i.e. when the recursive dfs goes back in // the call stack). This is faster than an alternate implementation that diff --git a/ortools/lp_data/sparse.h b/ortools/lp_data/sparse.h index 1df42881ba..101fa068c8 100644 --- a/ortools/lp_data/sparse.h +++ b/ortools/lp_data/sparse.h @@ -285,7 +285,7 @@ class CompactSparseMatrix { public: CompactSparseMatrix() {} - // Convenient constructor for tests. + // Convenient constructors for tests. // TODO(user): If this is needed in production code, it can be done faster. explicit CompactSparseMatrix(const SparseMatrix& matrix) { PopulateFromMatrixView(MatrixView(matrix)); @@ -370,7 +370,11 @@ class CompactSparseMatrix { Fractional EntryCoefficient(EntryIndex i) const { return coefficients_[i.value()]; } + Fractional GetFirstCoefficient() const { + return EntryCoefficient(EntryIndex(0)); + } RowIndex EntryRow(EntryIndex i) const { return rows_[i.value()]; } + RowIndex GetFirstRow() const { return EntryRow(EntryIndex(0)); } private: const EntryIndex num_entries_; @@ -381,7 +385,7 @@ class CompactSparseMatrix { ColumnView column(ColIndex col) const { DCHECK_LT(col, num_cols_); - // Note that the start may be equals to row.size() if the last columns + // Note that the start may be equal to row.size() if the last columns // are empty, it is why we don't use &row[start]. const EntryIndex start = starts_[col]; return ColumnView(starts_[col + 1] - start, rows_.data() + start.value(), diff --git a/ortools/lp_data/sparse_column.h b/ortools/lp_data/sparse_column.h index 45f74e4335..8532617970 100644 --- a/ortools/lp_data/sparse_column.h +++ b/ortools/lp_data/sparse_column.h @@ -23,7 +23,7 @@ namespace glop { const RowIndex kNonPivotal(-1); // Specialization of SparseVectorEntry and SparseColumnIterator for the -// SparseColumn class. In addtion to index(), it also provides row() for better +// SparseColumn class. In addition to index(), it also provides row() for better // readability on the client side. class SparseColumnEntry : public SparseVectorEntry { public: diff --git a/ortools/sat/lp_utils.cc b/ortools/sat/lp_utils.cc index 0245a81238..7b39098efa 100644 --- a/ortools/sat/lp_utils.cc +++ b/ortools/sat/lp_utils.cc @@ -80,6 +80,14 @@ std::vector ScaleContinuousVariables(double scaling, for (MPConstraintProto& mp_constraint : *mp_model->mutable_constraint()) { ScaleConstraint(var_scaling, &mp_constraint); } + for (MPGeneralConstraintProto& general_constraint : + *mp_model->mutable_general_constraint()) { + CHECK_EQ(general_constraint.general_constraint_case(), + MPGeneralConstraintProto::kIndicatorConstraint); + ScaleConstraint(var_scaling, + general_constraint.mutable_indicator_constraint() + ->mutable_constraint()); + } return var_scaling; } @@ -302,6 +310,22 @@ bool ConvertMPModelProtoToCpModelProto(const SatParameters& params, for (const MPConstraintProto& mp_constraint : mp_model.constraint()) { scaler.AddConstraint(mp_model, mp_constraint, cp_model); } + for (const MPGeneralConstraintProto& general_constraint : + mp_model.general_constraint()) { + CHECK_EQ(general_constraint.general_constraint_case(), + MPGeneralConstraintProto::kIndicatorConstraint); + const MPIndicatorConstraint indicator_constraint = + general_constraint.indicator_constraint(); + const MPConstraintProto& mp_constraint = indicator_constraint.constraint(); + ConstraintProto* ct = + scaler.AddConstraint(mp_model, mp_constraint, cp_model); + if (ct == nullptr) continue; + + // Add the indicator. + const int var = indicator_constraint.var_index(); + const int value = indicator_constraint.var_value(); + ct->add_enforcement_literal(value == 1 ? var : NegatedRef(var)); + } double max_relative_coeff_error = scaler.max_relative_coeff_error; double max_sum_error = scaler.max_sum_error; diff --git a/ortools/util/BUILD b/ortools/util/BUILD index 329b41460a..80ca66aba5 100644 --- a/ortools/util/BUILD +++ b/ortools/util/BUILD @@ -217,6 +217,8 @@ cc_library( "//ortools/base:file", "//ortools/base:hash", "//ortools/base:recordio", + "//ortools/base:statusor", + "//ortools/base:status_macros", # "//net/proto2/io/public", # "//net/proto2/io/public:io", # "//net/proto2/public", diff --git a/ortools/util/file_util.cc b/ortools/util/file_util.cc index 2ea976e549..428d353e03 100644 --- a/ortools/util/file_util.cc +++ b/ortools/util/file_util.cc @@ -19,11 +19,19 @@ #include "google/protobuf/text_format.h" #include "ortools/base/file.h" #include "ortools/base/logging.h" +#include "ortools/base/status_macros.h" namespace operations_research { using ::google::protobuf::TextFormat; +util::StatusOr ReadFileToString(absl::string_view filename) { + std::string contents; + RETURN_IF_ERROR(file::GetContents(filename, &contents, file::Defaults())); + // Note that gzipped files are currently not supported. + return contents; +} + bool ReadFileToProto(absl::string_view filename, google::protobuf::Message* proto) { std::string data; diff --git a/ortools/util/file_util.h b/ortools/util/file_util.h index 795b8372f2..c456411006 100644 --- a/ortools/util/file_util.h +++ b/ortools/util/file_util.h @@ -21,9 +21,13 @@ #include "google/protobuf/message.h" #include "ortools/base/file.h" #include "ortools/base/recordio.h" +#include "ortools/base/statusor.h" namespace operations_research { +// Reads a file, optionally gzipped, to a std::string. +util::StatusOr ReadFileToString(absl::string_view filename); + // Reads a proto from a file. Supports the following formats: binary, text, // JSON, all of those optionally gzipped. Returns false on failure. bool ReadFileToProto(absl::string_view filename,