Bugfixes and testing of linear templates

pyomo/common/errors.py

@@ -118,11 +118,14 @@ class ApplicationError(Exception):
 
 
 class PyomoException(Exception):
-    """
-    Exception class for other Pyomo exceptions to inherit from,
-    allowing Pyomo exceptions to be caught in a general way
-    (e.g., in other applications that use Pyomo).
-    Subclasses can define a class-level `default_message` attribute.
+    """General base class for Pyomo exceptions.
+
+    This allows Pyomo exceptions to be caught in a general way (e.g., in
+    other applications that use Pyomo).
+
+    Subclasses can define a class-level `default_message` attribute that
+    is used if no other message/arguments are provided.
+
     """
 
     def __init__(self, *args):
@@ -159,10 +162,37 @@ def __str__(self):
 
 
 class InfeasibleConstraintException(PyomoException):
+    """Exception raised by Pyomo transformations or solver interfaces
+    to indicate that an infeasible constraint has been identified
+    (e.g. in the course of range reduction).
+
+    """
+
+
+class InvalidExpressionError(PyomoException, ValueError):
+    """Exception raised for expressions that cannot be represented or emitted.
+
+    Pyomo will raise this exception when:
+
+      - Compiling an expression that cannot be expressed / written in the
+        target format or interface.
+
+    See also the derived :class:`InvalidConstraintError`.
     """
-    Exception class used by Pyomo transformations to indicate
-    that an infeasible constraint has been identified (e.g. in
-    the course of range reduction).
+
+
+class InvalidConstraintError(InvalidExpressionError):
+    """Exception raised for constraints that cannot be represented or emitted.
+
+    Pyomo will raise this exception when:
+
+      - Creating a constraint with a trivial (Boolean) expression.
+      - Creating a constraint from an incorrectly structured tuple.
+      - Compiling a ranged constraint (``lb <= body <= ub``) where
+        either ``lb`` or ``ub`` are variable expressions.
+      - Compiling a constraint that cannot be expressed / written in the
+        target format or interface.
+
     """
 
 

pyomo/common/tests/test_timing.py

@@ -96,6 +96,7 @@ def test_report_timing(self):
            (0(\.\d+)?) seconds to construct Var x; 2 indices total
            (0(\.\d+)?) seconds to construct Var y; 0 indices total
            (0(\.\d+)?) seconds to construct Suffix Suffix
+           (0(\.\d+)?) seconds to construct SetOf 'y._data'
            (0(\.\d+)?) seconds to apply Transformation RelaxIntegerVars \(in-place\)
            """.strip()
 
@@ -145,6 +146,7 @@ def test_report_timing_context_manager(self):
            (0(\.\d+)?) seconds to construct Var x; 2 indices total
            (0(\.\d+)?) seconds to construct Var y; 0 indices total
            (0(\.\d+)?) seconds to construct Suffix Suffix
+           (0(\.\d+)?) seconds to construct SetOf 'y._data'
            (0(\.\d+)?) seconds to apply Transformation RelaxIntegerVars \(in-place\)
            """.strip()
 

pyomo/contrib/incidence_analysis/tests/test_scc_solver.py

@@ -150,7 +150,7 @@ def test_dynamic_backward_disc_with_initial_conditions(self):
                     # At t1, "input var" height[t0] is fixed, so
                     # it is not included here.
                     self.assertEqual(len(inputs), len(other_var_set))
-                    for var in block.input_vars[:]:
+                    for var in block.input_vars.values():
                         self.assertIn(var, other_var_set)
                         self.assertTrue(var.fixed)
 
@@ -214,7 +214,7 @@ def test_dynamic_backward_disc_without_initial_conditions(self):
                 # At t1, "input var" height[t0] is fixed, so
                 # it is not included here.
                 self.assertEqual(len(inputs), len(other_var_set))
-                for var in block.input_vars[:]:
+                for var in block.input_vars.values():
                     self.assertIn(var, other_var_set)
                     self.assertTrue(var.fixed)
 

pyomo/core/base/constraint.py

@@ -15,7 +15,11 @@
 from typing import Union, Type
 
 from pyomo.common.deprecation import RenamedClass, deprecated
-from pyomo.common.errors import DeveloperError, TemplateExpressionError
+from pyomo.common.errors import (
+    DeveloperError,
+    InvalidConstraintError,
+    TemplateExpressionError,
+)
 from pyomo.common.formatting import tabular_writer
 from pyomo.common.log import is_debug_set
 from pyomo.common.modeling import NOTSET
@@ -216,19 +220,19 @@ def to_bounded_expression(self, evaluate_bounds=False):
                 and lb.is_potentially_variable()
                 and not lb.is_fixed()
             ):
-                raise ValueError(
+                raise InvalidConstraintError(
                     f"Constraint '{self.name}' is a Ranged Inequality with a "
-                    "variable lower bound.  Cannot normalize the "
+                    "variable lower bound.  Cannot standardize the "
                     "constraint or send it to a solver."
                 )
             if (
                 ub.__class__ not in native_types
                 and ub.is_potentially_variable()
                 and not ub.is_fixed()
             ):
-                raise ValueError(
+                raise InvalidConstraintError(
                     f"Constraint '{self.name}' is a Ranged Inequality with a "
-                    "variable upper bound.  Cannot normalize the "
+                    "variable upper bound.  Cannot standardize the "
                     "constraint or send it to a solver."
                 )
         elif expr is None:
@@ -411,7 +415,7 @@ def set_value(self, expr):
                     or isinstance(arg, NumericValue)
                 ):
                     continue
-                raise ValueError(
+                raise InvalidConstraintError(
                     "Constraint '%s' does not have a proper value. "
                     "Constraint expressions expressed as tuples must "
                     "contain native numeric types or Pyomo NumericValue "
@@ -423,7 +427,7 @@ def set_value(self, expr):
                 # Form equality expression
                 #
                 if expr[0] is None or expr[1] is None:
-                    raise ValueError(
+                    raise InvalidConstraintError(
                         "Constraint '%s' does not have a proper value. "
                         "Equality Constraints expressed as 2-tuples "
                         "cannot contain None [received %s]" % (self.name, expr)
@@ -442,7 +446,7 @@ def set_value(self, expr):
                     self._expr = RangedExpression(expr, False)
                 return
             else:
-                raise ValueError(
+                raise InvalidConstraintError(
                     "Constraint '%s' does not have a proper value. "
                     "Found a tuple of length %d. Expecting a tuple of "
                     "length 2 or 3:\n"
@@ -461,7 +465,7 @@ def set_value(self, expr):
             raise ValueError(_rule_returned_none_error % (self.name,))
 
         elif expr.__class__ is bool:
-            raise ValueError(
+            raise InvalidConstraintError(
                 "Invalid constraint expression. The constraint "
                 "expression resolved to a trivial Boolean (%s) "
                 "instead of a Pyomo object. Please modify your "

pyomo/core/base/indexed_component.py

@@ -460,13 +460,16 @@ def keys(self, sort=SortComponents.UNSORTED, ordered=NOTSET):
             # of the underlying Set, there should be no warning if the
             # user iterates over the set when the _data dict is empty.
             #
-            if (
-                SortComponents.SORTED_INDICES in sort
-                or SortComponents.ORDERED_INDICES in sort
-            ):
-                return iter(sorted_robust(self._data))
+            # We will leverage SetOf here so we will cleanly pick up the
+            # iter overrides when we are templatizing
+            #
+            tmp_set = BASE.set.FiniteSetOf(self._data, name=self.name + "._data")
+            if SortComponents.SORTED_INDICES in sort:
+                return tmp_set.sorted_iter()
+            elif SortComponents.ORDERED_INDICES in sort:
+                return tmp_set.ordered_iter()
             else:
-                return self._data.__iter__()
+                return iter(tmp_set)
 
         if SortComponents.SORTED_INDICES in sort:
             ans = self._index_set.sorted_iter()
@@ -570,7 +573,9 @@ def values(self, sort=SortComponents.UNSORTED, ordered=NOTSET):
         def getter(s):
             try:
                 return _getdata(s)
-            except KeyError:
+            except (KeyError, TypeError):
+                # KeyError for missing indices, TypeError for unhashable
+                # keys (e.g., a template)
                 return self[s]
 
         return map(getter, self.keys(sort))

pyomo/core/base/param.py

@@ -8,6 +8,7 @@
 # ____________________________________________________________________________________
 
 from __future__ import annotations
+import collections
 import sys
 import types
 import logging
@@ -508,18 +509,26 @@ def extract_values(self):
             # The parameter is mutable so parameter data are ParamData
             # types.  We need to evaluate the ParamData back to POD
             # (numeric) data when creating the result.
-            return {key: param_data() for key, param_data in self.items()}
+            ans = {key: param_data() for key, param_data in self.items()}
         elif not self.is_indexed():
             # The scalar could be defined (in which case items() will
             # return the ScalarParam), OR it could be defined by a
             # default value (in which case items() will return the
             # actual numeric value).  To cover both cases we will use
             # value():
-            return {key: expr_value(param_data) for key, param_data in self.items()}
+            ans = {key: expr_value(param_data) for key, param_data in self.items()}
         else:
             # The parameter is not mutable, so iteritems() can be
             # converted into a dictionary containing parameter values.
-            return dict(self.items())
+            ans = dict(self.items())
+
+        # We need to fill-in the "missing" values with the declared default
+        #
+        # TBD [11/2025]: should we declare __missing__ so we can still
+        # validate the index for any missing values?
+        if self._default_val is not Param.NoValue and not self._index_set.isfinite():
+            ans = collections.defaultdict(lambda: self._default_val, ans)
+        return ans
 
     def extract_values_sparse(self):
         """
@@ -540,11 +549,20 @@ def extract_values_sparse(self):
             # case it will still be evaluatable by calling it).
             # ScalarParams whose value comes from the default are not
             # returned by sparse_items()
-            return {key: param_data() for key, param_data in self.sparse_items()}
+            ans = {key: param_data() for key, param_data in self.sparse_items()}
         else:
             # The parameter is not mutable, so sparse_items() can be
             # converted into a dictionary containing parameter values.
-            return dict(self.sparse_items())
+            #
+            ans = dict(self.sparse_iteritems())
+
+        # We need to fill-in the "missing" values with the declared default
+        #
+        # TBD [11/2025]: should we declare __missing__ so we can still
+        # validate the index for any missing values?
+        if self._default_val is not Param.NoValue:
+            ans = collections.defaultdict(lambda: self._default_val, ans)
+        return ans
 
     def store_values(self, new_values, check=True):
         """

pyomo/core/base/set.py

@@ -2632,15 +2632,17 @@ def construct(self, data=None):
     def dimen(self):
         if isinstance(self._ref, SetData):
             return self._ref.dimen
-        _iter = iter(self)
+        _iter = iter(self._ref)
         try:
             x = next(_iter)
             if type(x) is tuple:
                 ans = len(x)
             else:
                 ans = 1
-        except:
-            return 0
+        except StopIteration:
+            # The referenced object is empty, so we can't infer / verify
+            # the dimensionality.
+            return UnknownSetDimen
         for x in _iter:
             _this = len(x) if type(x) is tuple else 1
             if _this != ans: