diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c92b5150..49291fade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ it releases that version ([RELEASING.md](https://github.com/energy-models/mathsp ## Upcoming version +- feat(language)!: a construct that reads the order of a dimension needs the dimension declared ordered ([#793](https://github.com/energy-models/mathspec/pull/793)) - docs(pypsa): the pypsa reference targets master at 51986084, where the eight rungs that diverged now match ([#844](https://github.com/energy-models/mathspec/pull/844)) - feat(language)!: a call names the columns of a relation as relation[column], and a sum through a relation is a sum over the axes its join opens ([#664](https://github.com/energy-models/mathspec/pull/664)) - docs(pypsa): the linearized unit commitment is a patch over pypsa.yaml, and relaxes a committable link and process too ([#835](https://github.com/energy-models/mathspec/pull/835)) diff --git a/docs/about/limits.md b/docs/about/limits.md index b06ea5ab4..fe608ec33 100644 --- a/docs/about/limits.md +++ b/docs/about/limits.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: CC-BY-4.0 A spec can only say what the language has words for. This page says which words can be added, and which cannot. Read it before you ask for a new operator, block or keyword. For the rules a spec itself has to obey, read -[the ten rules](../reference/language/index.md#the-ten-rules). +[the eleven rules](../reference/language/index.md#the-eleven-rules). ## How a new construct enters diff --git a/docs/examples/commitment.md b/docs/examples/commitment.md index 43ccc2c6a..eb880a622 100644 --- a/docs/examples/commitment.md +++ b/docs/examples/commitment.md @@ -22,7 +22,7 @@ description: >- inequality is written once. dimensions: - snapshot: { dtype: int, description: dispatch periods } + snapshot: { dtype: int, description: dispatch periods, ordered: true } generator: { description: generating units } parameters: diff --git a/docs/examples/operators.md b/docs/examples/operators.md index 834701c01..d20977cd2 100644 --- a/docs/examples/operators.md +++ b/docs/examples/operators.md @@ -287,7 +287,7 @@ description: >- it would have fed is not built. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } variables: p: @@ -314,7 +314,7 @@ description: >- reads the last and nothing is vacated. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } variables: p: @@ -341,7 +341,7 @@ description: >- number instead of being absent, so the row survives. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } variables: p: @@ -370,7 +370,7 @@ description: >- dimensions: technology: { dtype: str } - month: { dtype: int } + month: { dtype: int, ordered: true } parameters: lead: { dims: [technology], dtype: int } @@ -401,7 +401,7 @@ description: >- snapshot reads that season's last and no level crosses the boundary. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } season: { dtype: str } relations: @@ -433,7 +433,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } parameters: min_up: { dims: [unit], dtype: int } @@ -467,7 +467,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } parameters: min_up: { dims: [unit], dtype: int } @@ -501,7 +501,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } parameters: min_up: { dims: [unit], dtype: int } @@ -536,7 +536,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } day: { dtype: str } relations: diff --git a/docs/examples/pypsa/carrier.md b/docs/examples/pypsa/carrier.md index 6467b4b7d..4f03b7117 100644 --- a/docs/examples/pypsa/carrier.md +++ b/docs/examples/pypsa/carrier.md @@ -13,6 +13,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/generator.md b/docs/examples/pypsa/generator.md index 85078cfb7..f2a670330 100644 --- a/docs/examples/pypsa/generator.md +++ b/docs/examples/pypsa/generator.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes generator: @@ -24,6 +25,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/generator_commitment.md b/docs/examples/pypsa/generator_commitment.md index e92c8d3d8..aa3ab50c3 100644 --- a/docs/examples/pypsa/generator_commitment.md +++ b/docs/examples/pypsa/generator_commitment.md @@ -15,11 +15,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true generator: description: generating units, each on one bus period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/examples/pypsa/generator_maintenance.md b/docs/examples/pypsa/generator_maintenance.md index 9831fede3..09f731f64 100644 --- a/docs/examples/pypsa/generator_maintenance.md +++ b/docs/examples/pypsa/generator_maintenance.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true generator: description: generating units, each on one bus diff --git a/docs/examples/pypsa/generator_ramping.md b/docs/examples/pypsa/generator_ramping.md index f07435fb4..695390eec 100644 --- a/docs/examples/pypsa/generator_ramping.md +++ b/docs/examples/pypsa/generator_ramping.md @@ -15,11 +15,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true generator: description: generating units, each on one bus period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/examples/pypsa/line.md b/docs/examples/pypsa/line.md index 9fa44bbc0..345ff802d 100644 --- a/docs/examples/pypsa/line.md +++ b/docs/examples/pypsa/line.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes line: @@ -36,6 +37,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/link.md b/docs/examples/pypsa/link.md index fa18e6cfd..c41449a5d 100644 --- a/docs/examples/pypsa/link.md +++ b/docs/examples/pypsa/link.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes link: @@ -29,6 +30,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/link_commitment.md b/docs/examples/pypsa/link_commitment.md index d7f5bd61f..9884826e6 100644 --- a/docs/examples/pypsa/link_commitment.md +++ b/docs/examples/pypsa/link_commitment.md @@ -15,11 +15,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true link: description: controllable connections, each from one bus to the buses it delivers to period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/examples/pypsa/link_maintenance.md b/docs/examples/pypsa/link_maintenance.md index 6ccfc997d..07a283420 100644 --- a/docs/examples/pypsa/link_maintenance.md +++ b/docs/examples/pypsa/link_maintenance.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true link: description: controllable connections, each from one bus to the buses it delivers to diff --git a/docs/examples/pypsa/link_ramping.md b/docs/examples/pypsa/link_ramping.md index a7efc2381..3105ca16d 100644 --- a/docs/examples/pypsa/link_ramping.md +++ b/docs/examples/pypsa/link_ramping.md @@ -15,11 +15,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true link: description: controllable connections, each from one bus to the buses it delivers to period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/examples/pypsa/load.md b/docs/examples/pypsa/load.md index 77d187405..53582b938 100644 --- a/docs/examples/pypsa/load.md +++ b/docs/examples/pypsa/load.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes load: diff --git a/docs/examples/pypsa/network.md b/docs/examples/pypsa/network.md index e5dbb1420..b52002a64 100644 --- a/docs/examples/pypsa/network.md +++ b/docs/examples/pypsa/network.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes diff --git a/docs/examples/pypsa/power_flow.md b/docs/examples/pypsa/power_flow.md index 728a7a176..50377935f 100644 --- a/docs/examples/pypsa/power_flow.md +++ b/docs/examples/pypsa/power_flow.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true cycle: description: >- independent cycles of the passive network graph — the cycle basis, data diff --git a/docs/examples/pypsa/process.md b/docs/examples/pypsa/process.md index 9f652ef43..39a06c4ea 100644 --- a/docs/examples/pypsa/process.md +++ b/docs/examples/pypsa/process.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes process: @@ -29,6 +30,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/process_commitment.md b/docs/examples/pypsa/process_commitment.md index bd8e82e3a..b68a58d84 100644 --- a/docs/examples/pypsa/process_commitment.md +++ b/docs/examples/pypsa/process_commitment.md @@ -15,11 +15,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true process: description: generalized multi-port converters, each with an internal power that every port draws or delivers at its own rate period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/examples/pypsa/process_maintenance.md b/docs/examples/pypsa/process_maintenance.md index 725a110c8..33bb93a04 100644 --- a/docs/examples/pypsa/process_maintenance.md +++ b/docs/examples/pypsa/process_maintenance.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true process: description: generalized multi-port converters, each with an internal power that every port draws or delivers at its own rate diff --git a/docs/examples/pypsa/process_ramping.md b/docs/examples/pypsa/process_ramping.md index 0747175b6..65c24d92e 100644 --- a/docs/examples/pypsa/process_ramping.md +++ b/docs/examples/pypsa/process_ramping.md @@ -15,11 +15,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true process: description: generalized multi-port converters, each with an internal power that every port draws or delivers at its own rate period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/examples/pypsa/security.md b/docs/examples/pypsa/security.md index 5a1ef88c9..e63786a4f 100644 --- a/docs/examples/pypsa/security.md +++ b/docs/examples/pypsa/security.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true line: description: passive branches, each between two buses, their flow set by impedance transformer: @@ -27,6 +28,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/examples/pypsa/settings.md b/docs/examples/pypsa/settings.md index 0edaca9fe..f4c9f0dfa 100644 --- a/docs/examples/pypsa/settings.md +++ b/docs/examples/pypsa/settings.md @@ -15,11 +15,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true global_constraint: description: PyPSA's `GlobalConstraint` rows, one label per declared limit period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/storage_unit.md b/docs/examples/pypsa/storage_unit.md index 053d01f50..ca5e99445 100644 --- a/docs/examples/pypsa/storage_unit.md +++ b/docs/examples/pypsa/storage_unit.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes storage_unit: @@ -24,6 +25,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/store.md b/docs/examples/pypsa/store.md index 343d23c09..fce45eeca 100644 --- a/docs/examples/pypsa/store.md +++ b/docs/examples/pypsa/store.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes store: @@ -24,6 +25,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/docs/examples/pypsa/transformer.md b/docs/examples/pypsa/transformer.md index fc4008a8e..a74e06dd6 100644 --- a/docs/examples/pypsa/transformer.md +++ b/docs/examples/pypsa/transformer.md @@ -15,6 +15,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes transformer: @@ -34,6 +35,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/docs/howto/compose.md b/docs/howto/compose.md index bdc9df52b..d1f2b50b6 100644 --- a/docs/howto/compose.md +++ b/docs/howto/compose.md @@ -120,7 +120,7 @@ takes its files as a list, and the two compose as `override(merge([…]), […]) ```yaml title="store.yaml" dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } bus: { dtype: str } store: { dtype: str } relations: @@ -201,6 +201,7 @@ which each component pins at its own port. | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | a dimension or a relation | every fragment may declare it, and the ones that do say the same thing about it | | a `description` on a shared dimension or relation | it is prose rather than a claim, and the first wording in the list is carried | +| `ordered: true` on a shared dimension | it is a claim about the dimension rather than the dimension, so the dimension is ordered if one fragment says so | | any other declaration | one fragment declares it, and a second is refused | | an entry under `given:` | it is checked against the fragment that introduces the name, then folded into it. Its description fills the declaration where the introducer wrote none | | a given expression | the definition's body carries no dimension the reader's `dims` do not name | @@ -355,7 +356,8 @@ Given variable 'Generator_p' collides with the variable of the same name. Names | `null` under a declaration's name | it is removed | | `null` on a field of a declaration | the field takes its default, and the rest of the declaration stays | | `null` under a section's name | it is refused | -| a dimension or a relation | it is added, or restated word for word as the base declares it | +| a dimension or a relation | it is added, or restated as the base declares it | +| `ordered:` on a restated dimension | `true` makes the dimension ordered; `false` over an ordered one is refused | | an entry under one kind of `given:` | it is edited, added or removed like any declaration, and the other kinds stay | | `version`, `description` | the patch's value replaces the base's | | a field an earlier patch writes | the later patch's value replaces it | @@ -376,14 +378,24 @@ as the base spells it. ## A dimension redeclared A patch may add a dimension or a relation, and may restate one the base -declares. The restatement is word for word: half a declaration is a second -reading of the same name. Changing one under the expressions already written +declares. The restatement says what the base says: half a declaration is a +second reading of the same name, and a field written at its default is the +same as one left out. Changing one under the expressions already written over it is refused, and so is removing one: ```text patch 'relabelled.yaml' declares the dimension 'snapshot' as {'dtype': 'str'}, where its base declares {'dtype': 'int'}. A patch adjusts the math, not the coordinate space the math is already written over: restate the declaration word for word, leave it out, or give the patch a dimension of its own under a name of its own. ``` +`ordered` is a claim about the dimension, not the dimension. A patch may add +it, so a patch that steps along `snapshot` declares it `ordered: true` over a +base that does not. A patch may not take it back, because the base may +already step along it: + +```text +patch 'unordered.yaml' says the dimension 'snapshot' is not ordered, where its base declares it ordered. A construct in the base may step along it, and a patch adds the claim of order but never withdraws it: leave `ordered` out of the patch. +``` + ## A section set to `null` A `null` removes the declaration it names. A section holds declarations rather diff --git a/docs/howto/see-an-expansion.md b/docs/howto/see-an-expansion.md index e92988ef6..152d00797 100644 --- a/docs/howto/see-an-expansion.md +++ b/docs/howto/see-an-expansion.md @@ -152,7 +152,7 @@ the set out too. ```yaml dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } @@ -209,7 +209,7 @@ the set out too. ```yaml dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } @@ -312,7 +312,7 @@ the set out too. ```yaml dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/docs/reference/language/assumptions.md b/docs/reference/language/assumptions.md index d8d1edb9c..bdb9fa27c 100644 --- a/docs/reference/language/assumptions.md +++ b/docs/reference/language/assumptions.md @@ -57,7 +57,7 @@ includes arithmetic on either side: ```yaml dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } generator: { dtype: str } parameters: eta: { dims: [generator] } diff --git a/docs/reference/language/dimensions.md b/docs/reference/language/dimensions.md index 922073226..688862d2b 100644 --- a/docs/reference/language/dimensions.md +++ b/docs/reference/language/dimensions.md @@ -13,21 +13,46 @@ onto another is a [relation](relations.md). ```yaml dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } generator: { dtype: str } ``` Every dimension named anywhere in the file is declared here. -| Field | | | -| ------------- | --------------------------------- | -------------- | -| `dtype` | `float`, `int`, `str`, `datetime` | default `str` | -| `description` | free text | default `null` | +| Field | | | +| ------------- | ----------------------------------------------------- | --------------- | +| `dtype` | `float`, `int`, `str`, `datetime` | default `str` | +| `ordered` | whether the order of the members is part of the model | default `false` | +| `description` | free text | default `null` | The members of a dimension arrive with the data, in the order the table gives -them. [`shift`](operators.md#shift), `sum_back` and `position()` count along -that order, and everything indexed by the dimension is matched to its members -by label. +them. Everything indexed by the dimension is matched to its members by label. + +## Order + +**Only an ordered dimension has an order a construct may read.** A snapshot +comes after the one before it. A generator does not come after another +generator, and a file that steps from one to the next would read the row order +of a data table. Five constructs read the order, and each needs its dimension +declared `ordered: true`: + +| Construct | Reads | +| ---------------------------------------------------------------------------------- | -------------------------------- | +| [`shift(x, along=d, offset=n)`](operators.md#shift), in an expression or a `where` | the member `n` positions back | +| [`sum_back(x, along=d, window=n)`](operators.md#sum_back) | the last `n` members | +| [`position(d)`](expressions.md#position) | where a member sits along `d` | +| [`piecewise:`](piecewise.md) with `over: d` | which breakpoints are neighbours | +| [`sos:`](piecewise.md#sos) with `type: 2` and `along: d` | which members are consecutive | + +The loader refuses any of them along a dimension that is not ordered: + +```text +Constraint 'ramp': shift(along=snapshot) reads the order of 'snapshot', which is not declared ordered, so the order it read would be the row order of the data. Declare the order part of the model with 'snapshot: {ordered: true}' under dimensions:. +``` + +A `dtype` does not make a dimension ordered: an `int` dimension may number +things that have no order. An `sos` block with `type: 1` reads no order, and +`edge='wrap'` makes one `shift` cyclic without changing the dimension. Whether to declare a column of data as a dimension, a relation or a parameter is decided in [declare a column of data](../../howto/declare-a-column.md). diff --git a/docs/reference/language/expressions.md b/docs/reference/language/expressions.md index 1ea8151d7..29f9ecb75 100644 --- a/docs/reference/language/expressions.md +++ b/docs/reference/language/expressions.md @@ -281,7 +281,7 @@ is one fewer. A comparison against the previous row gives its `shift` an ```yaml dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } parameters: load: { dims: [snapshot] } ramp: { dims: [] } @@ -304,7 +304,7 @@ refused everywhere. ```yaml dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } parameters: soc_initial: { dims: [] } variables: @@ -323,7 +323,7 @@ makes, so each period gets one seeded row: ```yaml dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } period: { dtype: int } relations: period_of: { key: snapshot, values: period } diff --git a/docs/reference/language/index.md b/docs/reference/language/index.md index db5131161..59384fc75 100644 --- a/docs/reference/language/index.md +++ b/docs/reference/language/index.md @@ -39,7 +39,7 @@ objective: That file is a complete spec. The pages of this section give the exact rules, and the [glossary](../glossary.md) defines each word they use in a fixed sense. -## The ten rules +## The eleven rules `to_spec` refuses a file that breaks one of these rules, with a message that names the fix. @@ -56,3 +56,4 @@ names the fix. | 8 | A parameter row missing from the table reads as `0` in arithmetic and as false in a `where`. Where `0` would change the meaning, the row is refused. | [Absence](absence.md#what-creates-absence) | | 9 | Two variables may be multiplied in the objective and in a constraint, and nowhere else. `x / y` and `a ** b` need their divisor, base and exponent free of variables. | [Expressions](expressions.md) | | 10 | The operators are `sum`, `sum_back`, `at` and `shift`, plus `dual` in a reported expression. A file cannot add one. | [Operators](operators.md) | +| 11 | A construct that steps or counts along a dimension needs it declared `ordered: true`: `shift`, `sum_back`, `position`, `piecewise` and an `sos` of `type: 2`. | [Order](dimensions.md#order) | diff --git a/docs/reference/language/operators.md b/docs/reference/language/operators.md index 27b0ce7d7..f67289912 100644 --- a/docs/reference/language/operators.md +++ b/docs/reference/language/operators.md @@ -80,7 +80,7 @@ The dimension **survives**, and a width of `1` is `x` itself. ```yaml dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } parameters: min_up: { dims: [unit], dtype: int } @@ -107,12 +107,13 @@ window that reaches past the start of the axis is **short**, and no row is lost. ## `shift` -`shift` counts positions in the dimension's **declared order**. `edge=` says +`shift` counts positions in the order of an [ordered](dimensions.md#order) +dimension. `edge=` says what stands where nothing moved in. ```yaml dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } storage: { dtype: str } parameters: eta: { dims: [storage] } @@ -144,7 +145,7 @@ coordinate is the one before it in its own group, such as a season: ```yaml dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } season: { dtype: str } relations: season_of: { key: snapshot, values: season } @@ -170,7 +171,7 @@ the data carries as a column: ```yaml dimensions: technology: { dtype: str } - month: { dtype: int } + month: { dtype: int, ordered: true } parameters: lead: { dims: [technology], dtype: int } demand: { dims: [technology, month] } diff --git a/docs/reference/language/piecewise.md b/docs/reference/language/piecewise.md index b48f17716..6a14472b1 100644 --- a/docs/reference/language/piecewise.md +++ b/docs/reference/language/piecewise.md @@ -51,7 +51,8 @@ piecewise: A block states one weight per breakpoint in `[0, 1]`, a row making the weights sum to 1, and a row per link tying its expression to the weighted breakpoints. -The breakpoint order is the declared order of `over`. What a block assumes of +The breakpoint order is the order of `over`, which is an +[ordered](dimensions.md#order) dimension. What a block assumes of its numbers is on [what a curve assumes](assumptions.md#what-a-curve-assumes). ### `activity` @@ -139,7 +140,8 @@ A set is over **one** variable, and a variable holds **one** set. A second block naming the same variable is a load error. A member the variable's `where` masks out is not in the set. The order is the -declared order of the `along` dimension. +order of the `along` dimension, which is +[ordered](dimensions.md#order) for `type: 2`. ### What a set is written out as diff --git a/docs/reference/notation.md b/docs/reference/notation.md index bd20c85f0..0938e0d3a 100644 --- a/docs/reference/notation.md +++ b/docs/reference/notation.md @@ -28,13 +28,13 @@ A dimension, a relation and a parameter declare no equation; what they print is ```yaml dimensions: - snapshot: { dtype: int } - generator: { dtype: str } + snapshot: { dtype: int, ordered: true } + generator: { dtype: str, ordered: true } bus: { dtype: str } zone: { dtype: str } season: { dtype: str } technology: { dtype: str } - bp: { dtype: int } # the breakpoints every curve below runs through + bp: { dtype: int, ordered: true } # the breakpoints every curve below runs through relations: gen_bus: { key: generator, values: bus } diff --git a/docs/reference/reading.md b/docs/reference/reading.md index 1bc759127..9677b2a9e 100644 --- a/docs/reference/reading.md +++ b/docs/reference/reading.md @@ -37,7 +37,7 @@ a convexity row and one row per link: ```yaml title="curve.yaml" dimensions: generator: { dtype: str } - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: bp_x: { dims: [generator, bp] } bp_y: { dims: [generator, bp] } diff --git a/examples/commitment.yaml b/examples/commitment.yaml index 079c534b5..638e5a9bd 100644 --- a/examples/commitment.yaml +++ b/examples/commitment.yaml @@ -10,7 +10,7 @@ description: >- inequality is written once. dimensions: - snapshot: { dtype: int, description: dispatch periods } + snapshot: { dtype: int, description: dispatch periods, ordered: true } generator: { description: generating units } parameters: diff --git a/examples/operators/shift.yaml b/examples/operators/shift.yaml index 2e31c65e6..33c1bce0a 100644 --- a/examples/operators/shift.yaml +++ b/examples/operators/shift.yaml @@ -7,7 +7,7 @@ description: >- it would have fed is not built. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } variables: p: diff --git a/examples/operators/shift_by_parameter.yaml b/examples/operators/shift_by_parameter.yaml index 6be3cb2ea..25482ceb2 100644 --- a/examples/operators/shift_by_parameter.yaml +++ b/examples/operators/shift_by_parameter.yaml @@ -9,7 +9,7 @@ description: >- dimensions: technology: { dtype: str } - month: { dtype: int } + month: { dtype: int, ordered: true } parameters: lead: { dims: [technology], dtype: int } diff --git a/examples/operators/shift_edge.yaml b/examples/operators/shift_edge.yaml index 721f601e8..9fcab13e5 100644 --- a/examples/operators/shift_edge.yaml +++ b/examples/operators/shift_edge.yaml @@ -7,7 +7,7 @@ description: >- number instead of being absent, so the row survives. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } variables: p: diff --git a/examples/operators/shift_partitioned.yaml b/examples/operators/shift_partitioned.yaml index f8a6ae901..62e7c895e 100644 --- a/examples/operators/shift_partitioned.yaml +++ b/examples/operators/shift_partitioned.yaml @@ -7,7 +7,7 @@ description: >- snapshot reads that season's last and no level crosses the boundary. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } season: { dtype: str } relations: diff --git a/examples/operators/shift_wrap.yaml b/examples/operators/shift_wrap.yaml index 0323f8862..e094ad781 100644 --- a/examples/operators/shift_wrap.yaml +++ b/examples/operators/shift_wrap.yaml @@ -7,7 +7,7 @@ description: >- reads the last and nothing is vacated. dimensions: - snapshot: { dtype: int } + snapshot: { dtype: int, ordered: true } variables: p: diff --git a/examples/operators/sum_back.yaml b/examples/operators/sum_back.yaml index e70afcfb0..d93271f19 100644 --- a/examples/operators/sum_back.yaml +++ b/examples/operators/sum_back.yaml @@ -8,7 +8,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } parameters: min_up: { dims: [unit], dtype: int } diff --git a/examples/operators/sum_back_by_parameter.yaml b/examples/operators/sum_back_by_parameter.yaml index 699a0262b..8e4b169bc 100644 --- a/examples/operators/sum_back_by_parameter.yaml +++ b/examples/operators/sum_back_by_parameter.yaml @@ -8,7 +8,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } parameters: min_up: { dims: [unit], dtype: int } diff --git a/examples/operators/sum_back_partitioned.yaml b/examples/operators/sum_back_partitioned.yaml index 289465d9f..9892d4e20 100644 --- a/examples/operators/sum_back_partitioned.yaml +++ b/examples/operators/sum_back_partitioned.yaml @@ -9,7 +9,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } day: { dtype: str } relations: diff --git a/examples/operators/sum_back_wrap.yaml b/examples/operators/sum_back_wrap.yaml index 69fea5ec0..b31f4b651 100644 --- a/examples/operators/sum_back_wrap.yaml +++ b/examples/operators/sum_back_wrap.yaml @@ -8,7 +8,7 @@ description: >- dimensions: unit: { dtype: str } - hour: { dtype: int } + hour: { dtype: int, ordered: true } parameters: min_up: { dims: [unit], dtype: int } diff --git a/examples/piecewise.yaml b/examples/piecewise.yaml index c50ccd342..c5ca957ab 100644 --- a/examples/piecewise.yaml +++ b/examples/piecewise.yaml @@ -16,6 +16,7 @@ dimensions: bp: description: breakpoints of the cost curve dtype: int + ordered: true parameters: capacity: diff --git a/examples/piecewise_lp.yaml b/examples/piecewise_lp.yaml index 817c97655..2029aa8dd 100644 --- a/examples/piecewise_lp.yaml +++ b/examples/piecewise_lp.yaml @@ -19,6 +19,7 @@ dimensions: bp: description: breakpoints of the cost curve dtype: int + ordered: true parameters: capacity: diff --git a/examples/piecewise_ragged.yaml b/examples/piecewise_ragged.yaml index 582a28c21..fd6b0ec14 100644 --- a/examples/piecewise_ragged.yaml +++ b/examples/piecewise_ragged.yaml @@ -20,6 +20,7 @@ dimensions: bp: description: breakpoints of the cost curve, as many as the longest curve needs dtype: int + ordered: true parameters: capacity: diff --git a/examples/ports/transport_pwl.yaml b/examples/ports/transport_pwl.yaml index 678aa6fca..71b63b0cb 100644 --- a/examples/ports/transport_pwl.yaml +++ b/examples/ports/transport_pwl.yaml @@ -16,6 +16,7 @@ dimensions: bp: description: breakpoints of the discretised square-root curve dtype: int + ordered: true parameters: capacity: diff --git a/examples/pypsa.yaml b/examples/pypsa.yaml index aab9dbbb1..c51b7a09e 100644 --- a/examples/pypsa.yaml +++ b/examples/pypsa.yaml @@ -23,6 +23,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes generator: @@ -73,6 +74,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/carrier.yaml b/examples/pypsa/carrier.yaml index aa18eb7a5..f1bc04297 100644 --- a/examples/pypsa/carrier.yaml +++ b/examples/pypsa/carrier.yaml @@ -6,6 +6,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/generator.yaml b/examples/pypsa/generator.yaml index 7413feea9..f55ccf07d 100644 --- a/examples/pypsa/generator.yaml +++ b/examples/pypsa/generator.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes generator: @@ -17,6 +18,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/generator_commitment.yaml b/examples/pypsa/generator_commitment.yaml index 57fab4c56..ab87bf2c0 100644 --- a/examples/pypsa/generator_commitment.yaml +++ b/examples/pypsa/generator_commitment.yaml @@ -8,11 +8,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true generator: description: generating units, each on one bus period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/pypsa/generator_maintenance.yaml b/examples/pypsa/generator_maintenance.yaml index 87897bda0..397212430 100644 --- a/examples/pypsa/generator_maintenance.yaml +++ b/examples/pypsa/generator_maintenance.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true generator: description: generating units, each on one bus diff --git a/examples/pypsa/generator_ramping.yaml b/examples/pypsa/generator_ramping.yaml index fd04f36ff..86453e535 100644 --- a/examples/pypsa/generator_ramping.yaml +++ b/examples/pypsa/generator_ramping.yaml @@ -8,11 +8,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true generator: description: generating units, each on one bus period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/pypsa/line.yaml b/examples/pypsa/line.yaml index 42d8c6617..2eb5750ad 100644 --- a/examples/pypsa/line.yaml +++ b/examples/pypsa/line.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes line: @@ -29,6 +30,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/link.yaml b/examples/pypsa/link.yaml index 615a11896..08fe4389c 100644 --- a/examples/pypsa/link.yaml +++ b/examples/pypsa/link.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes link: @@ -22,6 +23,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/link_commitment.yaml b/examples/pypsa/link_commitment.yaml index 10d9d2619..51f9c7366 100644 --- a/examples/pypsa/link_commitment.yaml +++ b/examples/pypsa/link_commitment.yaml @@ -8,11 +8,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true link: description: controllable connections, each from one bus to the buses it delivers to period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/pypsa/link_maintenance.yaml b/examples/pypsa/link_maintenance.yaml index e9876dd59..aefd65cd0 100644 --- a/examples/pypsa/link_maintenance.yaml +++ b/examples/pypsa/link_maintenance.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true link: description: controllable connections, each from one bus to the buses it delivers to diff --git a/examples/pypsa/link_ramping.yaml b/examples/pypsa/link_ramping.yaml index 0c68246d4..4dd25f7c2 100644 --- a/examples/pypsa/link_ramping.yaml +++ b/examples/pypsa/link_ramping.yaml @@ -8,11 +8,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true link: description: controllable connections, each from one bus to the buses it delivers to period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/pypsa/load.yaml b/examples/pypsa/load.yaml index 33f0a73cd..456853288 100644 --- a/examples/pypsa/load.yaml +++ b/examples/pypsa/load.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes load: diff --git a/examples/pypsa/network.yaml b/examples/pypsa/network.yaml index 47938d68d..eb2fd89c9 100644 --- a/examples/pypsa/network.yaml +++ b/examples/pypsa/network.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes diff --git a/examples/pypsa/power_flow.yaml b/examples/pypsa/power_flow.yaml index ed632ec58..1fb4b00d2 100644 --- a/examples/pypsa/power_flow.yaml +++ b/examples/pypsa/power_flow.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true cycle: description: >- independent cycles of the passive network graph — the cycle basis, data diff --git a/examples/pypsa/process.yaml b/examples/pypsa/process.yaml index 381283722..fa9ee9660 100644 --- a/examples/pypsa/process.yaml +++ b/examples/pypsa/process.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes process: @@ -22,6 +23,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/process_commitment.yaml b/examples/pypsa/process_commitment.yaml index 5007005c5..134c82a95 100644 --- a/examples/pypsa/process_commitment.yaml +++ b/examples/pypsa/process_commitment.yaml @@ -8,11 +8,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true process: description: generalized multi-port converters, each with an internal power that every port draws or delivers at its own rate period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/pypsa/process_maintenance.yaml b/examples/pypsa/process_maintenance.yaml index 519ad0e83..af1beb1cd 100644 --- a/examples/pypsa/process_maintenance.yaml +++ b/examples/pypsa/process_maintenance.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true process: description: generalized multi-port converters, each with an internal power that every port draws or delivers at its own rate diff --git a/examples/pypsa/process_ramping.yaml b/examples/pypsa/process_ramping.yaml index c3726ff22..af2e36e67 100644 --- a/examples/pypsa/process_ramping.yaml +++ b/examples/pypsa/process_ramping.yaml @@ -8,11 +8,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true process: description: generalized multi-port converters, each with an internal power that every port draws or delivers at its own rate period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/pypsa/security.yaml b/examples/pypsa/security.yaml index 7bc5f3055..e09c6e1ad 100644 --- a/examples/pypsa/security.yaml +++ b/examples/pypsa/security.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true line: description: passive branches, each between two buses, their flow set by impedance transformer: @@ -20,6 +21,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/pypsa/settings.yaml b/examples/pypsa/settings.yaml index 32e82bf52..35cdc6100 100644 --- a/examples/pypsa/settings.yaml +++ b/examples/pypsa/settings.yaml @@ -8,11 +8,13 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true global_constraint: description: PyPSA's `GlobalConstraint` rows, one label per declared limit period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/storage_unit.yaml b/examples/pypsa/storage_unit.yaml index 6e692910b..d5118e1be 100644 --- a/examples/pypsa/storage_unit.yaml +++ b/examples/pypsa/storage_unit.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes storage_unit: @@ -17,6 +18,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/store.yaml b/examples/pypsa/store.yaml index 4987a05cd..937472960 100644 --- a/examples/pypsa/store.yaml +++ b/examples/pypsa/store.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes store: @@ -17,6 +18,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true carrier: description: energy carriers, what a growth limit is set per diff --git a/examples/pypsa/transformer.yaml b/examples/pypsa/transformer.yaml index 05ce8ba4e..24400869c 100644 --- a/examples/pypsa/transformer.yaml +++ b/examples/pypsa/transformer.yaml @@ -8,6 +8,7 @@ dimensions: snapshot: description: dispatch periods dtype: datetime + ordered: true bus: description: network nodes transformer: @@ -27,6 +28,7 @@ dimensions: period: description: investment periods — PyPSA's `investment_periods` dtype: int + ordered: true relations: snapshot_period: diff --git a/examples/sos.yaml b/examples/sos.yaml index 527b22998..89d4a94b5 100644 --- a/examples/sos.yaml +++ b/examples/sos.yaml @@ -16,6 +16,7 @@ dimensions: bp: description: breakpoints of the cost curve dtype: int + ordered: true parameters: capacity: diff --git a/schema/mathspec.schema.json b/schema/mathspec.schema.json index f08b859eb..4016f4be0 100644 --- a/schema/mathspec.schema.json +++ b/schema/mathspec.schema.json @@ -133,7 +133,7 @@ }, "DimensionBlock": { "additionalProperties": false, - "description": "A declared dimension, and the dtype its coordinates must be.\n\nA dimension is an axis and nothing else: it declares that the axis exists\nand what its coordinates are typed as, never which coordinates there are \u2014\nthose are data, and arrive when the data is attached. The maps its members carry \u2014 a\ngenerator's bus, a snapshot's period \u2014 are top-level ``relations:``\n([`RelationBlock`][]), keyed by their own name.", + "description": "A declared dimension, the dtype its coordinates must be, and whether their order means anything.\n\nA dimension is an axis and nothing else: it declares that the axis exists\nand what its coordinates are typed as, never which coordinates there are \u2014\nthose are data, and arrive when the data is attached. The maps its members carry \u2014 a\ngenerator's bus, a snapshot's period \u2014 are top-level ``relations:``\n([`RelationBlock`][]), keyed by their own name.", "properties": { "description": { "anyOf": [ @@ -157,6 +157,11 @@ ], "title": "Dtype", "type": "string" + }, + "ordered": { + "default": false, + "title": "Ordered", + "type": "boolean" } }, "title": "DimensionBlock", diff --git a/src/mathspec/_expression_resolver.py b/src/mathspec/_expression_resolver.py index a1ab13fcd..b93437378 100644 --- a/src/mathspec/_expression_resolver.py +++ b/src/mathspec/_expression_resolver.py @@ -32,7 +32,7 @@ shown, ) from mathspec.dimensions import dims_of -from mathspec.errors import DimensionError, SchemaError, did_you_mean +from mathspec.errors import DimensionError, SchemaError, did_you_mean, unordered from mathspec.operators import ( AMOUNTS, BUILTINS, @@ -323,6 +323,9 @@ def _translation( partition: Partition | None, ) -> Expression | None: """``shift`` or ``sum_back`` from its read arguments, or ``None`` with the refusal appended.""" + if self.ns.unordered(along): + self.errors.append(unordered(self.context, f'{operator}(along={along})', along)) + return None wrap, fill = edge if edge is not None else (False, None) if operator == 'shift': offset = amounts['offset'] diff --git a/src/mathspec/_where_resolver.py b/src/mathspec/_where_resolver.py index ccb5ff923..d3c956891 100644 --- a/src/mathspec/_where_resolver.py +++ b/src/mathspec/_where_resolver.py @@ -35,7 +35,7 @@ UnresolvedWhereNode, ) from mathspec.dimensions import dims_of, join_dims -from mathspec.errors import DimensionError, LanguageError, did_you_mean, prefixed +from mathspec.errors import DimensionError, LanguageError, did_you_mean, prefixed, unordered from mathspec.expansion import expand from mathspec.program import ( Add, @@ -222,6 +222,9 @@ def _predicate_call(self, node: UnresolvedPredicateCallNode) -> Predicate | Unre f'it does not carry — it reads {_listed(sorted(mask.dims))}. Translate it along one of those.' ) return node + if self.ns.unordered(along.name): + self.errors.append(unordered(context, f'shift(, along={along.name})', along.name)) + return node return TranslatedPredicate(mask, along.name, int(offset.value), tuple(sorted(mask.dims))) def _joined(self, node: UnresolvedPredicateCallNode, mask: Mask) -> Predicate | UnresolvedWhereNode: @@ -424,6 +427,9 @@ def _position( f'{did_you_mean(dimension, ns.dimensions, label="Dimensions")}' ) return node + if ns.unordered(dimension): + self.errors.append(unordered(context, f'position({dimension})', dimension)) + return node if within is None: return DimensionPosition(dimension, node.op, position) columns = self._expressions.columns_ref(within, 'position', 'within') diff --git a/src/mathspec/composition.py b/src/mathspec/composition.py index b6f027c13..77047cd0a 100644 --- a/src/mathspec/composition.py +++ b/src/mathspec/composition.py @@ -25,6 +25,8 @@ * **A dimension or a relation every fragment may declare**, and the ones that do have to say the same thing about it. Prose is not a claim, so two descriptions of one dimension agree, and the first one given is carried. + ``ordered`` is a claim about the space, not the space, so a dimension one + fragment declares ordered is ordered. * **Every other declaration is owned.** A name two fragments declare is refused, both named. * **One fragment sets the objective.** A second one is refused, both named. @@ -74,8 +76,9 @@ earlier patch laid on it, so a later patch wins a field an earlier one writes, and edits or removes a declaration an earlier one creates. * **A patch adjusts the math, not the coordinate space.** A ``dimensions`` or - ``relations`` entry may be added or restated word for word, never changed and - never removed. + ``relations`` entry may be added or restated as the schema reads its base, + never changed and never removed. A restated dimension may add + ``ordered: true``, and may not write it false over a base that makes it. * ``null`` **makes what it names absent.** A declaration set to ``null`` is removed, and a removal of what the base does not declare is refused. A field set to ``null`` is dropped, and takes its default when the result loads: @@ -271,24 +274,52 @@ def _agreed( neither declaration is the one being restated, so a field only one of them writes is a difference nothing settles. *claims* says what a block claims; a reading's frame is a set. Prose is not a claim, so the first description - given is carried. + given is carried. A dimension's ``ordered`` folds by [`_joined`][]. """ merged: dict[str, object] = {} for name, sections in read.items(): for key, block in _mapping(sections.get(section)).items(): - if key in merged and claims(merged[key]) != claims(block): + if key not in merged: + merged[key] = block + elif (joined := _joined(section, merged[key], block, claims)) is None: raise LanguageError( f"fragments '{_author_of(read, section, key)}' and '{name}' say different things about " f'the {label} {key!r}: {merged[key]!r} against {block!r}. A declaration two fragments ' f'share is one both say the same thing about: make the two identical, or {repair}.' ) - merged.setdefault(key, block) + else: + merged[key] = joined for key, block in merged.items(): if said := _said(read, section, key): merged[key] = {**_mapping(block), 'description': said} return merged +def _joined(section: str, kept: object, block: object, claims: Callable[[object], object] = _claims) -> object | None: + """*kept* and *block* as the one declaration both say, or ``None`` where they say different things. + + A dimension's ``ordered`` is a claim about the space, not the space: it + lets a construct read the order the data gives, and the coordinates are + the same either way. So one declaration that makes the claim joins one + that does not, and the two are one ordered dimension. Every other field + is the space itself, and has to be equal under *claims*. + """ + if section != 'dimensions': + return kept if claims(kept) == claims(block) else None + if claims(_without(kept, 'ordered')) != claims(_without(block, 'ordered')): + return None + ordered = bool(_mapping(kept).get('ordered') or _mapping(block).get('ordered')) + return {**_mapping(kept), 'ordered': True} if ordered else kept + + +def _declared(section: str, block: object) -> dict[str, object]: + """*block* as the schema reads it, so a field written at its default says what leaving it out says.""" + try: + return _entry_class(Spec, section).model_validate(block).model_dump() + except ValidationError as e: + raise schema_error(e) from None + + def _claimed(read: Mapping[str, dict[str, object]], section: str) -> dict[str, object]: """One block of owned declarations, a name claimed twice being the refusal.""" merged: dict[str, object] = {} @@ -800,9 +831,12 @@ def _given(declared: dict[str, object], patch: dict[str, object], name: str) -> def _shared(declared: dict[str, object], patch: dict[str, object], section: str, name: str) -> dict[str, object]: """One ``dimensions`` or ``relations`` block: a patch adds one or restates one, never changes or drops it. - The restatement is compared for equality rather than field by field: a - patch that names half a declaration is as much a second reading of the - coordinate space as one that names another value. + The restatement is compared as the schema reads both sides rather than + field by field: a patch that names half a declaration is as much a second + reading of the coordinate space as one that names another value, and a + field written at its default is no change. A dimension's ``ordered`` folds + by [`_joined`][], so a patch may add the claim; writing it false over a + base that makes it is the one narrowing [`_joined`][] cannot see. """ out = dict(declared) singular = _singular(section) @@ -815,13 +849,22 @@ def _shared(declared: dict[str, object], patch: dict[str, object], section: str, ) if key not in out: out[key] = block - elif out[key] != block: + continue + base, laid = _declared(section, out[key]), _declared(section, block) + if base.get('ordered') and _mapping(block).get('ordered') is False: + raise LanguageError( + f"patch '{name}' says the {singular} '{key}' is not ordered, where its base declares it " + f'ordered. A construct in the base may step along it, and a patch adds the claim of order ' + f'but never withdraws it: leave `ordered` out of the patch.' + ) + if (joined := _joined(section, base, laid, lambda written: written)) is None: raise LanguageError( f"patch '{name}' declares the {singular} '{key}' as {block!r}, where its base " f'declares {out[key]!r}. A patch adjusts the math, not the coordinate space the math is ' f'already written over: restate the declaration word for word, leave it out, or give the ' f'patch {_a(singular)} of its own under a name of its own.' ) + out[key] = joined return out diff --git a/src/mathspec/errors.py b/src/mathspec/errors.py index 7a81d1ec8..6da5172a0 100644 --- a/src/mathspec/errors.py +++ b/src/mathspec/errors.py @@ -68,6 +68,15 @@ def did_you_mean(name: str, known: Iterable[str], *, label: str = 'Declared', li return f'{label}: {", ".join(candidates) or "nothing"}.' if listing else '' +def unordered(context: str, construct: str, dimension: str) -> str: + """The refusal for *construct* reading the order of *dimension*, which is not declared ordered.""" + return ( + f"{context}: {construct} reads the order of '{dimension}', which is not declared ordered, so the " + f'order it read would be the row order of the data. Declare the order part of the model with ' + f"'{dimension}: {{ordered: true}}' under dimensions:." + ) + + def schema_error(exc: ValidationError) -> LanguageError: """A pydantic ``ValidationError`` as one of ours. diff --git a/src/mathspec/lowering.py b/src/mathspec/lowering.py index cabeb1d8f..fdf181006 100644 --- a/src/mathspec/lowering.py +++ b/src/mathspec/lowering.py @@ -211,7 +211,8 @@ def lower(schema: Spec) -> Program: constraints=constraints, objective=objective, dimensions={ - name: DimensionDeclaration(ddef.dtype, ddef.description) for name, ddef in schema.dimensions.items() + name: DimensionDeclaration(dtype=ddef.dtype, ordered=ddef.ordered, description=ddef.description) + for name, ddef in schema.dimensions.items() }, relations=ns.relations, sos={ diff --git a/src/mathspec/program.py b/src/mathspec/program.py index cca727b3b..307a03bfd 100644 --- a/src/mathspec/program.py +++ b/src/mathspec/program.py @@ -624,6 +624,9 @@ class DimensionDeclaration: #: checked against — the same claim ``ParameterDeclaration.dtype`` makes #: about a value column, one axis over. dtype: DimensionDtype = 'str' + #: Whether the order of the labels is part of the model, so a consumer + #: must keep the order the data gives them in. + ordered: bool = False description: str | None = None diff --git a/src/mathspec/resolution.py b/src/mathspec/resolution.py index 3f8fcf237..97c250013 100644 --- a/src/mathspec/resolution.py +++ b/src/mathspec/resolution.py @@ -202,6 +202,10 @@ def _references(self, name: str) -> tuple[str, ...]: names = (n.name for n in nodes(*arithmetic) if isinstance(n, NameNode) and n.name in self.bodies) return tuple(dict.fromkeys(names)) + def unordered(self, name: str) -> bool: + """Whether the declared dimension *name* is one whose order the file does not declare part of the model.""" + return not self.schema.dimensions[name].ordered + def kind(self, name: str) -> DeclarationKind | None: """What *name* was declared as, or ``None`` where the file declares it nowhere.""" if name in self.variables: diff --git a/src/mathspec/spec.py b/src/mathspec/spec.py index 42874b173..b9398eedb 100644 --- a/src/mathspec/spec.py +++ b/src/mathspec/spec.py @@ -193,7 +193,7 @@ def value_roles(self) -> tuple[str, ...]: class DimensionBlock(_StrictBlock): - """A declared dimension, and the dtype its coordinates must be. + """A declared dimension, the dtype its coordinates must be, and whether their order means anything. A dimension is an axis and nothing else: it declares that the axis exists and what its coordinates are typed as, never which coordinates there are — @@ -205,8 +205,20 @@ class DimensionBlock(_StrictBlock): _label: ClassVar[str] = 'a dimension declaration' dtype: DimensionDtype = 'str' + #: Whether the order the data gives the coordinates in is part of the + #: model. Only an ordered dimension is one a construct may step or count + #: along; every other one is a set whose row order means nothing. + ordered: bool = False description: str | None = None + @model_serializer(mode='wrap') + def _as_written(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]: + """``ordered`` is written where it is true: false is what leaving it out says.""" + written = cast('dict[str, object]', handler(self)) + if not self.ordered: + written.pop('ordered', None) + return written + class ParameterBlock(_StrictBlock): """A declared parameter with dims and dtype.""" diff --git a/src/mathspec/validation.py b/src/mathspec/validation.py index 3b7424ae6..32f3bc56a 100644 --- a/src/mathspec/validation.py +++ b/src/mathspec/validation.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING from mathspec._yaml import read_spec -from mathspec.errors import SchemaError +from mathspec.errors import SchemaError, unordered from mathspec.operators import BUILTIN_NAMES from mathspec.piecewise import Emitted as EmittedCurve from mathspec.sos import Emitted as EmittedSet @@ -257,6 +257,8 @@ def _sos_shapes(schema: Spec) -> Iterator[str]: f"'{block.variable}' (dims {schema.variables[block.variable].dims}). The set runs " f"along one of the variable's own dims — one set per coordinate of the rest." ) + elif block.type == 2 and not schema.dimensions[block.along].ordered: + yield unordered(context, f'type: 2 along {block.along}', block.along) elif block.variable in claimed: yield ( f"{context}: variable '{block.variable}' already carries the set declared by " @@ -302,6 +304,8 @@ def _piecewise_references(schema: Spec) -> Iterator[str]: if pw.over not in schema.dimensions: yield undeclared_dimension('piecewise', name, pw.over) continue + if not schema.dimensions[pw.over].ordered: + yield unordered(context, f'over: {pw.over}', pw.over) for i, link in enumerate(pw.links): if link.values not in schema.parameters: yield f"{context}: link {i} values references undeclared parameter '{link.values}'" diff --git a/tests/expand/curve-activity/after.yaml b/tests/expand/curve-activity/after.yaml index 4ec420855..21767569c 100644 --- a/tests/expand/curve-activity/after.yaml +++ b/tests/expand/curve-activity/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-activity/before.yaml b/tests/expand/curve-activity/before.yaml index 41948c3dc..75f852ea2 100644 --- a/tests/expand/curve-activity/before.yaml +++ b/tests/expand/curve-activity/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-adjacency-points/after.yaml b/tests/expand/curve-adjacency-points/after.yaml index 0dd6a7d6f..1ce7bc261 100644 --- a/tests/expand/curve-adjacency-points/after.yaml +++ b/tests/expand/curve-adjacency-points/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-adjacency-points/before.yaml b/tests/expand/curve-adjacency-points/before.yaml index 45b82dfe8..6573f78b4 100644 --- a/tests/expand/curve-adjacency-points/before.yaml +++ b/tests/expand/curve-adjacency-points/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-adjacency/after.yaml b/tests/expand/curve-adjacency/after.yaml index 317493b6e..3b607d0f9 100644 --- a/tests/expand/curve-adjacency/after.yaml +++ b/tests/expand/curve-adjacency/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-adjacency/before.yaml b/tests/expand/curve-adjacency/before.yaml index 133038e25..10d771f74 100644 --- a/tests/expand/curve-adjacency/before.yaml +++ b/tests/expand/curve-adjacency/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-convex/after.yaml b/tests/expand/curve-convex/after.yaml index 4d69388a3..05e0f3b5e 100644 --- a/tests/expand/curve-convex/after.yaml +++ b/tests/expand/curve-convex/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-convex/before.yaml b/tests/expand/curve-convex/before.yaml index 269b47614..c41665e9d 100644 --- a/tests/expand/curve-convex/before.yaml +++ b/tests/expand/curve-convex/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-lp-points/after.yaml b/tests/expand/curve-lp-points/after.yaml index 9a24e7b0d..6918dbe25 100644 --- a/tests/expand/curve-lp-points/after.yaml +++ b/tests/expand/curve-lp-points/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-lp-points/before.yaml b/tests/expand/curve-lp-points/before.yaml index 17748873c..8c9210046 100644 --- a/tests/expand/curve-lp-points/before.yaml +++ b/tests/expand/curve-lp-points/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-lp/after.yaml b/tests/expand/curve-lp/after.yaml index 0c7f627bc..961a6861c 100644 --- a/tests/expand/curve-lp/after.yaml +++ b/tests/expand/curve-lp/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-lp/before.yaml b/tests/expand/curve-lp/before.yaml index 7f2815334..0d2330ef4 100644 --- a/tests/expand/curve-lp/before.yaml +++ b/tests/expand/curve-lp/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-sos2-piecewise/after.yaml b/tests/expand/curve-sos2-piecewise/after.yaml index 2efdb329b..ff4466034 100644 --- a/tests/expand/curve-sos2-piecewise/after.yaml +++ b/tests/expand/curve-sos2-piecewise/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-sos2-piecewise/before.yaml b/tests/expand/curve-sos2-piecewise/before.yaml index c3c82ec71..7cb049498 100644 --- a/tests/expand/curve-sos2-piecewise/before.yaml +++ b/tests/expand/curve-sos2-piecewise/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-sos2/after.yaml b/tests/expand/curve-sos2/after.yaml index 317493b6e..3b607d0f9 100644 --- a/tests/expand/curve-sos2/after.yaml +++ b/tests/expand/curve-sos2/after.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/curve-sos2/before.yaml b/tests/expand/curve-sos2/before.yaml index c3c82ec71..7cb049498 100644 --- a/tests/expand/curve-sos2/before.yaml +++ b/tests/expand/curve-sos2/before.yaml @@ -1,5 +1,5 @@ dimensions: - bp: { dtype: int } + bp: { dtype: int, ordered: true } parameters: x_bp: { dims: [bp] } diff --git a/tests/expand/set-type2/after.yaml b/tests/expand/set-type2/after.yaml index 0fa7a4c63..583f060ed 100644 --- a/tests/expand/set-type2/after.yaml +++ b/tests/expand/set-type2/after.yaml @@ -1,5 +1,5 @@ dimensions: - g: { dtype: str } + g: { dtype: str, ordered: true } variables: p: diff --git a/tests/expand/set-type2/before.yaml b/tests/expand/set-type2/before.yaml index 1a6d936be..ae4a2eac7 100644 --- a/tests/expand/set-type2/before.yaml +++ b/tests/expand/set-type2/before.yaml @@ -1,5 +1,5 @@ dimensions: - g: { dtype: str } + g: { dtype: str, ordered: true } variables: p: diff --git a/tests/fixtures.py b/tests/fixtures.py index e4267ea62..d654020fa 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -32,7 +32,7 @@ #: names, so a test that prints it asserts on the math rather than on the #: example's own vocabulary. DISPATCH_MODEL: dict[str, Any] = { - 'dimensions': {'snapshot': {'dtype': 'int'}, 'generator': {'dtype': 'str'}}, + 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'generator': {'dtype': 'str'}}, 'parameters': { 'p_max': {'dims': ['generator']}, 'cost': {'dims': ['generator']}, @@ -48,7 +48,7 @@ #: a rule can name, and no objective, so a test adds what it judges. `p` and `r` #: share no dimension, which is what a rule about *different* dims needs. SMALL_MODEL: dict[str, Any] = { - 'dimensions': {'g': {'dtype': 'str'}, 'h': {'dtype': 'str'}}, + 'dimensions': {'g': {'dtype': 'str', 'ordered': True}, 'h': {'dtype': 'str', 'ordered': True}}, 'relations': {'lk': {'key': 'g', 'values': 'h'}}, 'parameters': { 'c': {'dims': ['g']}, diff --git a/tests/fixtures/every_program_node.yaml b/tests/fixtures/every_program_node.yaml index bc541ed87..711da00c0 100644 --- a/tests/fixtures/every_program_node.yaml +++ b/tests/fixtures/every_program_node.yaml @@ -4,7 +4,7 @@ description: every node a program can carry, in one file that lowers dimensions: - t: { dtype: int } + t: { dtype: int, ordered: true } g: { dtype: str } zone: { dtype: str } relations: diff --git a/tests/test_advice.py b/tests/test_advice.py index f34f86549..61b37ec8a 100644 --- a/tests/test_advice.py +++ b/tests/test_advice.py @@ -38,7 +38,7 @@ CURVED = varied( UNREACHED, objective={'sense': 'minimize', 'expression': 'sum(p)'}, - dimensions={'g': {'dtype': 'str'}, 'h': {'dtype': 'str'}, 'bp': {'dtype': 'int'}}, + dimensions={'g': {'dtype': 'str'}, 'h': {'dtype': 'str'}, 'bp': {'dtype': 'int', 'ordered': True}}, parameters={'c': {'dims': ['g']}, 'bp_x': {'dims': ['bp']}, 'bp_y': {'dims': ['bp']}}, variables={'p': {'dims': ['g']}, 'cost': {'dims': ['g']}}, piecewise={'curve': {'over': 'bp', 'links': [['p', 'bp_x'], ['cost', 'bp_y']]}}, diff --git a/tests/test_boundedness.py b/tests/test_boundedness.py index e0549a872..4ebcbbe16 100644 --- a/tests/test_boundedness.py +++ b/tests/test_boundedness.py @@ -103,7 +103,7 @@ def test_a_named_constant_coefficient_carries_its_sign(objective, side): ), pytest.param( { - 'dimensions.bp': {'dtype': 'int'}, + 'dimensions.bp': {'dtype': 'int', 'ordered': True}, 'parameters.bp_x': {'dims': ['bp']}, 'parameters.bp_y': {'dims': ['bp']}, 'piecewise': {'curve': {'over': 'bp', 'links': [['v', 'bp_x'], ['w', 'bp_y']]}}, diff --git a/tests/test_canonical.py b/tests/test_canonical.py index 2b645f47e..54edf0e34 100644 --- a/tests/test_canonical.py +++ b/tests/test_canonical.py @@ -491,14 +491,20 @@ def test_the_form_declares_the_same_spec(path): def test_a_named_expression_written_on_one_line_is_normalised_too(): """`name: a + b` serialises back as a bare string, which the form passed through as written.""" - frame = {'dimensions': {'t': {'dtype': 'int'}}, 'variables': {'a': {'dims': ['t']}, 'b': {'dims': ['t']}}} + frame = { + 'dimensions': {'t': {'dtype': 'int', 'ordered': True}}, + 'variables': {'a': {'dims': ['t']}, 'b': {'dims': ['t']}}, + } one, other = ({**frame, 'expressions': {'total': text}} for text in ('a + b', 'b + a')) assert ms.to_spec(one).to_yaml(canonical=True) == ms.to_spec(other).to_yaml(canonical=True) def test_the_names_a_file_reads_are_sorted_like_the_names_it_declares(): """`given:` nests its kinds one level below a section, so sorting the sections alone left them in file order.""" - frame = {'dimensions': {'t': {'dtype': 'int'}}, 'constraints': {'c': {'dims': ['t'], 'expression': 'a + b >= 0'}}} + frame = { + 'dimensions': {'t': {'dtype': 'int', 'ordered': True}}, + 'constraints': {'c': {'dims': ['t'], 'expression': 'a + b >= 0'}}, + } one, other = ( {**frame, 'given': {'variables': dict(entries)}} for entries in ( diff --git a/tests/test_composition.py b/tests/test_composition.py index b1849c7bc..feb0b4593 100644 --- a/tests/test_composition.py +++ b/tests/test_composition.py @@ -206,6 +206,23 @@ def test_a_peer_s_description_is_carried_where_the_first_one_given_has_none(): ) +def _ordered(fragment: dict[str, object]) -> dict[str, object]: + """*fragment* with ``snapshot`` declared ordered.""" + return {**fragment, 'dimensions': {**fragment['dimensions'], 'snapshot': {'dtype': 'int', 'ordered': True}}} + + +@pytest.mark.parametrize( + 'fragments', + [ + pytest.param([SUPPLY, _ordered(DEMAND)], id='the-second-says-ordered'), + pytest.param([_ordered(SUPPLY), DEMAND], id='the-first-says-ordered'), + ], +) +def test_a_dimension_one_fragment_declares_ordered_is_ordered_in_the_composition(fragments): + """Each fragment loads alone, and two that differed only in ``ordered`` were refused as saying different things.""" + assert merge(fragments).dimensions['snapshot'].ordered, 'ordered is a claim one fragment adds to the space' + + @pytest.mark.parametrize( ('owner', 'carried'), [ @@ -489,13 +506,33 @@ def test_a_later_patch_is_laid_on_what_the_earlier_ones_make(patches, field, val def test_a_patch_adds_a_dimension_and_may_restate_one_it_shares(): laid = override( DISPATCH_MODEL, - [{'dimensions': {'snapshot': {'dtype': 'int'}, 'investment_period': {'dtype': 'int'}}}], + [{'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'investment_period': {'dtype': 'int'}}}], ) assert sorted(laid.dimensions) == ['generator', 'investment_period', 'snapshot'], ( 'the dimension the patch adds joins the two the base declares, and the restated one is not doubled' ) +@pytest.mark.parametrize( + ('dimension', 'restated', 'ordered'), + [ + pytest.param('generator', {'dtype': 'str', 'ordered': False}, False, id='false-written-out'), + pytest.param('generator', {'dtype': 'str', 'ordered': True}, True, id='widened-to-ordered'), + pytest.param('snapshot', {'dtype': 'int'}, True, id='ordered-left-out'), + ], +) +def test_a_patch_restates_a_dimension_as_the_schema_reads_it(dimension, restated, ordered): + """A written ``ordered: false`` differed from the omitted one, and a patch could not add the claim.""" + laid = override(DISPATCH_MODEL, [{'dimensions': {dimension: restated}}]) + assert laid.dimensions[dimension].ordered is ordered + + +def test_a_patch_that_withdraws_ordered_is_refused(): + with pytest.raises(LanguageError, match=r"says the dimension 'snapshot' is not ordered") as raised: + override(DISPATCH_MODEL, [{'dimensions': {'snapshot': {'dtype': 'int', 'ordered': False}}}]) + assert 'leave `ordered` out of the patch' in str(raised.value), 'the refusal names the rewrite' + + @pytest.mark.parametrize( ('patch', 'says'), [ diff --git a/tests/test_dimensions.py b/tests/test_dimensions.py index b30a56907..845b7e775 100644 --- a/tests/test_dimensions.py +++ b/tests/test_dimensions.py @@ -28,8 +28,8 @@ #: over a dim `p` does not carry, so it is readable only through a `by=`. BASE = { 'dimensions': { - 'snapshot': {'dtype': 'int'}, - 'generator': {'dtype': 'str'}, + 'snapshot': {'dtype': 'int', 'ordered': True}, + 'generator': {'dtype': 'str', 'ordered': True}, 'bus': {'dtype': 'str'}, 'zone': {'dtype': 'str'}, }, @@ -498,7 +498,7 @@ class TestTheEdgeRulesAreDecidedAtLoad: """ BASE: ClassVar[dict[str, Any]] = { - 'dimensions': {'t': {'dtype': 'int'}, 'g': {'dtype': 'str'}}, + 'dimensions': {'t': {'dtype': 'int', 'ordered': True}, 'g': {'dtype': 'str', 'ordered': True}}, 'parameters': {'cap': {'dims': ['g']}, 'lead': {'dims': ['g'], 'dtype': 'int'}}, 'variables': {'p': {'dims': ['t', 'g'], 'bounds': {'lower': 0, 'upper': 1}}}, 'constraints': {'k': {'dims': ['t', 'g'], 'expression': 'p <= 1'}}, diff --git a/tests/test_exclusivity.py b/tests/test_exclusivity.py index f7cc3d92b..3b6f592cf 100644 --- a/tests/test_exclusivity.py +++ b/tests/test_exclusivity.py @@ -31,8 +31,8 @@ #: Every axis takes its coordinates from data, so nothing here sizes one. STORAGE: dict[str, Any] = { 'dimensions': { - 'snapshot': {'dtype': 'int'}, - 'storage': {}, + 'snapshot': {'dtype': 'int', 'ordered': True}, + 'storage': {'ordered': True}, 'period': {'dtype': 'int'}, }, 'relations': {'period_of': {'key': 'snapshot', 'values': 'period'}}, diff --git a/tests/test_lowering.py b/tests/test_lowering.py index 37924a0d6..ad19cd47d 100644 --- a/tests/test_lowering.py +++ b/tests/test_lowering.py @@ -74,7 +74,7 @@ #: the smallest model that loads, for a claim about the plan's record rather #: than about the math in it. A test adds what it judges with :func:`varied`. TINY = { - 'dimensions': {'g': {}}, + 'dimensions': {'g': {'ordered': True}}, 'parameters': {'cost': {'dims': ['g']}}, 'variables': {'p': {'dims': ['g'], 'bounds': {'lower': 0, 'upper': 1}}}, 'constraints': {'c': {'dims': [], 'expression': 'sum(p, over=g) >= 1'}}, @@ -164,9 +164,9 @@ def test_a_file_with_no_objective_lowers_to_no_sense(): assert program.objective is None, 'no objective declared is no objective, not a minimisation of nothing' -def test_a_literal_amount_resolves_to_one_signed_number(dispatch_schema): +def test_a_literal_amount_resolves_to_one_signed_number(): """`offset=-1` parses as a unary minus over `1`; after resolution it is `-1`, for every reader alike.""" - ns = Namespace(dispatch_schema) + ns = Namespace(schema_of(DISPATCH_YAML, **{'dimensions.snapshot': {'dtype': 'int', 'ordered': True}})) node = expression_of('shift(dispatch, along=snapshot, offset=-1, edge=+0)', ns, 't') assert isinstance(node, Translate) assert (node.offset, node.fill) == (-1, 0.0) @@ -655,7 +655,7 @@ def test_a_partition_keeps_its_group_when_the_relation_gains_a_value_column(): for values in ('day', ['day', 'week']): program = to_spec( { - 'dimensions': {'hour': {'dtype': 'int'}, 'day': {}, 'week': {}}, + 'dimensions': {'hour': {'dtype': 'int', 'ordered': True}, 'day': {}, 'week': {}}, 'relations': {'cal': {'key': 'hour', 'values': values}}, 'variables': {'p': {'dims': ['hour']}}, 'constraints': { @@ -683,7 +683,7 @@ def test_a_relation_lowers_with_the_join_each_call_names(): """Every node reading a relation carries its columns, its key and the join, so a consumer joins on the right columns.""" program = to_spec( { - 'dimensions': {'snapshot': {'dtype': 'int'}, 'generator': {}, 'zone': {}}, + 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'generator': {'ordered': True}, 'zone': {}}, 'relations': {'zone_of': {'key': ['generator', 'snapshot'], 'values': 'zone'}}, 'parameters': {'price': {'dims': ['snapshot', 'zone']}}, 'variables': { @@ -836,6 +836,14 @@ def test_a_relation_is_declared_as_the_file_declares_it(): assert program.dimensions['g'] == DimensionDeclaration(dtype='str'), 'a dimension carries its dtype and no relation' +@pytest.mark.parametrize('ordered', [pytest.param(True, id='ordered'), pytest.param(False, id='unordered')]) +def test_a_program_reports_whether_a_dimension_is_ordered(ordered: bool): + """The program dropped ``ordered``, so a consumer of it read every dimension as unordered.""" + program = to_spec(varied(TINY, dimensions={'g': {'ordered': ordered}})).program + + assert program.dimensions['g'].ordered is ordered + + def test_a_program_is_built_by_keyword_so_a_field_added_later_cannot_reorder_an_old_call(): """Positional construction made every field's *position* part of the contract.""" with pytest.raises(TypeError, match='positional'): @@ -946,7 +954,7 @@ def test_a_dimension_carries_the_dtype_its_labels_are_checked_against(): CASED = { - 'dimensions': {'t': {'dtype': 'int'}, 'g': {'dtype': 'str'}}, + 'dimensions': {'t': {'dtype': 'int', 'ordered': True}, 'g': {'dtype': 'str', 'ordered': True}}, 'parameters': {'committable': {'dims': ['g'], 'dtype': 'bool'}, 'initial': {'dims': ['g']}}, 'variables': {'status': {'dims': ['t', 'g'], 'domain': 'binary'}}, 'expressions': { @@ -1124,7 +1132,7 @@ def test_a_lowered_spec_still_pickles_and_lowers_to_the_same_program(): spec = Spec.model_validate( { - 'dimensions': {'t': {'dtype': 'int'}, 'g': {'dtype': 'str'}}, + 'dimensions': {'t': {'dtype': 'int', 'ordered': True}, 'g': {'dtype': 'str', 'ordered': True}}, 'parameters': {'load': {'dims': ['t']}, 'cost': {'dims': ['g']}}, 'variables': {'p': {'dims': ['t', 'g'], 'bounds': {'lower': 0}}}, 'constraints': {'balance': {'dims': ['t'], 'expression': 'sum(p, over=g) >= load'}}, @@ -1152,7 +1160,7 @@ def test_a_lowered_program_pickles_and_is_the_same_program(): program = to_spec( { - 'dimensions': {'t': {'dtype': 'int'}, 'g': {'dtype': 'str'}}, + 'dimensions': {'t': {'dtype': 'int', 'ordered': True}, 'g': {'dtype': 'str', 'ordered': True}}, 'parameters': {'load': {'dims': ['t']}, 'cost': {'dims': ['g']}}, 'variables': {'p': {'dims': ['t', 'g'], 'bounds': {'lower': 0}}}, 'constraints': {'balance': {'dims': ['t'], 'expression': 'sum(p, over=g) >= load'}}, @@ -1176,7 +1184,7 @@ def test_two_groups_of_a_program_merge_with_or_as_they_did_behind_the_proxy(): where the seal answered `|` with a `TypeError`.""" program = to_spec( { - 'dimensions': {'t': {'dtype': 'int'}}, + 'dimensions': {'t': {'dtype': 'int', 'ordered': True}}, 'parameters': {'load': {'dims': ['t']}}, 'variables': {'p': {'dims': ['t'], 'bounds': {'lower': 0}}}, 'constraints': {'meet': {'dims': ['t'], 'expression': 'p >= load'}}, diff --git a/tests/test_piecewise.py b/tests/test_piecewise.py index 9ff50e544..4c2cfae75 100644 --- a/tests/test_piecewise.py +++ b/tests/test_piecewise.py @@ -25,8 +25,8 @@ #: binaries and links is not something a smaller one can stand in for. NONCONVEX_YAML = """ dimensions: - snapshot: {dtype: int} - bp: {dtype: int} + snapshot: {dtype: int, ordered: true} + bp: {dtype: int, ordered: true} parameters: load: {dims: [snapshot]} diff --git a/tests/test_separability.py b/tests/test_separability.py index 0287cba02..ab912e514 100644 --- a/tests/test_separability.py +++ b/tests/test_separability.py @@ -24,7 +24,12 @@ FIXTURE = Path(__file__).resolve().parent / 'fixtures' / 'every_program_node.yaml' BASE: dict[str, Any] = { - 'dimensions': {'h': {'dtype': 'int'}, 'u': {'dtype': 'str'}, 'zone': {'dtype': 'str'}, 'day': {'dtype': 'int'}}, + 'dimensions': { + 'h': {'dtype': 'int', 'ordered': True}, + 'u': {'dtype': 'str', 'ordered': True}, + 'zone': {'dtype': 'str'}, + 'day': {'dtype': 'int'}, + }, 'relations': {'zone_of': {'key': 'u', 'values': 'zone'}, 'day_of': {'key': 'h', 'values': 'day'}}, 'parameters': { 'cost': {'dims': ['u']}, diff --git a/tests/test_sos.py b/tests/test_sos.py index 9da45d48a..7f15d593f 100644 --- a/tests/test_sos.py +++ b/tests/test_sos.py @@ -29,7 +29,7 @@ #: The curve of `examples/sos.yaml`, as a dict a test can vary. CURVE = { - 'dimensions': {'snapshot': {'dtype': 'int'}, 'bp': {'dtype': 'int'}}, + 'dimensions': {'snapshot': {'dtype': 'int'}, 'bp': {'dtype': 'int', 'ordered': True}}, 'parameters': {'load': {'dims': ['snapshot']}, 'bp_x': {'dims': ['bp']}, 'bp_y': {'dims': ['bp']}}, 'variables': { 'p': {'dims': ['snapshot'], 'bounds': {'lower': 0, 'upper': 100}}, diff --git a/tests/test_validation.py b/tests/test_validation.py index 96ae82e08..0a46cd357 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -211,7 +211,7 @@ def _kwarg_model(expression: str, dims: list[str] | None = None) -> dict[str, An """ return { 'dimensions': { - 'snapshot': {'dtype': 'int'}, + 'snapshot': {'dtype': 'int', 'ordered': True}, 'bus': {'dtype': 'str'}, 'generator': {'dtype': 'str'}, }, @@ -552,7 +552,7 @@ def test_the_version_gates_no_behaviour(self): #: `position(dim)` needs a relation over *that* dimension, so one over it and one into it. POSITION_SCHEMA = to_spec( { - 'dimensions': {'snapshot': {'dtype': 'int'}, 'period': {'dtype': 'int'}}, + 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'period': {'dtype': 'int'}}, 'relations': { 'period_of': {'key': 'snapshot', 'values': 'period'}, 'starts_at': {'key': 'period', 'values': 'snapshot'}, @@ -1854,7 +1854,7 @@ def test_a_default_is_written_out_and_an_absence_is_not(self): #: A model with room for a cased expression: two dimensions, so an arm can be #: narrower than the frame, and a variable, so an arm can reach one. CASED_BASE = { - 'dimensions': {'snapshot': {'dtype': 'int'}, 'generator': {}}, + 'dimensions': {'snapshot': {'dtype': 'int', 'ordered': True}, 'generator': {}}, 'parameters': {'p_max': {'dims': ['generator']}, 'load': {'dims': ['snapshot']}}, 'variables': {'p': {'dims': ['snapshot', 'generator']}}, } @@ -2253,7 +2253,7 @@ def record(*args, **kwargs): 'otherwise': 'p_max - p', }, 'constraints.spare': {'dims': ['snapshot', 'generator'], 'expression': 'p <= headroom'}, - 'dimensions.bp': {'dtype': 'int'}, + 'dimensions.bp': {'dtype': 'int', 'ordered': True}, 'parameters.bp_x': {'dims': ['generator', 'bp']}, 'parameters.bp_y': {'dims': ['generator', 'bp']}, 'variables.op_cost': {'dims': ['snapshot', 'generator'], 'bounds': {'lower': 0}}, @@ -2330,3 +2330,104 @@ def test_an_infinite_bound_is_refused_with_the_null_that_opens_a_side(side, valu message = _refusal(DISPATCH_MODEL, **{f'variables.p.bounds.{side}': value}) assert f'bounds.{side} is {value}, and a bound is finite' in message assert f'{side}: null' in message, 'the refusal names the spelling of an open side' + + +#: One dimension every construct below reads the order of, and a breakpoint +#: dimension for the two formulations; neither declares its order. +UNORDERED: dict[str, Any] = { + 'dimensions': {'t': {'dtype': 'int'}, 'bp': {'dtype': 'int'}}, + 'parameters': { + 'on': {'dims': ['t'], 'dtype': 'bool'}, + 'bp_x': {'dims': ['bp']}, + 'bp_y': {'dims': ['bp']}, + }, + 'variables': { + 'x': {'dims': ['t'], 'bounds': {'lower': 0, 'upper': 1}}, + 'y': {'dims': ['t'], 'bounds': {'lower': 0, 'upper': 1}}, + 'z': {'dims': ['bp'], 'bounds': {'lower': 0, 'upper': 1}}, + }, +} + +_READS_ORDER = [ + pytest.param( + {'constraints.c': {'dims': ['t'], 'expression': 'x <= shift(x, along=t, offset=1, edge=0)'}}, + 't', + id='shift', + ), + pytest.param( + {'constraints.c': {'dims': ['t'], 'expression': 'x <= sum_back(y, along=t, window=2)'}}, + 't', + id='sum_back', + ), + pytest.param( + {'constraints.c': {'dims': ['t'], 'where': 'shift(on, along=t, offset=1)', 'expression': 'x <= 0'}}, + 't', + id='a-where-shift', + ), + pytest.param( + {'constraints.c': {'dims': ['t'], 'where': 'position(t) == 0', 'expression': 'x <= 0'}}, + 't', + id='position', + ), + pytest.param( + {'piecewise.curve': {'over': 'bp', 'links': [['x', 'bp_x'], ['y', 'bp_y']]}}, + 'bp', + id='a-piecewise-curve', + ), + pytest.param({'sos.s': {'variable': 'z', 'along': 'bp', 'type': 2}}, 'bp', id='a-type-2-set'), +] + + +@pytest.mark.parametrize(('patch', 'dimension'), _READS_ORDER) +def test_a_construct_that_reads_order_along_an_unordered_dimension_is_refused(patch, dimension): + """The order an unordered dimension has is the data's row order, which is not part of the model.""" + message = _refusal(UNORDERED, **patch) + assert f"reads the order of '{dimension}', which is not declared ordered" in message + assert f"'{dimension}: {{ordered: true}}' under dimensions:" in message, 'the refusal names the rewrite' + + +@pytest.mark.parametrize(('patch', 'dimension'), _READS_ORDER) +def test_the_same_construct_along_an_ordered_dimension_loads(patch, dimension): + ordered = varied(UNORDERED, **{f'dimensions.{dimension}.ordered': True}) + to_spec(varied(ordered, **patch)) + + +def test_a_where_shift_along_a_dimension_its_predicate_lacks_is_refused_for_that_first(): + """The order refusal came first, so a file that declared ``bp`` ordered met a second refusal after it.""" + where = {'constraints.c': {'dims': ['t'], 'where': 'shift(on, along=bp, offset=1)', 'expression': 'x <= 0'}} + message = _refusal(UNORDERED, **where) + assert 'reads the predicate back along a dimension it does not carry' in message + assert 'not declared ordered' not in message, 'the predicate cannot be read along bp, ordered or not' + + +def test_a_type_1_set_reads_no_order(): + """At most one member is nonzero, whichever it is, so the set says nothing about neighbours.""" + to_spec(varied(UNORDERED, **{'sos.s': {'variable': 'z', 'along': 'bp', 'type': 1}})) + + +@pytest.mark.parametrize( + ('ordered', 'written'), + [ + pytest.param(False, {'dtype': 'int'}, id='unordered'), + pytest.param(True, {'dtype': 'int', 'ordered': True}, id='ordered'), + ], +) +def test_a_dimension_writes_ordered_only_where_it_is_true(ordered, written): + """Leaving it out is what `false` says, so a file that never wrote it round-trips unchanged.""" + spec = to_spec({'dimensions': {'t': {'dtype': 'int', 'ordered': ordered}}}) + assert spec.to_dict()['dimensions'] == {'t': written} + assert to_spec(spec.to_dict()) == spec + + +@pytest.mark.parametrize( + 'dump', + [ + pytest.param({'exclude_defaults': True}, id='exclude_defaults'), + pytest.param({'exclude_unset': True}, id='exclude_unset'), + pytest.param({'exclude': {'dimensions': {'t': {'ordered'}}}}, id='exclude'), + ], +) +def test_a_dump_that_leaves_ordered_out_still_writes_the_dimension(dump): + """The serializer dropped `ordered` with `del`, so a dump that had already left it out raised a KeyError.""" + spec = to_spec({'dimensions': {'t': {'dtype': 'int'}}}) + assert spec.model_dump(**dump)['dimensions'] == {'t': {'dtype': 'int'}} diff --git a/tests/typesetting/golden/model.yaml b/tests/typesetting/golden/model.yaml index 9f39c0300..259a4b74d 100644 --- a/tests/typesetting/golden/model.yaml +++ b/tests/typesetting/golden/model.yaml @@ -13,13 +13,13 @@ description: >- $5 {net} ~ ^ \ *star* @ref