From c4b4890fb1add06cdcb9bd53c8c7c44f4f7a5e9b Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Thu, 3 Sep 2026 15:09:54 -0700 Subject: [PATCH] Merge UcieTL modules into single UcieComplex module --- .github/workflows/ci.yml | 23 ++++-- software/ucie.h | 146 +++++++++++++++++++-------------------- src/GenUcieHeader.scala | 17 ++--- src/Iris.scala | 29 ++++---- src/UcieComplex.scala | 83 ++++++++++++++++++++++ ucie | 2 +- 6 files changed, 192 insertions(+), 108 deletions(-) create mode 100644 src/UcieComplex.scala diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b30a58..fa03246 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,19 +119,28 @@ jobs: ./mill test.testOnly edu.berkeley.cs.iris.IrisSpec \ -- -t "TinyIris should generate valid System Verilog for TinyIris" - # PD hardens the two UCIe blocks separately, so they have to stay two - # SystemVerilog modules. They fold back into one as soon as their - # parameters -- orientation included -- match, and nothing else in the - # flow would notice. - - name: Check the UCIe blocks did not dedup + # PD hardens the two UCIe blocks as one hierarchical block, so both links + # have to sit inside a single UcieComplex module and share one UcieTL. + # Giving a port its own UcieTLParams.moduleId splits them again, and + # nothing else in the flow would notice. + - name: Check the UCIe blocks are grouped and deduped run: | out=build/Iris_should_generate_valid_System_Verilog n=$(grep -lE '^module UcieTL(_[A-Za-z0-9]+)?\(' "$out"/*.sv | wc -l) - if [ "$n" -ne 2 ]; then - echo "::error::expected 2 UcieTL modules in $out, found $n" + if [ "$n" -ne 1 ]; then + echo "::error::expected 1 UcieTL module in $out, found $n" grep -hE '^module UcieTL(_[A-Za-z0-9]+)?\(' "$out"/*.sv || true exit 1 fi + if [ ! -f "$out/UcieComplex.sv" ]; then + echo "::error::expected a UcieComplex wrapper module in $out" + exit 1 + fi + links=$(grep -cE '^ *UcieChipletLink(_[A-Za-z0-9]+)? ' "$out/UcieComplex.sv" || true) + if [ "$links" -ne 2 ]; then + echo "::error::expected 2 UCIe links inside UcieComplex, found $links" + exit 1 + fi # Catches a ucie bump (or a UCIe parameter change) that moves the # register map without the checked-in header following it. diff --git a/software/ucie.h b/software/ucie.h index 0d8d5be..a01c8fe 100644 --- a/software/ucie.h +++ b/software/ucie.h @@ -12,53 +12,52 @@ // === Register offsets (relative to the UCIe MMIO base, e.g. 0x8000) === #define UCIE_TEST_TARGET 0x0ULL -#define UCIE_DIV_RESETB 0x8ULL -#define UCIE_TX_TEST_MODE 0x10ULL -#define UCIE_TX_DATA_MODE 0x18ULL -#define UCIE_TX_LFSR_SEED 0x20ULL +#define UCIE_TX_TEST_MODE 0x8ULL +#define UCIE_TX_DATA_MODE 0x10ULL +#define UCIE_TX_LFSR_SEED 0x18ULL #define UCIE_TX_LFSR_SEED_WIDTH 0x8ULL -#define UCIE_TX_RST 0xb8ULL -#define UCIE_TX_EXECUTE 0xc0ULL -#define UCIE_TX_WRITE_CHUNK 0xc8ULL -#define UCIE_TX_PACKETS_SENT 0xd0ULL -#define UCIE_TX_MANUAL_REPEAT_PERIOD 0xd8ULL -#define UCIE_TX_PACKETS_TO_SEND 0xe0ULL -#define UCIE_TX_CLK_P 0xe8ULL -#define UCIE_TX_CLK_N 0xf0ULL -#define UCIE_TX_DATA_LANE_GROUP 0xf8ULL -#define UCIE_TX_DATA_OFFSET 0x100ULL -#define UCIE_TX_DATA_CHUNK_IN0 0x108ULL -#define UCIE_TX_DATA_CHUNK_IN1 0x110ULL -#define UCIE_TX_DATA_CHUNK_OUT0 0x118ULL -#define UCIE_TX_DATA_CHUNK_OUT1 0x120ULL -#define UCIE_TX_TEST_STATE 0x128ULL -#define UCIE_RX_DATA_MODE 0x130ULL -#define UCIE_RX_LFSR_SEED 0x138ULL +#define UCIE_TX_RST 0xb0ULL +#define UCIE_TX_EXECUTE 0xb8ULL +#define UCIE_TX_WRITE_CHUNK 0xc0ULL +#define UCIE_TX_PACKETS_SENT 0xc8ULL +#define UCIE_TX_MANUAL_REPEAT_PERIOD 0xd0ULL +#define UCIE_TX_PACKETS_TO_SEND 0xd8ULL +#define UCIE_TX_CLK_P 0xe0ULL +#define UCIE_TX_CLK_N 0xe8ULL +#define UCIE_TX_DATA_LANE_GROUP 0xf0ULL +#define UCIE_TX_DATA_OFFSET 0xf8ULL +#define UCIE_TX_DATA_CHUNK_IN0 0x100ULL +#define UCIE_TX_DATA_CHUNK_IN1 0x108ULL +#define UCIE_TX_DATA_CHUNK_OUT0 0x110ULL +#define UCIE_TX_DATA_CHUNK_OUT1 0x118ULL +#define UCIE_TX_TEST_STATE 0x120ULL +#define UCIE_RX_DATA_MODE 0x128ULL +#define UCIE_RX_LFSR_SEED 0x130ULL #define UCIE_RX_LFSR_SEED_WIDTH 0x8ULL -#define UCIE_RX_BIT_ERRORS 0x1d0ULL +#define UCIE_RX_BIT_ERRORS 0x1c8ULL #define UCIE_RX_BIT_ERRORS_WIDTH 0x8ULL -#define UCIE_RX_BIT_ERRORS_EARLY 0x268ULL +#define UCIE_RX_BIT_ERRORS_EARLY 0x260ULL #define UCIE_RX_BIT_ERRORS_EARLY_WIDTH 0x8ULL -#define UCIE_RX_BIT_ERRORS_LATE 0x300ULL +#define UCIE_RX_BIT_ERRORS_LATE 0x2f8ULL #define UCIE_RX_BIT_ERRORS_LATE_WIDTH 0x8ULL -#define UCIE_RX_RST 0x398ULL -#define UCIE_RX_PACKETS_TO_RECEIVE 0x3a0ULL -#define UCIE_RX_PAUSE_COUNTERS 0x3a8ULL -#define UCIE_RX_PACKETS_RECEIVED 0x3b0ULL -#define UCIE_RX_SIGNATURE 0x3b8ULL -#define UCIE_RX_DATA_LANE 0x3c0ULL -#define UCIE_RX_DATA_OFFSET 0x3c8ULL -#define UCIE_RX_DATA_CHUNK 0x3d0ULL -#define UCIE_CLK_PHASE_SEL 0x3d8ULL -#define UCIE_CLK_FREQ_SEL 0x3e0ULL -#define UCIE_TXCTL 0x3e8ULL +#define UCIE_RX_RST 0x390ULL +#define UCIE_RX_PACKETS_TO_RECEIVE 0x398ULL +#define UCIE_RX_PAUSE_COUNTERS 0x3a0ULL +#define UCIE_RX_PACKETS_RECEIVED 0x3a8ULL +#define UCIE_RX_SIGNATURE 0x3b0ULL +#define UCIE_RX_DATA_LANE 0x3b8ULL +#define UCIE_RX_DATA_OFFSET 0x3c0ULL +#define UCIE_RX_DATA_CHUNK 0x3c8ULL +#define UCIE_CLK_PHASE_SEL 0x3d0ULL +#define UCIE_CLK_FREQ_SEL 0x3d8ULL +#define UCIE_TXCTL 0x3e0ULL #define UCIE_TXCTL_TILE_OFS 0x0ULL #define UCIE_TXCTL_SHUFFLER_OFS 0x8ULL #define UCIE_TXCTL_SHUFFLER_WIDTH 0x8ULL #define UCIE_TXCTL_SAMPLE_NEGEDGE_OFS 0x108ULL #define UCIE_TXCTL_DELAY_OFS 0x110ULL #define UCIE_TXCTL_WIDTH 0x118ULL -#define UCIE_RXCTL 0x1ae0ULL +#define UCIE_RXCTL 0x1ad8ULL #define UCIE_RXCTL_ZEN_OFS 0x0ULL #define UCIE_RXCTL_ZCTL_OFS 0x8ULL #define UCIE_RXCTL_VREF_SEL_OFS 0x10ULL @@ -71,46 +70,46 @@ #define UCIE_RXCTL_SAMPLE_NEGEDGE_OFS 0x138ULL #define UCIE_RXCTL_RX_DELAY_OFS 0x140ULL #define UCIE_RXCTL_WIDTH 0x148ULL -#define UCIE_DEBUG_TX_TEST_MODE 0x35c8ULL -#define UCIE_DEBUG_TX_DATA_MODE 0x35d0ULL -#define UCIE_DEBUG_TX_LFSR_SEED 0x35d8ULL -#define UCIE_DEBUG_TX_FSM_RST 0x35e0ULL -#define UCIE_DEBUG_TX_EXECUTE 0x35e8ULL -#define UCIE_DEBUG_TX_MANUAL_REPEAT_PERIOD 0x35f0ULL -#define UCIE_DEBUG_TX_PACKETS_TO_SEND 0x35f8ULL -#define UCIE_DEBUG_DATA 0x3600ULL +#define UCIE_DEBUG_TX_TEST_MODE 0x35c0ULL +#define UCIE_DEBUG_TX_DATA_MODE 0x35c8ULL +#define UCIE_DEBUG_TX_LFSR_SEED 0x35d0ULL +#define UCIE_DEBUG_TX_FSM_RST 0x35d8ULL +#define UCIE_DEBUG_TX_EXECUTE 0x35e0ULL +#define UCIE_DEBUG_TX_MANUAL_REPEAT_PERIOD 0x35e8ULL +#define UCIE_DEBUG_TX_PACKETS_TO_SEND 0x35f0ULL +#define UCIE_DEBUG_DATA 0x35f8ULL #define UCIE_DEBUG_DATA_WIDTH 0x8ULL -#define UCIE_DEBUG_DRIVERCTL 0x3680ULL +#define UCIE_DEBUG_DRIVERCTL 0x3678ULL #define UCIE_DEBUG_DRIVERCTL_WIDTH 0x8ULL -#define UCIE_DEBUG_TXCTL_TILE 0x36a0ULL -#define UCIE_DEBUG_TXCTL_SHUFFLER 0x36a8ULL +#define UCIE_DEBUG_TXCTL_TILE 0x3698ULL +#define UCIE_DEBUG_TXCTL_SHUFFLER 0x36a0ULL #define UCIE_DEBUG_TXCTL_SHUFFLER_WIDTH 0x8ULL -#define UCIE_DEBUG_TX_TEST_STATE 0x37a8ULL -#define UCIE_DEBUG_TX_PACKETS_SENT 0x37b0ULL -#define UCIE_DEBUG_CLK_MUX_SEL 0x37b8ULL -#define UCIE_DEBUG_RX_LANE 0x37c0ULL -#define UCIE_DEBUG_RX_BIT 0x37c8ULL -#define UCIE_TX_VALID 0x37d0ULL -#define UCIE_RX_LFSR_VALID 0x37d8ULL -#define UCIE_CONTROLLER_SEL 0x37e0ULL -#define UCIE_MAINBAND_MODE 0x37e8ULL -#define UCIE_SIDEBAND_MODE 0x37f0ULL -#define UCIE_CREDIT_FLOW_ENABLE 0x37f8ULL -#define UCIE_TX_VALID_LANE_SEL 0x3800ULL -#define UCIE_RX_VALID_LANE_SEL 0x3808ULL -#define UCIE_LAST_SEEN_TL_REQ 0x3810ULL -#define UCIE_SB_TX_PACKET 0x3818ULL -#define UCIE_SB_TX_SEND 0x3820ULL -#define UCIE_SB_TX_BUSY 0x3828ULL -#define UCIE_SB_RX_PACKET 0x3830ULL -#define UCIE_SB_RX_VALID 0x3838ULL -#define UCIE_SB_RX_POP 0x3840ULL -#define UCIE_SB_RX_OVERFLOW 0x3848ULL -#define UCIE_SB_RX_RST 0x3850ULL -#define UCIE_SB_TL_RX_OVERFLOW 0x3858ULL -#define UCIE_UCIE_LINK_RESET 0x3860ULL -#define UCIE_UCIE_PWR_GOOD 0x3868ULL -#define UCIE_UCIE_RETRY_TRAINING_AMT 0x3870ULL +#define UCIE_DEBUG_TX_TEST_STATE 0x37a0ULL +#define UCIE_DEBUG_TX_PACKETS_SENT 0x37a8ULL +#define UCIE_DEBUG_CLK_MUX_SEL 0x37b0ULL +#define UCIE_DEBUG_RX_LANE 0x37b8ULL +#define UCIE_DEBUG_RX_BIT 0x37c0ULL +#define UCIE_TX_VALID 0x37c8ULL +#define UCIE_RX_LFSR_VALID 0x37d0ULL +#define UCIE_CONTROLLER_SEL 0x37d8ULL +#define UCIE_MAINBAND_MODE 0x37e0ULL +#define UCIE_SIDEBAND_MODE 0x37e8ULL +#define UCIE_CREDIT_FLOW_ENABLE 0x37f0ULL +#define UCIE_TX_VALID_LANE_SEL 0x37f8ULL +#define UCIE_RX_VALID_LANE_SEL 0x3800ULL +#define UCIE_LAST_SEEN_TL_REQ 0x3808ULL +#define UCIE_SB_TX_PACKET 0x3810ULL +#define UCIE_SB_TX_SEND 0x3818ULL +#define UCIE_SB_TX_BUSY 0x3820ULL +#define UCIE_SB_RX_PACKET 0x3828ULL +#define UCIE_SB_RX_VALID 0x3830ULL +#define UCIE_SB_RX_POP 0x3838ULL +#define UCIE_SB_RX_OVERFLOW 0x3840ULL +#define UCIE_SB_RX_RST 0x3848ULL +#define UCIE_SB_TL_RX_OVERFLOW 0x3850ULL +#define UCIE_UCIE_LINK_RESET 0x3858ULL +#define UCIE_UCIE_PWR_GOOD 0x3860ULL +#define UCIE_UCIE_RETRY_TRAINING_AMT 0x3868ULL // === Constants === #define UCIE_TX_TEST_STATE_IDLE 0x0ULL @@ -161,7 +160,6 @@ static inline void setup_ucie(uintptr_t base) { reg_write64(base + UCIE_TX_CLK_N, UCIE_DEFAULT_CLK_N); reg_write64(base + UCIE_TX_VALID, UCIE_DEFAULT_VALID); reg_write64(base + UCIE_RX_LFSR_VALID, UCIE_DEFAULT_VALID); - reg_write64(base + UCIE_DIV_RESETB, 0x1ULL); for (int i = 0; i < 4; i++) { reg_write64(base + UCIE_DEBUG_DRIVERCTL + 8 * i, UCIE_ENABLE_DRIVER_CTL); } diff --git a/src/GenUcieHeader.scala b/src/GenUcieHeader.scala index 3f59c07..cba06f1 100644 --- a/src/GenUcieHeader.scala +++ b/src/GenUcieHeader.scala @@ -1,7 +1,5 @@ package edu.berkeley.cs.iris -import org.chipsalliance.cde.config.Parameters -import testchipip.soc.ChipletRoutingKey import edu.berkeley.cs.uciedigital.tilelink.UcieTLParams /** Writes the bringup software's `ucie.h` from the UCIe parameters IrisConfig @@ -23,10 +21,7 @@ object GenUcieHeader { /** The UCIe ports of IrisConfig, in chiplet-router port order. */ def uciePorts: Seq[UcieTLParams] = { - val p: Parameters = new IrisConfig - val ports = p(ChipletRoutingKey) - .map(_.ports.collect { case u: UcieTLParams => u }) - .getOrElse(Nil) + val ports = UciePort.all(new IrisConfig) require(ports.nonEmpty, "IrisConfig has no UCIe chiplet-router ports") ports } @@ -34,19 +29,19 @@ object GenUcieHeader { def render(): String = { val ports = uciePorts // The header holds one register layout, so every port has to agree on - // everything that layout depends on. Address and orientation do not reach - // it: the offsets are relative to each port's own MMIO base, and the - // orientation only names modules. + // everything that layout depends on. Address and moduleId do not reach it: + // the offsets are relative to each port's own MMIO base, and the moduleId + // only names modules. val canonical = ports.map( _.copy( address = ports.head.address, - orientation = ports.head.orientation + moduleId = ports.head.moduleId ) ) require( canonical.forall(_ == canonical.head), "UCIe ports must share a register layout, but their parameters differ " + - s"beyond address and orientation: ${canonical.distinct.mkString("\n")}" + s"beyond address and moduleId: ${canonical.distinct.mkString("\n")}" ) edu.berkeley.cs.uciedigital.tilelink.GenUcieHeader .render(ports.head, regenCommand) diff --git a/src/Iris.scala b/src/Iris.scala index 2ab545a..73ef937 100644 --- a/src/Iris.scala +++ b/src/Iris.scala @@ -115,8 +115,9 @@ class IrisTop(implicit p: Parameters) extends LazyModule with BindingScope { uart <> system.uart(0) // Connect D2D UCIe - val c2c_ucie0 = IO(new edu.berkeley.cs.uciedigital.tilelink.UcieBumpsIO(p(ChipletRoutingKey).get.ports(0).asInstanceOf[edu.berkeley.cs.uciedigital.tilelink.UcieTLParams].numLanes)) - val c2c_ucie1 = IO(new edu.berkeley.cs.uciedigital.tilelink.UcieBumpsIO(p(ChipletRoutingKey).get.ports(1).asInstanceOf[edu.berkeley.cs.uciedigital.tilelink.UcieTLParams].numLanes)) + val uciePorts = UciePort.all(p) + val c2c_ucie0 = IO(new edu.berkeley.cs.uciedigital.tilelink.UcieBumpsIO(uciePorts(0).numLanes)) + val c2c_ucie1 = IO(new edu.berkeley.cs.uciedigital.tilelink.UcieBumpsIO(uciePorts(1).numLanes)) c2c_ucie0 <> system.d2d_port_ios.get(0) c2c_ucie1 <> system.d2d_port_ios.get(1) } @@ -476,25 +477,23 @@ class IrisConfig(sim: Boolean = false) new testchipip.soc.WithChipletRouting(testchipip.soc.ChipletRoutingParams( routerParams = testchipip.soc.OffchipRouterParams(tableEntries = 4), ports = Seq( - // The two ports are identical logic, so they would dedup into - // one SystemVerilog module. The orientation splits them into - // UcieTL_NS and UcieTL_EW so PD can harden each separately: port 0 - // is placed unrotated facing down, port 1 is rotated 90 degrees - // facing right. - edu.berkeley.cs.uciedigital.tilelink.UcieTLParams( + // PD hardens the two UCIe links together, so UcieComplexPort puts + // both instances inside one UcieComplex module. They are identical + // logic and share a `moduleId`, so they dedup into a single UcieTL + // underneath it; splitting them again is a matter of giving one its + // own `moduleId`. + UcieComplexPort(edu.berkeley.cs.uciedigital.tilelink.UcieTLParams( address = 0x200000, managerWhere = SBUS, numLanes = 16, - includeDefaultModels = true, - orientation = edu.berkeley.cs.uciedigital.tilelink.UcieOrientation.NS - ), - edu.berkeley.cs.uciedigital.tilelink.UcieTLParams( + includeDefaultModels = true + )), + UcieComplexPort(edu.berkeley.cs.uciedigital.tilelink.UcieTLParams( address = 0x208000, managerWhere = SBUS, numLanes = 16, - includeDefaultModels = true, - orientation = edu.berkeley.cs.uciedigital.tilelink.UcieOrientation.EW - ) + includeDefaultModels = true + )) ))) ++ new WithIrisUncore(sim) ) diff --git a/src/UcieComplex.scala b/src/UcieComplex.scala new file mode 100644 index 0000000..820782f --- /dev/null +++ b/src/UcieComplex.scala @@ -0,0 +1,83 @@ +package edu.berkeley.cs.iris + +import org.chipsalliance.cde.config.Parameters +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.subsystem.TLBusWrapperLocation +import edu.berkeley.cs.uciedigital.tilelink.UcieTLParams +import testchipip.soc.{ + ChipletLinkParams, + ChipletLinkWrapper, + ChipletLinkWrapperInstantiationLike, + ChipletRoutingKey, + OffchipSubsystemParams +} + +/** The module both UCIe links live in. + * + * The chiplet router instantiates its D2D ports as siblings of the router, its + * address translators and its source shrinkers, all directly inside the + * router's clock domain. Physical design hardens the two UCIe links as one + * block, which needs them under a module of their own: this is that module, + * and it holds nothing but the links and the wires their diplomatic ports + * punch out to the level above. + */ +class UcieComplex(implicit p: Parameters) + extends SimpleLazyModule + with LazyScope { + override lazy val desiredName = "UcieComplex" +} + +/** A chiplet-router port whose UCIe link goes inside the shared [[UcieComplex]]. + * + * The router builds each port by calling `instantiate` on the port parameters, + * inside the scope of its own clock domain -- so this is the hook that decides + * what the link's parent module is, and it needs nothing from testchipip + * beyond what [[ChipletLinkWrapperInstantiationLike]] already asks for. + * + * The wrapper is created by whichever port is instantiated first and found + * again by the rest. Looking it up in the router domain's children, rather + * than holding it in a field here, is what keeps a config safe to elaborate + * more than once: the multi-chip test harness builds several IrisTops from one + * Parameters, and each gets its own router domain and hence its own wrapper. + */ +case class UcieComplexPort(ucie: UcieTLParams) + extends ChipletLinkParams + with ChipletLinkWrapperInstantiationLike { + def managerBusWhere: TLBusWrapperLocation = ucie.managerBusWhere + def controlManagerBusWhere: Option[TLBusWrapperLocation] = + ucie.controlManagerBusWhere + + def instantiate(params: OffchipSubsystemParams, id: Int)(implicit + p: Parameters + ): ChipletLinkWrapper = { + val routerDomain = LazyModule.getScope.getOrElse( + throw new IllegalStateException( + "UcieComplexPort.instantiate was called outside a LazyModule scope, " + + "so there is nothing to hang the UcieComplex wrapper off of" + ) + ) + val complex = routerDomain.getChildren + .collectFirst { case c: UcieComplex => c } + .getOrElse { + val d2d_ports = LazyModule(new UcieComplex) + d2d_ports.suggestName("d2d_ports") + d2d_ports + } + complex { ucie.instantiate(params, id) } + } +} + +object UciePort { + + /** The UCIe parameters of a chiplet-router port, wrapped or not. */ + def unapply(link: ChipletLinkParams): Option[UcieTLParams] = link match { + case wrapped: UcieComplexPort => Some(wrapped.ucie) + case direct: UcieTLParams => Some(direct) + case _ => None + } + + /** The UCIe ports of the chiplet router, in port order. */ + def all(p: Parameters): Seq[UcieTLParams] = p(ChipletRoutingKey) + .map(_.ports.collect { case UciePort(u) => u }) + .getOrElse(Nil) +} diff --git a/ucie b/ucie index 850dd9b..caf0daf 160000 --- a/ucie +++ b/ucie @@ -1 +1 @@ -Subproject commit 850dd9bb72914b09f83ba308c487e3ef521d2a88 +Subproject commit caf0daf1135f5b87225fa4903fcc3dfb87863de5