Bump Checkpointing Script + Docs

.github/scripts/defaults.sh

@@ -29,7 +29,7 @@ REMOTE_COURSIER_CACHE=$REMOTE_WORK_DIR/.coursier-cache
 # key value store to get the build groups
 declare -A grouping
 grouping["group-cores"]="chipyard-cva6 chipyard-ibex chipyard-rocket chipyard-hetero chipyard-boomv3 chipyard-boomv4 chipyard-sodor chipyard-digitaltop chipyard-multiclock-rocket chipyard-nomem-scratchpad chipyard-spike chipyard-clone chipyard-prefetchers chipyard-shuttle"
-grouping["group-peripherals"]="chipyard-dmirocket chipyard-dmiboomv3 chipyard-dmiboomv4 chipyard-spiflashwrite chipyard-mmios chipyard-nocores chipyard-manyperipherals chipyard-chiplike chipyard-tethered chipyard-symmetric chipyard-llcchiplet"
+grouping["group-peripherals"]="chipyard-cospikeckptdmirocket chipyard-ckptdmiboomv3 chipyard-ckptdmiboomv4 chipyard-spiflashwrite chipyard-mmios chipyard-nocores chipyard-manyperipherals chipyard-chiplike chipyard-tethered chipyard-symmetric chipyard-llcchiplet"
 grouping["group-accels"]="chipyard-compressacc chipyard-mempress chipyard-gemmini chipyard-manymmioaccels chipyard-nvdla chipyard-aes256ecb chipyard-rerocc"
 grouping["group-constellation"]="chipyard-constellation"
 grouping["group-tracegen"]="tracegen tracegen-boomv3 tracegen-boomv4"
@@ -39,7 +39,7 @@ grouping["group-fpga"]="arty35t arty100t nexysvideo vc707 vcu118"
 # key value store to get the build strings
 declare -A mapping
 mapping["chipyard-rocket"]=" CONFIG=QuadChannelRocketConfig"
-mapping["chipyard-dmirocket"]=" CONFIG=dmiRocketConfig"
+mapping["chipyard-cospikeckptdmirocket"]=" CONFIG=dmiCospikeCheckpointingRocketConfig"
 mapping["chipyard-mempress"]=" CONFIG=MempressRocketConfig"
 mapping["chipyard-compressacc"]=" CONFIG=ZstdCompressorRocketConfig"
 mapping["chipyard-prefetchers"]=" CONFIG=PrefetchingRocketConfig"
@@ -48,9 +48,9 @@ mapping["chipyard-manymmioaccels"]=" CONFIG=ManyMMIOAcceleratorRocketConfig"
 mapping["chipyard-nvdla"]=" CONFIG=SmallNVDLARocketConfig verilog"
 mapping["chipyard-hetero"]=" CONFIG=LargeBoomAndRocketConfig"
 mapping["chipyard-boomv3"]=" CONFIG=MediumBoomV3CosimConfig"
-mapping["chipyard-dmiboomv3"]=" CONFIG=dmiMediumBoomV3CosimConfig"
+mapping["chipyard-ckptdmiboomv3"]=" CONFIG=dmiCheckpointingMediumBoomV3CosimConfig"
 mapping["chipyard-boomv4"]=" CONFIG=MediumBoomV4CosimConfig"
-mapping["chipyard-dmiboomv4"]=" CONFIG=dmiMediumBoomV4CosimConfig"
+mapping["chipyard-ckptdmiboomv4"]=" CONFIG=dmiCheckpointingMediumBoomV4CosimConfig"
 mapping["chipyard-spike"]=" CONFIG=SpikeConfig EXTRA_SIM_FLAGS='+spike-ipc=10'"
 mapping["chipyard-gemmini"]=" CONFIG=GemminiRocketConfig"
 mapping["chipyard-cva6"]=" CONFIG=CVA6Config"

.github/scripts/run-tests.sh

@@ -41,10 +41,10 @@ case $1 in
         run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv LOADMEM=1
         run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv
         ;;
-    chipyard-dmirocket)
+    chipyard-cospikeckptdmirocket)
         # Test checkpoint-restore
         $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000
-        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
+        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.unused.10000.defaultspikedts.loadarch
         ;;
     chipyard-boomv3)
         run_bmark
@@ -55,15 +55,15 @@ case $1 in
     chipyard-shuttle)
         run_bmark ${mapping[$1]}
         ;;
-    chipyard-dmiboomv3)
+    chipyard-ckptdmiboomv3)
         # Test checkpoint-restore
         $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000
-        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
+        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.unused.10000.defaultspikedts.loadarch
         ;;
-    chipyard-dmiboomv4)
+    chipyard-ckptdmiboomv4)
         # Test checkpoint-restore
         $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000
-        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
+        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.unused.10000.defaultspikedts.loadarch
         ;;
     chipyard-spike)
         run_bmark

.github/workflows/chipyard-run-tests.yml

@@ -593,7 +593,7 @@ jobs:
           group-key: "group-cores"
           project-key: "chipyard-spike"
 
-  chipyard-dmirocket-run-tests:
+  chipyard-cospikeckptdmirocket-run-tests:
     name: chipyard-dmirocket-run-tests
     needs: prepare-chipyard-peripherals
     runs-on: as4
@@ -614,10 +614,10 @@ jobs:
         uses: ./.github/actions/run-tests
         with:
           group-key: "group-peripherals"
-          project-key: "chipyard-dmirocket"
+          project-key: "chipyard-cospikeckptdmirocket"
 
-  chipyard-dmiboomv3-run-tests:
-    name: chipyard-dmiboomv3-run-tests
+  chipyard-ckptdmiboomv3-run-tests:
+    name: chipyard-ckptdmiboomv3-run-tests
     needs: prepare-chipyard-peripherals
     runs-on: as4
     steps:
@@ -637,10 +637,10 @@ jobs:
         uses: ./.github/actions/run-tests
         with:
           group-key: "group-peripherals"
-          project-key: "chipyard-dmiboomv3"
+          project-key: "chipyard-ckptdmiboomv3"
 
-  chipyard-dmiboomv4-run-tests:
-    name: chipyard-dmiboomv4-run-tests
+  chipyard-ckptdmiboomv4-run-tests:
+    name: chipyard-ckptdmiboomv4-run-tests
     needs: prepare-chipyard-peripherals
     runs-on: as4
     steps:
@@ -660,7 +660,7 @@ jobs:
         uses: ./.github/actions/run-tests
         with:
           group-key: "group-peripherals"
-          project-key: "chipyard-dmiboomv4"
+          project-key: "chipyard-ckptdmiboomv4"
 
   chipyard-spiflashwrite-run-tests:
     name: chipyard-spiflashwrite-run-tests
@@ -1181,9 +1181,9 @@ jobs:
             chipyard-cva6-run-tests,
             chipyard-ibex-run-tests,
             chipyard-sodor-run-tests,
-            chipyard-dmiboomv3-run-tests,
-            chipyard-dmiboomv4-run-tests,
-            chipyard-dmirocket-run-tests,
+            chipyard-ckptdmiboomv3-run-tests,
+            chipyard-ckptdmiboomv4-run-tests,
+            chipyard-cospikeckptdmirocket-run-tests,
             chipyard-spiflashwrite-run-tests,
             chipyard-manyperipherals-run-tests,
             chipyard-tethered-run-tests,

docs/Advanced-Concepts/Architectural-Checkpoints.rst

@@ -9,16 +9,16 @@ The checkpoints include the contents of cacheable memory, core architectural reg
 RTL simulations of SoCs can resume execution from checkpoints after restoring the architectural state.
 
 .. note::
-   Currently, only checkpoints of single-core systems are supported
+   Currently, only checkpoints of single-core systems are supported.
 
 Generating Checkpoints
 ------------------------
 
-``scripts/generate-ckpt.sh`` is a script that runs spike with the right commands to generate an architectural checkpoint
+``scripts/generate-ckpt.sh`` is a script that runs Spike with the right commands to generate an architectural checkpoint.
 ``scripts/generate-ckpt.sh -h`` lists options for checkpoint generation.
 
 Example: run the ``hello.riscv`` binary for 1000 instructions before generating a checkpoint.
-This should produce a directory named ``hello.riscv.0x80000000.1000.loadarch``
+This should produce a directory named ``hello.riscv.*.loadarch``
 
 .. code::
 
@@ -29,10 +29,34 @@ Loading Checkpoints in RTL Simulation
 --------------------------------------
 
 Checkpoints can be loaded in RTL simulations with the ``LOADARCH`` flag.
-The target config **MUST** use dmi-based bringup (as opposed to the default TSI-based bringup), and support fast ``LOADMEM``.
-The target config should also match the architectural configuration of however spike was configured when generating the checkpoint.
+The target config needs the following properties:
+
+- **MUST** use DMI-based bringup (as opposed to the default TSI-based bringup)
+- **MUST** support fast ``LOADMEM``
+- Should match the architectural configuration of however Spike was configured when generating the checkpoint (i.e. same ISA, no PMPs, etc).
 
 .. code::
 
    cd sims/vcs
-   make CONFIG=dmiRocketConfig run-binary LOADARCH=../../hello.riscv.0x80000000.1000.loadarch
+   make CONFIG=dmiRocketConfig run-binary LOADARCH=../../hello.riscv.*.loadarch
+
+Checkpointing Linux Binaries
+----------------------------
+
+Checkpoints can be used to run Linux binaries with the following caveats:
+
+- The binary must only use the HTIF console and should be non-interactive (i.e no stdin available)
+- The target config must be built without a serial device (i.e. the Rocket Chip Blocks UART can't be used)
+- The binary must only use an initramfs (i.e. no block device)
+- The target config must be built without a block device (i.e. the IceBlk block device can't be used).
+- The binary size must be smaller than the size of the target configs memory region (for example if FireMarshal's ``rootfs-size`` is 1GB, and OpenSBI is 350KB then you must have at least 1G + 1KB of space)
+
+This means that you most likely need to do the following:
+
+- By default Spike has a default UART device that is used during most Linux boot's.
+  This can be bypassed by creating a DTS without a serial device then passing it to the ``generate-ckpt.sh`` script.
+  You can copy the DTS of the design you want to checkpoint into - located in Chipyards ``sims/<simulator>/generated-src/`` - and modify it to pass to the checkpointing script (needs to be stripped down of extra devices and nodes).
+  An example of a config made for checkpointing is ``dmiCheckpointingRocketConfig`` or ``dmiCheckpointingSpikeUltraFastConfig``.
+- Additionally, you need to change your Linux config in FireMarshal to default to only use HTIF during OpenSBI and force Linux to use the OpenSBI HTIF console.
+  This can be done by the following in the ``linux-config``: changing to ``CONFIG_CMDLINE="console=hvc0 earlycon=sbi"``, adding ``CONFIG_RISCV_SBI_V01=y``, adding ``CONFIG_HVC_RISCV_SBI=y``, and adding ``CONFIG_SERIAL_EARLYCON_RISCV_SBI=y``.
+  An example workload with these changes can be found at ``example-workloads/br-base-htif-only-serial.yaml``.

generators/chipyard/src/main/scala/SpikeTile.scala

@@ -16,7 +16,9 @@ import freechips.rocketchip.util._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.prci._
 
-case class SpikeCoreParams() extends CoreParams {
+case class SpikeCoreParams(
+  nPMPs: Int = 16
+) extends CoreParams {
   val useVM = true
   val useHypervisor = false
   val useSupervisor = true
@@ -33,7 +35,6 @@ case class SpikeCoreParams() extends CoreParams {
   val nLocalInterrupts = 0
   val useNMI = false
   val nPTECacheEntries = 0
-  val nPMPs = 16
   val pmpGranularity = 4
   val nBreakpoints = 0
   val useBPWatch = false
@@ -115,6 +116,7 @@ class SpikeTile(
   val masterNode = visibilityNode
   val slaveNode = TLIdentityNode()
 
+  // Note: Rocket doesn't support zicntr but Spike does (err on the side of having Rocket's ISA)
   override def isaDTS = "rv64imafdcv_zicsr_zifencei_zihpm_zvl128b_zve64d"
 
   // Required entry of CPU device in the device tree for interrupt purpose

generators/chipyard/src/main/scala/clocking/ClockGroupCombiner.scala

@@ -51,9 +51,9 @@ class ClockGroupCombiner(implicit p: Parameters, v: ValName) extends LazyModule
       val names = g.map(_.name.getOrElse("unamed"))
       val takes = g.map(_.take).flatten
       require(takes.distinct.size <= 1,
-        s"Clock group $name has non-homogeneous requested ClockParameters ${names.zip(takes)}")
+        s"Clock group '$name' has non-homogeneous requested ClockParameters ${names.zip(takes)}")
       require(takes.size > 0,
-        s"Clock group $name has no inheritable frequencies")
+        s"Clock group '$name' has no inheritable frequencies")
       (grouped ++ Seq(ClockSinkParameters(take = takes.headOption, name = Some(name))), r)
     }
 

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -121,7 +121,6 @@ class AbstractConfig extends Config(
 
   // Bus/interconnect settings
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++     /** hierarchical buses including sbus/mbus/pbus/fbus/cbus/l2 */
-  new chipyard.config.WithSV48IfPossible ++                         /** use sv48 if possible */
 
   // ================================================
   //   Set up power, reset and clocking

generators/chipyard/src/main/scala/config/BoomConfigs.scala

@@ -48,7 +48,8 @@ class MediumBoomV3CosimConfig extends Config(
   new boom.v3.common.WithNMediumBooms(1) ++
   new chipyard.config.AbstractConfig)
 
-class dmiMediumBoomV3Config extends Config(
+class dmiCheckpointingMediumBoomV3Config extends Config(
+  new chipyard.config.WithNPMPs(0) ++                            // remove PMPs (reduce non-core arch state)
   new chipyard.harness.WithSerialTLTiedOff ++                    // don't attach anything to serial-tl
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new boom.v3.common.WithNMediumBooms(1) ++
@@ -103,7 +104,8 @@ class MediumBoomV4CosimConfig extends Config(
   new boom.v4.common.WithNMediumBooms(1) ++
   new chipyard.config.AbstractConfig)
 
-class dmiMediumBoomV4Config extends Config(
+class dmiCheckpointingMediumBoomV4Config extends Config(
+  new chipyard.config.WithNPMPs(0) ++                            // remove PMPs (reduce non-core arch state)
   new chipyard.harness.WithSerialTLTiedOff ++                    // don't attach anything to serial-tl
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new boom.v4.common.WithNMediumBooms(1) ++

generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala

@@ -67,6 +67,13 @@ class dmiRocketConfig extends Config(
   new chipyard.config.AbstractConfig)
 // DOC include end: DmiRocket
 
+class dmiCospikeCheckpointingRocketConfig extends Config(
+  new chipyard.config.WithNPMPs(0) ++                            // remove PMPs (reduce non-core arch state)
+  new chipyard.harness.WithCospike ++                            // attach spike-cosim
+  new chipyard.config.WithTraceIO ++                             // enable the traceio
+  new freechips.rocketchip.subsystem.WithRocketDebugROB ++       // cospike needs wdata given by the unsynth. debug rom
+  new dmiRocketConfig)
+
 class ManyPeripheralsRocketConfig extends Config(
   new testchipip.iceblk.WithBlockDevice ++                   // add block-device module to peripherybus
   new testchipip.soc.WithOffchipBusClient(MBUS) ++           // OBUS provides backing memory to the MBUS

generators/chipyard/src/main/scala/config/RocketConfigs.scala

@@ -104,3 +104,7 @@ class ClusteredRocketConfig extends Config(
 class FastRTLSimRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithoutTLMonitors ++
   new chipyard.RocketConfig)
+
+class SV48RocketConfig extends Config(
+  new chipyard.config.WithSV48 ++
+  new chipyard.RocketConfig)

generators/chipyard/src/main/scala/config/SpikeConfigs.scala

@@ -18,40 +18,36 @@ class dmiSpikeConfig extends Config(
 // Avoids polling on the UART registers
 class SpikeFastUARTConfig extends Config(
   new chipyard.WithNSpikeCores(1) ++
-  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++ // Spike sim requires a larger UART FIFO buffer, 
-  new chipyard.config.WithNoUART() ++                           // so we overwrite the default one
+  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++   // Spike sim requires a larger UART FIFO buffer,
+  new chipyard.config.WithNoUART() ++                             // so we overwrite the default one
+  new chipyard.config.WithPeripheryBusFrequency(2) ++             // configured to be as fast as possible
   new chipyard.config.WithMemoryBusFrequency(2) ++
-  new chipyard.config.WithPeripheryBusFrequency(2) ++
+  new chipyard.config.WithControlBusFrequency(2) ++
+  new chipyard.config.WithSystemBusFrequency(2) ++
+  new chipyard.config.WithFrontBusFrequency(2) ++
+  new chipyard.config.WithOffchipBusFrequency(2) ++
   new chipyard.config.AbstractConfig)
 
-// Makes the UART fast, also builds no L2 and a ludicrous L1D
+// No L2 and a ludicrous L1D
 class SpikeUltraFastConfig extends Config(
+  new testchipip.soc.WithNoScratchpads ++
   new chipyard.WithSpikeTCM ++
-  new chipyard.WithNSpikeCores(1) ++
-  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++ // Spike sim requires a larger UART FIFO buffer, 
-  new chipyard.config.WithNoUART() ++                           // so we overwrite the default one
-  new chipyard.config.WithMemoryBusFrequency(2) ++
-  new chipyard.config.WithPeripheryBusFrequency(2) ++
   new chipyard.config.WithBroadcastManager ++
-  new chipyard.config.AbstractConfig)
+  new SpikeFastUARTConfig)
 
 class dmiSpikeUltraFastConfig extends Config(
   new chipyard.harness.WithSerialTLTiedOff ++                    // don't attach anything to serial-tilelink
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new SpikeUltraFastConfig)
 
+class dmiCheckpointingSpikeUltraFastConfig extends Config(
+  new chipyard.config.WithNPMPs(0) ++                            // remove PMPs (reduce non-core arch state)
+  new dmiSpikeUltraFastConfig)
+
 // Add the default firechip devices
 class SpikeUltraFastDevicesConfig extends Config(
   new chipyard.harness.WithSimBlockDevice ++
   new chipyard.harness.WithLoopbackNIC ++
   new icenet.WithIceNIC ++
   new testchipip.iceblk.WithBlockDevice ++
-
-  new chipyard.WithSpikeTCM ++
-  new chipyard.WithNSpikeCores(1) ++
-  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++ // Spike sim requires a larger UART FIFO buffer, 
-  new chipyard.config.WithNoUART() ++                           // so we overwrite the default one
-  new chipyard.config.WithMemoryBusFrequency(2) ++
-  new chipyard.config.WithPeripheryBusFrequency(2) ++
-  new chipyard.config.WithBroadcastManager ++
-  new chipyard.config.AbstractConfig)
+  new SpikeUltraFastConfig)

generators/chipyard/src/main/scala/config/fragments/TileFragments.scala

@@ -71,6 +71,8 @@ class WithNPMPs(n: Int = 8) extends Config((site, here, up) => {
       core = tp.tileParams.core.copy(nPMPs = n)))
     case tp: boom.v4.common.BoomTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(nPMPs = n)))
+    case tp: chipyard.SpikeTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
+      core = tp.tileParams.core.copy(nPMPs = n)))
     case other => other
   }
 })
@@ -127,12 +129,12 @@ class WithRocketBoundaryBuffers(buffers: Option[RocketTileBoundaryBufferParams]
   }
 })
 
-// Uses SV48 if possible, otherwise default to the Rocket Chip core default
-class WithSV48IfPossible extends Config((site, here, up) => {
-  case PgLevels => if (site(XLen) == 64) 4 /* Sv48 */ else up(PgLevels)
+// Use SV48
+class WithSV48 extends Config((site, here, up) => {
+  case PgLevels => { require(site(XLen) == 64); 4; }
 })
 
-// Uses SV39 if possible, otherwise default to the Rocket Chip core default
+// Use SV39
 class WithSV39 extends Config((site, here, up) => {
   case PgLevels => { require(site(XLen) == 64); 3; }
 })