[DRAMsys 2.0] Terapool DRAMsys Merge Request

.gitmodules

@@ -34,3 +34,6 @@
 [submodule "hardware/deps/apb"]
 	path = hardware/deps/apb
 	url = https://github.com/pulp-platform/apb.git
+[submodule "hardware/deps/dram_rtl_sim"]
+	path = hardware/deps/dram_rtl_sim
+	url = https://github.com/pulp-platform/dram_rtl_sim.git

Bender.yml

@@ -56,6 +56,10 @@ sources:
       - hardware/tb/traffic_generator.sv
       # Level 2
       - hardware/tb/mempool_tb.sv
+      # DRAMsys
+      - hardware/deps/dram_rtl_sim/src/sim_dram.sv
+      - hardware/deps/dram_rtl_sim/src/axi_dram_sim.sv
+      - hardware/deps/dram_rtl_sim/src/dram_sim_engine.sv
 
   - target: mempool_verilator
     files:

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Add `apb` dependency of version 0.2.4
 - Add support for the `FENCE` instruction
+- Add support for DRAMsys5.0 co-simulation
 
 ### Changes
 - Add physical feasible TeraPool configuration with SubGroup hierarchy.

Makefile

@@ -24,7 +24,7 @@ BENDER_INSTALL_DIR    ?= ${INSTALL_DIR}/bender
 VERILATOR_INSTALL_DIR ?= ${INSTALL_DIR}/verilator
 RISCV_TESTS_DIR       ?= ${ROOT_DIR}/${SOFTWARE_DIR}/riscv-tests
 
-CMAKE ?= cmake
+CMAKE ?= cmake-3.28.3
 # CC and CXX are Makefile default variables that are always defined in a Makefile. Hence, overwrite
 # the variable if it is only defined by the Makefile (its origin in the Makefile's default).
 ifeq ($(origin CC),default)
@@ -157,6 +157,40 @@ update-deps:
 	done
 	git apply hardware/deps/patches/*
 
+# Build, update and patch the DRAMsys submodule
+$(eval DRAM_PATH=$(realpath $(shell git config --file .gitmodules --get-regexp dram_rtl_sim.path | awk '/hardware/{ print $$2 }')))
+$(eval DRAM_LIB_PATH=$(DRAM_PATH)/dramsys_lib)
+$(eval DRAMSYS_PATH=$(DRAM_LIB_PATH)/DRAMSys)
+$(eval DRAMSYS_PATCH_PATH=$(DRAM_LIB_PATH)/dramsys_lib_patch)
+$(eval DRAMSYS_SO_PATH=$(DRAMSYS_PATH)/build)
+
+clean-dram:
+	if [ -d "$(DRAMSYS_PATH)" ]; then \
+		rm -rf $(DRAMSYS_PATH); \
+	fi
+
+build-dram: clean-dram
+	if [ ! -d "$(DRAMSYS_PATH)" ]; then \
+		git clone https://github.com/tukl-msd/DRAMSys.git $(DRAMSYS_PATH); \
+	fi
+	cd $(DRAMSYS_PATH) && git reset --hard 8e021ea && git apply $(DRAMSYS_PATCH_PATH)
+
+config-dram: build-dram
+	@cp hardware/include/dram_config/am_hbm2e_16Gb_pc_brc.json $(DRAMSYS_PATH)/configs/addressmapping/.
+	@cp hardware/include/dram_config/mc_hbm2e_fr_fcfs_grp.json $(DRAMSYS_PATH)/configs/mcconfig/.
+	@cp hardware/include/dram_config/ms_hbm2e_16Gb_3600.json $(DRAMSYS_PATH)/configs/memspec/.
+	@cp hardware/include/dram_config/simconfig_hbm2e.json $(DRAMSYS_PATH)/configs/simconfig/.
+	@mv $(DRAMSYS_PATH)/configs/hbm2-example.json $(DRAMSYS_PATH)/configs/hbm2-example.json.ori
+	@cp hardware/include/dram_config/HBM2E-3600.json $(DRAMSYS_PATH)/configs/hbm2-example.json
+
+setup-dram: config-dram
+	cd $(DRAMSYS_PATH) && \
+	if [ ! -d "build" ]; then \
+		mkdir build && cd build; \
+		CC=gcc-11.2.0 CXX=g++-11.2.0 cmake -DCMAKE_CXX_FLAGS=-fPIC -DCMAKE_C_FLAGS=-fPIC -D DRAMSYS_WITH_DRAMPOWER=ON .. ; \
+		make -j; \
+	fi
+
 # Helper targets
 .PHONY: clean format apps
 

README.md

@@ -187,6 +187,39 @@ To get a visualization of the traces, check out the `scripts/tracevis.py` script
 
 We also provide Synopsys Spyglass linting scripts in the `hardware/spyglass`. Run `make lint` in the `hardware` folder, with a specific MemPool configuration, to run the tests associated with the `lint_rtl` target.
 
+## DRAMsys Co-Simulation
+
+The MemPool system supports both on-chip SRAM or off-chip DRAM co-simulation for higher hierarchy memory transfering. For off-chip DRAM co-simulation, it incorporates the `dram_rtl_sim` tool as a submodule, build at `hardware/deps/dram_rtl_sim`. Leveraging DRAMSys5.0, it facilitates an effective co-simulation environment between RTL models and DRAMSys5.0 for the simulation of DRAM + CTRL models, with contemporary off-chip DRAM technologies (e.g., LPDDR, DDR, HBM).
+
+The DRAMsys tool aids are open-sourced and can be found here:
+[https://github.com/pulp-platform/dram_rtl_sim](https://github.com/pulp-platform/dram_rtl_sim)
+
+### Building DRAMsys Co-Simulation
+
+To prepare for DRAMsys co-simulation, adjust the system configuration by setting `l2_sim_type` to `dram` in `config/config.mk`. Then, execute the following command in the project's root directory to establish the DRAMsys tool aids environment:
+
+```bash
+make setup-dram
+```
+
+This makefile target automates several tasks:
+1. Cleans up the existing DRAMSys5.0 repository, if previously built.
+2. Rebuilds the DRAMSys5.0 repository and applies necessary patches within `hardware/deps/dramsys_rtl_sim/dramsys_lib/`.
+3. Applies HBM2 DRAM configuration patches tailored for the MemPool system simulation.
+4. Compiles the DRAMSys dynamic linkable library located at `hardware/deps/dramsys_rtl_sim/dramsys_lib/DRAMSys`.
+
+**Important:** This environment requires `cmake` version 3.28.1 or higher and GCC version 11.2.0 or above.
+
+### DRAM Chip Configuration
+
+DRAMsys supports a range of contemporary off-chip DRAM technologies, including LPDDR, DDR, and HBM. Configuration files, formatted as `.json`, are accessible in the following directory: `hardware/deps/dramsys_rtl_sim/dramsys_lib/DRAMSys/configs`. Additionally, we provide a recommended HBM2 configuration for the MemPool system located within `hardware/deps/dramsys_rtl_sim/dramsys_lib/DRAMSys`. This configuration is automatically applied as the default setting when establishing the DRAMsys tool aids environment. You are encouraged to review and modify these configurations as necessary to meet your specific simulation requirements.
+
+### Testing MemPool-DRAMSys Co-Simulation
+
+For data transfer testing between the MemPool system and higher hierarchy memory through DMA transfer, use the prepared example kernel located in `software/tests/baremetal/memcpy`. For more detailed methods on building applications and setting up RTL simulation, please refer to the sections aboves.
+
+**Note:** Currently, the simulation crafting tool for off-chip DRAM co-simulation is not open-sourced. We utilize the `Questasim` simulator exclusively.
+
 ## Publications
 If you use MemPool in your work or research, you can cite us:
 
@@ -602,5 +635,10 @@ The open-source simulator [Verilator](https://www.veripool.org/verilator) can be
 
 - `toolchain/verilator` is licensed under GPL. See [Verilator's license](https://github.com/verilator/verilator/blob/master/LICENSE) for more details.
 
+### DRAMsys5.0
+
+- The `dram_rtl_sim` submodule, located at `hardware/deps/dram_rtl_sim`, is licensed under the Solderpad Hardware License 0.51. You can review the license [here](https://github.com/pulp-platform/dram_rtl_sim/blob/main/LICENSE).
+- [DRAMSys5.0](https://github.com/tukl-msd/DRAMSys) is utilized for DRAM simulations. For details on its usage and licensing, please refer to the DRAMSys5.0 [license information](https://github.com/tukl-msd/DRAMSys).
+
 </p>
 </details>

config/config.mk

@@ -30,8 +30,6 @@ boot_addr ?= 2684354560 # A0000000
 
 # L2 memory configuration (in dec)
 l2_base  ?= 2147483648 # 80000000
-l2_size  ?= 4194304    # 400000
-l2_banks ?= 4
 
 # L1 size per bank (in dec)
 l1_bank_size ?= 1024
@@ -52,9 +50,6 @@ axi_data_width ?= 512
 # Read-only cache line width in AXI interconnect (in bits)
 ro_line_width ?= 512
 
-# Number of DMA backends in each group
-dmas_per_group ?= 4
-
 #############################
 ##  Xqueues configuration  ##
 #############################
@@ -72,3 +67,7 @@ xpulpimg ?= 1
 # This parameter is only used for TeraPool configurations
 num_sub_groups_per_group ?= 1
 remote_group_latency_cycles ?= 7
+
+# DRAMsys co-simulation: dram/sram
+l2_sim_type ?= sram
+dram_axi_width_interleaved ?= 16

config/mempool.mk

@@ -21,7 +21,14 @@ num_cores_per_tile ?= 4
 banking_factor ?= 4
 
 # Radix for hierarchical AXI interconnect
-axi_hier_radix ?= 20
+axi_hier_radix ?= 17
 
 # Number of AXI masters per group
 axi_masters_per_group ?= 1
+
+# Number of DMA backends in each group
+dmas_per_group ?= 1 # Brust Length = 16
+
+# L2 Banks/Channels
+l2_size  ?= 4194304  # 400000
+l2_banks ?= 4

config/minpool.mk

@@ -29,11 +29,15 @@ axi_data_width ?= 256
 # Read-only cache line width in AXI interconnect (in bits)
 ro_line_width ?= 256
 
-# Number of DMA backends in each group
-dmas_per_group ?= 1
-
 # Radix for hierarchical AXI interconnect
 axi_hier_radix ?= 2
 
 # Number of AXI masters per group
 axi_masters_per_group ?= 1
+
+# Number of DMA backends in each group
+dmas_per_group ?= 1 # Brust Length = 16
+
+# L2 Banks/Channels
+l2_size  ?= 4194304  # 400000
+l2_banks ?= 4

config/terapool.mk

@@ -37,7 +37,8 @@ axi_hier_radix ?= 9
 axi_masters_per_group ?= 4
 
 # Number of DMA backends in each group
-dmas_per_group ?= 4
+dmas_per_group ?= 4 # Brust Length = 16
 
 # L2 Banks/Channels
 l2_banks = 16
+l2_size  ?= 16777216 # 1000000

hardware/Makefile

@@ -48,6 +48,12 @@ python          ?= python3
 # Enable tracing
 snitch_trace    ?= 1
 
+# Path to DRAMsys
+dramsys_resouces_path ?= $(MEMPOOL_DIR)/hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/configs
+dramsys_lib_path ?= $(MEMPOOL_DIR)/hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/build/lib
+questa_args += +DRAMSYS_RES=$(dramsys_resouces_path)
+questa_args += -sv_lib $(dramsys_lib_path)/libsystemc -sv_lib $(dramsys_lib_path)/libDRAMSys_Simulator
+
 # Check if the specified QuestaSim version exists
 ifeq (, $(shell which $(questa_cmd)))
   # Spaces are needed for indentation here!
@@ -112,8 +118,11 @@ vlog_defs += -DRO_LINE_WIDTH=$(ro_line_width)
 vlog_defs += -DDMAS_PER_GROUP=$(dmas_per_group)
 vlog_defs += -DAXI_HIER_RADIX=$(axi_hier_radix) -DAXI_MASTERS_PER_GROUP=$(axi_masters_per_group)
 vlog_defs += -DSEQ_MEM_SIZE=$(seq_mem_size) -DXQUEUE_SIZE=$(xqueue_size)
-# This parameter is only used for TeraPool configurations
+# The below parameter is only used for TeraPool configurations
 vlog_defs += -DNUM_SUB_GROUPS_PER_GROUP=$(num_sub_groups_per_group) -DREMOTE_GROUP_LATENCY_CYCLES=$(remote_group_latency_cycles)
+# The below parameter is only used for DRAMsys co-simulation
+vlog_defs += -D${l2_sim_type}
+vlog_defs += -DDRAM_AXI_WIDTH_INTERLEAVED=${dram_axi_width_interleaved}
 
 # Traffic generation enabled
 ifdef tg

hardware/include/dram_config/HBM2E-3600.json

@@ -0,0 +1,15 @@
+{
+    "simulation": {
+        "addressmapping": "am_hbm2e_16Gb_pc_brc.json",
+        "mcconfig": "mc_hbm2e_fr_fcfs_grp.json",
+        "memspec": "ms_hbm2e_16Gb_3600.json",
+        "simconfig": "simconfig_hbm2e.json",
+        "simulationid": "hbm2e",
+        "tracesetup": [
+            {
+                "clkMhz": 1800,
+                "name": "HBM2E.stl"
+            }
+        ]
+    }
+}

hardware/include/dram_config/am_hbm2e_16Gb_pc_brc.json

@@ -0,0 +1,47 @@
+{
+    "addressmapping": {
+        "BYTE_BIT": [
+            0,
+            1,
+            2
+        ],
+        "COLUMN_BIT": [
+            3,
+            4,
+            8,
+            9,
+            10,
+            11,
+            12
+        ],
+        "PSEUDOCHANNEL_BIT":[
+            5
+        ],
+        "BANK_BIT": [
+            16,
+            17
+        ],
+        "BANKGROUP_BIT":[
+            6,
+            7,
+            13
+        ],
+        "ROW_BIT": [
+            14,
+            15,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30
+        ]
+    }
+}

hardware/include/dram_config/mc_hbm2e_fr_fcfs_grp.json

@@ -0,0 +1,20 @@
+{
+    "mcconfig": {
+        "PagePolicy": "Open",
+        "Scheduler": "FrFcfsGrp",
+        "SchedulerBuffer": "Bankwise",
+        "RequestBufferSize": 128,
+        "CmdMux": "Oldest",
+        "RespQueue": "Fifo",
+        "RefreshPolicy": "AllBank",
+        "RefreshMaxPostponed": 8,
+        "RefreshMaxPulledin": 8,
+        "PowerDownPolicy": "NoPowerDown",
+        "Arbiter": "Simple",
+        "PhyDelayFw": 8,
+        "PhyDelayBw": 9,
+        "ThinkDelayFw": 12,
+        "ThinkDelayBW": 12,
+        "MaxActiveTransactions": 128
+    }
+}

hardware/include/dram_config/ms_hbm2e_16Gb_3600.json

@@ -0,0 +1,48 @@
+{
+    "memspec": {
+        "memarchitecturespec": {
+            "burstLength": 4,
+            "dataRate": 2,
+            "nbrOfBankGroups": 8,
+            "nbrOfBanks": 32,
+            "nbrOfColumns": 128,
+            "nbrOfPseudoChannels": 2,
+            "nbrOfRows": 32768,
+            "width": 64,
+            "nbrOfDevices": 1,
+            "nbrOfChannels": 1
+        },
+        "memoryId": "Test MemPool-TeraPool with HBM2 upto 3600bps (16Gb, Single Channel)",
+        "memoryType": "HBM2",
+        "memtimingspec": {
+            "CCDL": 4,
+            "CCDS": 2,
+            "CKE": 8,
+            "DQSCK": 2,
+            "FAW": 9,
+            "PL": 2,
+            "RAS": 30,
+            "RC": 45,
+            "RCDRD": 16,
+            "RCDWR": 12,
+            "REFI": 3900,
+            "REFISB": 122,
+            "RFC": 260,
+            "RFCSB": 200,
+            "RL": 41,
+            "RP": 15,
+            "RRDL": 2.22,
+            "RRDS": 2.22,
+            "RREFD": 8,
+            "RTP": 4,
+            "RTW": 18,
+            "WL": 8,
+            "WR": 41,
+            "WTRL": 6,
+            "WTRS": 4,
+            "XP": 10,
+            "XS": 270,
+            "clkMhz": 1800
+        }
+    }
+}

hardware/include/dram_config/simconfig_hbm2e.json

@@ -0,0 +1,15 @@
+{
+    "simconfig": {
+        "AddressOffset": 0,
+        "CheckTLM2Protocol": false,
+        "DatabaseRecording": true,
+        "Debug": false,
+        "EnableWindowing": true,
+        "PowerAnalysis": false,
+        "SimulationName": "hbm2e",
+        "SimulationProgressBar": true,
+        "StoreMode": "Store",
+        "UseMalloc": false,
+        "WindowSize": 300
+    }
+}