Merge pull request #82 from pulp-platform/terapool_merge

[TeraPool] Physically Feasible TeraPool Configuration Merge
pulp-platform · Dec 12, 2023 · 45a9e81 · 45a9e81
2 parents a341a7d + 6d40e35
commit 45a9e81
Show file tree

Hide file tree

Showing 21 changed files with 3,537 additions and 1,029 deletions.
diff --git a/Bender.yml b/Bender.yml
@@ -37,12 +37,14 @@ sources:
   # Level 1
   - hardware/src/mempool_tile.sv
   # Level 2
-  - hardware/src/mempool_group.sv
+  - hardware/src/mempool_sub_group.sv
   # Level 3
-  - hardware/src/mempool_cluster.sv
+  - hardware/src/mempool_group.sv
   # Level 4
-  - hardware/src/ctrl_registers.sv
+  - hardware/src/mempool_cluster.sv
   # Level 5
+  - hardware/src/ctrl_registers.sv
+  # Level 6
   - hardware/src/mempool_system.sv
 
   - target: mempool_vsim

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## Unreleased
 
 ### Changes
+- Add physical feasible TeraPool configuration with SubGroup hierarchy.
 - Extended `tracevis.py` to support the new Perfetto UI and compress large traces
 - Use custom compiler for VCS specified with `CC` and `CCX` environment variable
 - Implement operand gating for SIMD and MAC Units in Snitch IPU's DSP Unit

diff --git a/config/config.mk b/config/config.mk
@@ -68,3 +68,7 @@ xqueue_size ?= 0
 
 # Enable the XpulpIMG extension
 xpulpimg ?= 1
+
+# This parameter is only used for TeraPool configurations
+num_sub_groups_per_group ?= 1
+remote_group_latency_cycles ?= 7
diff --git a/config/terapool.mk b/config/terapool.mk
@@ -8,20 +8,36 @@
 ##  TeraPool  ##
 ################
 
+# Global Control
+terapool ?= 1
+
 # Number of cores
 num_cores ?= 1024
 
 # Number of groups
-num_groups ?= 8
+num_groups ?= 4
 
 # Number of cores per Terapool tile
 num_cores_per_tile ?= 8
 
+# Number of sub groups per Terapool group
+num_sub_groups_per_group ?= 4
+
 # L1 scratchpad banking factor
 banking_factor ?= 4
 
+# Access latency between remote groups
+# Options: "7", "9" or "11":
+remote_group_latency_cycles ?= 7
+
 # Radix for hierarchical AXI interconnect
-axi_hier_radix ?= 10
+axi_hier_radix ?= 9
 
 # Number of AXI masters per group
-axi_masters_per_group ?= 2
+axi_masters_per_group ?= 4
+
+# Number of DMA backends in each group
+dmas_per_group ?= 4
+
+# L2 Banks/Channels
+l2_banks = 16
diff --git a/hardware/Makefile b/hardware/Makefile
@@ -87,16 +87,19 @@ endif
 vlog_args += -suppress vlog-2583 -suppress vlog-13314 -suppress vlog-13233
 vlog_args += -work $(library)
 # Defines
+vlog_defs += -D$(config)
 vlog_defs += -DNUM_CORES=$(num_cores) -DNUM_CORES_PER_TILE=$(num_cores_per_tile) -DNUM_GROUPS=$(num_groups) -DBANKING_FACTOR=$(banking_factor)
-vlog_defs += -DL2_BASE=$(l2_base) -DL2_SIZE=$(l2_size) -DL2_BANKS=$(l2_banks)
+vlog_defs += -DL2_BASE=32\'d$(l2_base) -DL2_SIZE=32\'d$(l2_size) -DL2_BANKS=$(l2_banks)
 vlog_defs += -DL1_BANK_SIZE=$(l1_bank_size)
-vlog_defs += -DBOOT_ADDR=$(boot_addr) -DXPULPIMG=$(xpulpimg)
+vlog_defs += -DBOOT_ADDR=32\'d$(boot_addr) -DXPULPIMG=$(xpulpimg)
 vlog_defs += -DSNITCH_TRACE=$(snitch_trace)
 vlog_defs += -DAXI_DATA_WIDTH=$(axi_data_width)
 vlog_defs += -DRO_LINE_WIDTH=$(ro_line_width)
 vlog_defs += -DDMAS_PER_GROUP=$(dmas_per_group)
 vlog_defs += -DAXI_HIER_RADIX=$(axi_hier_radix) -DAXI_MASTERS_PER_GROUP=$(axi_masters_per_group)
 vlog_defs += -DSEQ_MEM_SIZE=$(seq_mem_size) -DXQUEUE_SIZE=$(xqueue_size)
+# This parameter is only used for TeraPool configurations
+vlog_defs += -DNUM_SUB_GROUPS_PER_GROUP=$(num_sub_groups_per_group) -DREMOTE_GROUP_LATENCY_CYCLES=$(remote_group_latency_cycles)
 
 # Traffic generation enabled
 ifdef tg
@@ -148,7 +151,7 @@ $(buildpath)/compile.tcl: $(bender) $(config_mk) Makefile $(MEMPOOL_DIR)/Bender.
 .PHONY: sim
 sim: clean-dasm compile
 	cd $(buildpath) && \
-	$(questa_cmd) vsim -voptargs=+acc $(questa_args) $(library).$(top_level) -do ../scripts/questa/run.tcl
+	$(questa_cmd) vsim -voptargs=+acc $(questa_args) $(library).$(top_level) -do "set config ${config}" -do ../scripts/questa/run.tcl
 	./scripts/return_status.sh $(buildpath)/transcript
 
 .PHONY: simc

diff --git a/hardware/scripts/questa/wave.tcl b/hardware/scripts/questa/wave.tcl
@@ -30,27 +30,49 @@ add wave /mempool_tb/wfi
 
 # Add all cores from group 0 tile 0
 for {set core 0}  {$core < [examine -radix dec mempool_pkg::NumCoresPerTile]} {incr core} {
-    do ../scripts/questa/wave_core.tcl 0 0 $core
+    if {$config == {terapool}} {
+        do ../scripts/questa/wave_core.tcl 0 0 0 $core
+    } else {
+        do ../scripts/questa/wave_core.tcl 0 0 $core
+    }
 }
 
 # Add specific cores from different tiles
-do ../scripts/questa/wave_core.tcl 1 0 0
+if {$config == {terapool}} {
+    do ../scripts/questa/wave_core.tcl 1 0 0 0
+} else {
+    do ../scripts/questa/wave_core.tcl 1 0 0
+}
 
 # Add groups
 for {set group 0} {$group < [examine -radix dec /mempool_pkg::NumGroups]} {incr group} {
     # Add tiles
-    for {set tile 0} {$tile < [expr min(4,[examine -radix dec /mempool_pkg::NumTilesPerGroup])]} {incr tile} {
-        do ../scripts/questa/wave_tile.tcl $group $tile
+    if {$config == {terapool}} {
+        for {set subgroup 0} {$subgroup < [expr min(4,[examine -radix dec /mempool_pkg::NumSubGroupsPerGroup])]} {incr subgroup} {
+            for {set tile 0} {$tile < [expr min(4,[examine -radix dec /mempool_pkg::NumTilesPerSubGroup])]} {incr tile} {
+                do ../scripts/questa/wave_tile.tcl $group $subgroup $tile
+            }
+        }
+    } else {
+        for {set tile 0} {$tile < [expr min(4,[examine -radix dec /mempool_pkg::NumTilesPerGroup])]} {incr tile} {
+            do ../scripts/questa/wave_tile.tcl $group $tile
+        }
     }
 
     # Interconnects
     for {set tgtgroup 0} {$tgtgroup < [examine -radix dec /mempool_pkg::NumGroups]} {incr tgtgroup} {
         if {$tgtgroup != $group} {
             set interco_idx [expr $group ^ $tgtgroup]
-            add wave -group group_[$group] -group interconnect_to_group[$tgtgroup] /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/gen_remote_interco[$interco_idx]/i_remote_interco/*
+            if {$config == {terapool}} {
+                add wave -group group_[$group] -group interconnect_to_group[$tgtgroup] /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/gen_remote_interco[$interco_idx]/i_remote_interco/*
+            } else {
+                add wave -group group_[$group] -group interconnect_to_group[$tgtgroup] /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/gen_remote_interco[$interco_idx]/i_remote_interco/*
+            }
         }
     }
-    add wave -group group_[$group] -group interconnect_local /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_local_interco/*
+    if {$config != {terapool}} {
+        add wave -group group_[$group] -group interconnect_local /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_local_interco/*
+    }
 }
 
 # Add cluster
@@ -61,15 +83,30 @@ add wave -Group Control_Registers /mempool_tb/dut/i_ctrl_registers/*
 add wave -Group DMA /mempool_tb/dut/i_mempool_dma/*
 add wave -Group DMA -Group Reg /mempool_tb/dut/i_mempool_dma/i_mempool_dma_frontend_reg_top/*
 for {set group 0} {$group < [examine -radix dec /mempool_pkg::NumGroups]} {incr group} {
-  add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/NoMstPorts
-  add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionWidth
-  add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionStart
-  add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionEnd
-  add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionAddressBits
-  add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/FullRegionAddressBits
-  add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/*
-  for {set dma 0} {$dma < [examine -radix dec /mempool_pkg::NumDmasPerGroup]} {incr dma} {
-    add wave -Group DMA_${group}_${dma} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/gen_dmas[$dma]/i_axi_dma_backend/*
+  if  {$config == {terapool}} {
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/i_idma_distributed_midend/NoMstPorts
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/i_idma_distributed_midend/DmaRegionWidth
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/i_idma_distributed_midend/DmaRegionStart
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/i_idma_distributed_midend/DmaRegionEnd
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/i_idma_distributed_midend/DmaRegionAddressBits
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/i_idma_distributed_midend/FullRegionAddressBits
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/i_idma_distributed_midend/*
+    for {set subgroup 0} {$subgroup < [examine -radix dec /mempool_pkg::NumSubGroupsPerGroup]} {incr subgroup} {
+        for {set dma 0} {$dma < [examine -radix dec /mempool_pkg::NumDmasPerSubGroup]} {incr dma} {
+            add wave -Group DMA_${group}_${subgroup}_${dma} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/gen_rtl_group/i_group/gen_sub_groups[$subgroup]/gen_rtl_sg/i_sub_group/gen_dmas[$dma]/i_axi_dma_backend/*
+        }
+    }
+  } else {
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/NoMstPorts
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionWidth
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionStart
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionEnd
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/DmaRegionAddressBits
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/FullRegionAddressBits
+    add wave -Group DMA_midend_${group} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/i_idma_distributed_midend/*
+    for {set dma 0} {$dma < [examine -radix dec /mempool_pkg::NumDmasPerGroup]} {incr dma} {
+      add wave -Group DMA_${group}_${dma} /mempool_tb/dut/i_mempool_cluster/gen_groups[$group]/i_group/gen_dmas[$dma]/i_axi_dma_backend/*
+    }
   }
 }
 
@@ -84,4 +121,8 @@ add wave -Group DMA_midend_cluster /mempool_tb/dut/i_mempool_cluster/i_idma_dist
 
 add wave -Group DMA_split /mempool_tb/dut/i_mempool_cluster/i_idma_split_midend/*
 
-do ../scripts/questa/wave_cache.tcl 0 0 0
+if {$config == {terapool}} {
+  do ../scripts/questa/wave_cache.tcl 0 0 0 0
+} else {
+  do ../scripts/questa/wave_cache.tcl 0 0 0
+}