diff --git a/machines/arch_filelist.mk b/machines/arch_filelist.mk index 28e67bb6c..1d0294ccd 100644 --- a/machines/arch_filelist.mk +++ b/machines/arch_filelist.mk @@ -129,6 +129,7 @@ VSOURCES += $(BASEJUMP_STL_DIR)/bsg_async/bsg_async_ptr_gray.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_decode.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_dma.v +VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_dma_to_wormhole.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_miss.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_sbuf.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_sbuf_queue.v @@ -181,7 +182,6 @@ VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_manycore_tile_compute_array_ruche.v VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_manycore_tile_compute_ruche.v VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_manycore_tile_vcache_array.v VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_manycore_tile_vcache.v -VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_cache_dma_to_wormhole.v VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_manycore_hetero_socket.v VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_manycore_mesh_node.v VSOURCES += $(BSG_MANYCORE_DIR)/v/bsg_manycore_endpoint.v diff --git a/machines/sim_filelist.mk b/machines/sim_filelist.mk index 8835b3ff1..dc92c0ffa 100644 --- a/machines/sim_filelist.mk +++ b/machines/sim_filelist.mk @@ -32,7 +32,7 @@ VSOURCES += $(BASEJUMP_STL_DIR)/bsg_test/bsg_nonsynth_dramsim3.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_test/bsg_nonsynth_dramsim3_map.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_test/bsg_nonsynth_dramsim3_unmap.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_test/bsg_dramsim3.cpp -CSOURCES += $(BASEJUMP_STL_DIR)/imports/DRAMSim3/src/bankstate.cc +CSOURCES += $(BASEJUMP_STL_DIR)/imports/DRAMSim3/src/bankstate.cc CSOURCES += $(BASEJUMP_STL_DIR)/imports/DRAMSim3/src/channel_state.cc CSOURCES += $(BASEJUMP_STL_DIR)/imports/DRAMSim3/src/command_queue.cc CSOURCES += $(BASEJUMP_STL_DIR)/imports/DRAMSim3/src/common.cc @@ -50,6 +50,7 @@ VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_to_test_dram.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_to_test_dram_tx.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_to_test_dram_rx.v VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_cache_to_test_dram_rx_reorder.v +VSOURCES += $(BASEJUMP_STL_DIR)/bsg_cache/bsg_wormhole_to_cache_dma_fanout.v VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/router_profiler.v VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/vanilla_core_trace.v @@ -64,7 +65,6 @@ VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/bsg_nonsynth_manycore_io_co VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/bsg_nonsynth_manycore_spmd_loader.v VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/bsg_nonsynth_manycore_monitor.v VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/bsg_nonsynth_wormhole_test_mem.v -VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/bsg_manycore_vcache_wh_to_cache_dma.v VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/bsg_nonsynth_manycore_testbench.v VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/vcache_dma_to_dram_channel_map.v VSOURCES += $(BSG_MANYCORE_DIR)/testbenches/common/v/spmd_testbench.v diff --git a/testbenches/common/v/bsg_manycore_vcache_wh_to_cache_dma.v b/testbenches/common/v/bsg_manycore_vcache_wh_to_cache_dma.v deleted file mode 100644 index 10eaa3674..000000000 --- a/testbenches/common/v/bsg_manycore_vcache_wh_to_cache_dma.v +++ /dev/null @@ -1,448 +0,0 @@ -/** - * bsg_manycore_vcache_wh_to_cache_dma.v - * - * this converts vcache wh link to an array of cache dma interface to that it can be interfaced to - * bsg_cache_to_test_dram.v - * - * Intended to be used for simulation only. - */ - -`include "bsg_manycore_defines.vh" -`include "bsg_cache.vh" - -module bsg_manycore_vcache_wh_to_cache_dma - import bsg_cache_pkg::*; - import bsg_manycore_pkg::*; - #(parameter `BSG_INV_PARAM(wh_flit_width_p) - , parameter `BSG_INV_PARAM(wh_cid_width_p) - , parameter `BSG_INV_PARAM(wh_len_width_p) - , parameter `BSG_INV_PARAM(wh_cord_width_p) - , parameter `BSG_INV_PARAM(wh_ruche_factor_p) - - , parameter `BSG_INV_PARAM(num_vcaches_p) - , parameter `BSG_INV_PARAM(vcache_addr_width_p) - , parameter `BSG_INV_PARAM(vcache_data_width_p) - , parameter `BSG_INV_PARAM(vcache_dma_data_width_p) - , parameter `BSG_INV_PARAM(vcache_block_size_in_words_p) - - - , parameter `BSG_INV_PARAM(num_pods_x_p) - , parameter pod_start_x_p = 0 - , parameter `BSG_INV_PARAM(num_tiles_x_p) - , parameter lg_num_tiles_x_lp=`BSG_SAFE_CLOG2(num_tiles_x_p) - , parameter lg_num_pods_x_lp = `BSG_SAFE_CLOG2(num_pods_x_p) - - , parameter no_concentration_p = 0 - - // FIFO parameters - , parameter in_fifo_els_p = 8 - - , parameter lg_num_vcaches_lp=`BSG_SAFE_CLOG2(num_vcaches_p) - , parameter data_len_lp=(vcache_data_width_p*vcache_block_size_in_words_p/vcache_dma_data_width_p) - , parameter count_width_lp = `BSG_SAFE_CLOG2(data_len_lp) - - , parameter wh_link_sif_width_lp=`bsg_ready_and_link_sif_width(wh_flit_width_p) - , parameter dma_pkt_width_lp=`bsg_cache_dma_pkt_width(vcache_addr_width_p) - - , parameter lg_wh_ruche_factor_lp = `BSG_SAFE_CLOG2(wh_ruche_factor_p) - ) - ( - input clk_i - , input reset_i - - - // wormhole link - , input [wh_link_sif_width_lp-1:0] wh_link_sif_i - , output [wh_link_sif_width_lp-1:0] wh_link_sif_o - - - // cache DMA - , output logic [num_vcaches_p-1:0][dma_pkt_width_lp-1:0] dma_pkt_o - , output logic [num_vcaches_p-1:0] dma_pkt_v_o - , input [num_vcaches_p-1:0] dma_pkt_yumi_i - - , input [num_vcaches_p-1:0][vcache_dma_data_width_p-1:0] dma_data_i - , input [num_vcaches_p-1:0] dma_data_v_i - , output logic [num_vcaches_p-1:0] dma_data_ready_o - - , output logic [num_vcaches_p-1:0][vcache_dma_data_width_p-1:0] dma_data_o - , output logic [num_vcaches_p-1:0] dma_data_v_o - , input [num_vcaches_p-1:0] dma_data_yumi_i - ); - - - // structs - `declare_bsg_ready_and_link_sif_s(wh_flit_width_p,wh_link_sif_s); - `declare_bsg_cache_dma_pkt_s(vcache_addr_width_p); - `declare_bsg_manycore_vcache_wh_header_flit_s(wh_flit_width_p,wh_cord_width_p,wh_len_width_p,wh_cid_width_p); - - - // cast wormhole links - wh_link_sif_s wh_link_sif_in; - wh_link_sif_s wh_link_sif_out; - assign wh_link_sif_in = wh_link_sif_i; - assign wh_link_sif_o = wh_link_sif_out; - - - // Buffer incoming flits. - logic [wh_flit_width_p-1:0] in_fifo_data_lo; - logic in_fifo_yumi_li; - logic in_fifo_v_lo; - - bsg_fifo_1r1w_small #( - .els_p(in_fifo_els_p) - ,.width_p(wh_flit_width_p) - ) in_fifo ( - .clk_i (clk_i) - ,.reset_i (reset_i) - - ,.v_i (wh_link_sif_in.v) - ,.data_i (wh_link_sif_in.data) - ,.ready_o (wh_link_sif_out.ready_and_rev) - - ,.v_o (in_fifo_v_lo) - ,.data_o (in_fifo_data_lo) - ,.yumi_i (in_fifo_yumi_li) - ); - - - // DMA pkt going out - bsg_cache_dma_pkt_s dma_pkt_out; - for (genvar i = 0; i < num_vcaches_p; i++) begin - assign dma_pkt_o[i] = dma_pkt_out; - end - - - // header flits coming in and going out - bsg_manycore_vcache_wh_header_flit_s header_flit_in, header_flit_out; - assign header_flit_in = in_fifo_data_lo; - - - // cid, src_cord table - logic [num_vcaches_p-1:0][wh_cid_width_p-1:0] cid_r; - logic [num_vcaches_p-1:0][wh_cord_width_p-1:0] src_cord_r; - logic [wh_cid_width_p-1:0] cid_n; - logic [wh_cord_width_p-1:0] src_cord_n; - logic [lg_num_vcaches_lp-1:0] table_w_addr; - logic table_we; - - always_ff @ (posedge clk_i) begin - if (reset_i) begin - cid_r <= '0; - src_cord_r <= '0; - end - else begin - if (table_we) begin - cid_r[table_w_addr] <= cid_n; - src_cord_r[table_w_addr] <= src_cord_n; - end - end - end - - - - // send FSM - // receives wh packets and cache dma pkts. - typedef enum logic [1:0] { - SEND_RESET, - SEND_READY, - SEND_DMA_PKT, - SEND_EVICT_DATA - } send_state_e; - - send_state_e send_state_r, send_state_n; - logic write_not_read_r, write_not_read_n; - logic [lg_num_vcaches_lp-1:0] send_cache_id_r, send_cache_id_n; - - logic send_clear_li; - logic send_up_li; - logic [count_width_lp-1:0] send_count_lo; - bsg_counter_clear_up #( - .max_val_p(data_len_lp-1) - ,.init_val_p(0) - ) send_count ( - .clk_i(clk_i) - ,.reset_i(reset_i) - ,.clear_i(send_clear_li) - ,.up_i(send_up_li) - ,.count_o(send_count_lo) - ); - - wire [lg_num_vcaches_lp-1:0] send_cache_id; - - if (no_concentration_p) begin - assign send_cache_id = header_flit_in.src_cord[lg_wh_ruche_factor_lp+:lg_num_vcaches_lp]; - - end - else begin - if (num_pods_x_p == 1) begin - // For pod 1x1, there are 1 HBM on each side of west and east. - // Left half of top and bottom vcaches (16 total) maps to ch0 of HBM2 on west. - // Right half of top and bottom vcaches (16 total) maps to ch0 of HBM2 on east. - assign send_cache_id = { - (1)'(header_flit_in.cid/wh_ruche_factor_p), - header_flit_in.src_cord[lg_num_tiles_x_lp-2:0] - }; - end - else begin - // The left half of the pod array maps to HBM2 on the left side, and the right half on the right. - // HBM2 channels are allocated to pods starting from the top left corner. - // Within a pod, a row of vcaches (16) is allocated to a channel, so that there is one-to-one mapping from - // vcache to HBM2 bank. - // - // - // For pod 4x4 - // - // [dev0-ch0] [dev0-ch2] [dev2-ch0] [dev2-ch2] - // [ m c ] [ mc ] [ m c ] [ mc ] - // [dev0-ch1] [dev0-ch3] [dev2-ch1] [dev2-ch3] - // - // [dev0-ch4] [dev0-ch6] [dev2-ch4] [dev2-ch6] - // [ m c ] [ mc ] [ m c ] [ mc ] - // [dev0-ch5] [dev0-ch7] [dev2-ch5] [dev2-ch7] - // - // [dev1-ch0] [dev1-ch2] [dev3-ch0] [dev3-ch2] - // [ m c ] [ mc ] [ m c ] [ mc ] - // [dev1-ch1] [dev0-ch3] [dev3-ch1] [dev3-ch3] - // - // [dev1-ch4] [dev1-ch6] [dev3-ch4] [dev3-ch6] - // [ m c ] [ mc ] [ m c ] [ mc ] - // [dev1-ch5] [dev1-ch7] [dev3-ch5] [dev3-ch7] - // - assign send_cache_id = { - (lg_num_pods_x_lp-1)'((header_flit_in.src_cord[wh_cord_width_p-1:lg_num_tiles_x_lp] - pod_start_x_p)%(num_pods_x_p/2)), - (1)'(header_flit_in.cid/wh_ruche_factor_p), - header_flit_in.src_cord[lg_num_tiles_x_lp-1:0] - }; - end - end - - - - always_comb begin - send_state_n = send_state_r; - write_not_read_n = write_not_read_r; - send_cache_id_n = send_cache_id_r; - table_we = 1'b0; - table_w_addr = '0; - src_cord_n = '0; - cid_n = '0; - - in_fifo_yumi_li = 1'b0; - - send_clear_li = 1'b0; - send_up_li = 1'b0; - dma_pkt_v_o = '0; - dma_pkt_out = '0; - - dma_data_v_o = '0; - dma_data_o = '0; - - case (send_state_r) - // coming out of reset - SEND_RESET: begin - send_state_n = SEND_READY; - end - - // wait for a header flit. - // store the write_not_read, src_cord. - // save the cid in a table. - SEND_READY: begin - if (in_fifo_v_lo) begin - in_fifo_yumi_li = 1'b1; - write_not_read_n = header_flit_in.write_not_read; - src_cord_n = header_flit_in.src_cord; - cid_n = header_flit_in.cid; - table_w_addr = send_cache_id; - table_we = 1'b1; - send_cache_id_n = send_cache_id; - send_state_n = SEND_DMA_PKT; - end - end - - // take the addr flit and send out the dma pkt. - // For read, return to SEND_READY. - // For write, move to SEND_EVICT_DATA to pass the evict data. - SEND_DMA_PKT: begin - dma_pkt_v_o[send_cache_id_r] = in_fifo_v_lo; - dma_pkt_out.write_not_read = write_not_read_r; - dma_pkt_out.addr = vcache_addr_width_p'(in_fifo_data_lo); - - in_fifo_yumi_li = dma_pkt_yumi_i[send_cache_id_r]; - send_state_n = dma_pkt_yumi_i[send_cache_id_r] - ? (write_not_read_r ? SEND_EVICT_DATA : SEND_READY) - : SEND_DMA_PKT; - end - - // once all evict data has been passed along return to SEND_READY - SEND_EVICT_DATA: begin - dma_data_v_o[send_cache_id_r] = in_fifo_v_lo; - dma_data_o[send_cache_id_r] = in_fifo_data_lo; - if (dma_data_yumi_i[send_cache_id_r]) begin - in_fifo_yumi_li = 1'b1; - send_up_li = send_count_lo != data_len_lp-1; - send_clear_li = send_count_lo == data_len_lp-1; - send_state_n = (send_count_lo == data_len_lp-1) - ? SEND_READY - : SEND_EVICT_DATA; - end - end - - endcase - - end - - - - always_ff @ (posedge clk_i) begin - if (reset_i) begin - send_state_r <= SEND_RESET; - write_not_read_r <= 1'b0; - send_cache_id_r <= '0; - end - else begin - send_state_r <= send_state_n; - write_not_read_r <= write_not_read_n; - send_cache_id_r <= send_cache_id_n; - end - end - - - - // receiver FSM - // receives dma_data_i and send them to the vcaches using wh link. - typedef enum logic [1:0] { - RECV_RESET, - RECV_READY, - RECV_HEADER, - RECV_FILL_DATA - } recv_state_e; - - recv_state_e recv_state_r, recv_state_n; - logic [lg_num_vcaches_lp-1:0] recv_cache_id_r, recv_cache_id_n; - - - logic rr_v_lo; - logic rr_yumi_li; - logic [lg_num_vcaches_lp-1:0] rr_addr_lo; - logic [num_vcaches_p-1:0] rr_grants_lo; - bsg_arb_round_robin #( - .width_p(num_vcaches_p) - ) rr0 ( - .clk_i(clk_i) - ,.reset_i(reset_i) - - ,.reqs_i(dma_data_v_i) - ,.grants_o(rr_grants_lo) - ,.yumi_i(rr_yumi_li) - ); - assign rr_v_lo = |dma_data_v_i; - bsg_encode_one_hot #( - .width_p(num_vcaches_p) - ) eoh ( - .i(rr_grants_lo) - ,.addr_o(rr_addr_lo) - ,.v_o() - ); - - - logic recv_clear_li; - logic recv_up_li; - logic [count_width_lp-1:0] recv_count_lo; - bsg_counter_clear_up #( - .max_val_p(data_len_lp-1) - ,.init_val_p(0) - ) recv_count ( - .clk_i(clk_i) - ,.reset_i(reset_i) - ,.clear_i(recv_clear_li) - ,.up_i(recv_up_li) - ,.count_o(recv_count_lo) - ); - - - - - always_comb begin - - wh_link_sif_out.v = 1'b0; - wh_link_sif_out.data = '0; - - rr_yumi_li = 1'b0; - - recv_state_n = recv_state_r; - recv_cache_id_n = recv_cache_id_r; - - recv_clear_li = 1'b0; - recv_up_li = 1'b0; - - header_flit_out.unused = '0; - header_flit_out.write_not_read = 1'b0; // dont matter - header_flit_out.src_cord = '0; // dont matter - header_flit_out.cid = cid_r[recv_cache_id_r]; - header_flit_out.len = data_len_lp; - header_flit_out.dest_cord = src_cord_r[recv_cache_id_r]; - - dma_data_ready_o = '0; - - case (recv_state_r) - - // coming out of reset - RECV_RESET: begin - recv_state_n = RECV_READY; - end - - // wait for one of dma_data_v_i to be 1. - // save the cache id. - RECV_READY: begin - if (rr_v_lo) begin - rr_yumi_li = 1'b1; - recv_cache_id_n = rr_addr_lo; - recv_state_n = RECV_HEADER; - end - end - - // send out header to dest vcache - RECV_HEADER: begin - wh_link_sif_out.v = 1'b1; - wh_link_sif_out.data = header_flit_out; - if (wh_link_sif_in.ready_and_rev) begin - recv_state_n = RECV_FILL_DATA; - end - end - - // send the data flits to the vcache. - // once it's done, go back to RECV_READY. - RECV_FILL_DATA: begin - wh_link_sif_out.v = dma_data_v_i[recv_cache_id_r]; - wh_link_sif_out.data = dma_data_i[recv_cache_id_r]; - dma_data_ready_o[recv_cache_id_r] = wh_link_sif_in.ready_and_rev; - if (dma_data_v_i[recv_cache_id_r] & wh_link_sif_in.ready_and_rev) begin - recv_clear_li = (recv_count_lo == data_len_lp-1); - recv_up_li = (recv_count_lo != data_len_lp-1); - recv_state_n = (recv_count_lo == data_len_lp-1) - ? RECV_READY - : RECV_FILL_DATA; - end - end - - endcase - end - - - always_ff @ (posedge clk_i) begin - if (reset_i) begin - recv_state_r <= RECV_RESET; - recv_cache_id_r <= '0; - end - else begin - recv_state_r <= recv_state_n; - recv_cache_id_r <= recv_cache_id_n; - end - end - - -endmodule - -`BSG_ABSTRACT_MODULE(bsg_manycore_vcache_wh_to_cache_dma) - diff --git a/testbenches/common/v/bsg_nonsynth_manycore_testbench.v b/testbenches/common/v/bsg_nonsynth_manycore_testbench.v index 3f62ec643..b482aed55 100644 --- a/testbenches/common/v/bsg_nonsynth_manycore_testbench.v +++ b/testbenches/common/v/bsg_nonsynth_manycore_testbench.v @@ -342,6 +342,8 @@ module bsg_nonsynth_manycore_testbench parameter num_total_channels_lp = num_total_vcaches_lp/num_vcaches_per_channel_p; parameter num_dram_lp = `BSG_CDIV(num_total_channels_lp,hbm2_num_channels_p); + parameter lg_wh_ruche_factor_lp = `BSG_SAFE_CLOG2(wh_ruche_factor_p); + parameter lg_num_vcaches_per_link_lp = `BSG_SAFE_CLOG2(num_vcaches_per_link_lp); // WH to cache dma `declare_bsg_cache_dma_pkt_s(vcache_addr_width_p); @@ -363,28 +365,77 @@ module bsg_nonsynth_manycore_testbench for (genvar k = N; k <= S; k++) begin: py for (genvar n = 0; n < num_vcache_rows_p; n++) begin: row for (genvar r = 0; r < wh_ruche_factor_p; r++) begin: rf - bsg_manycore_vcache_wh_to_cache_dma #( - .wh_flit_width_p(wh_flit_width_p) - ,.wh_cid_width_p(wh_cid_width_p) - ,.wh_len_width_p(wh_len_width_p) - ,.wh_cord_width_p(wh_cord_width_p) - ,.wh_ruche_factor_p(wh_ruche_factor_p) - ,.num_vcaches_p(num_vcaches_per_link_lp) - ,.vcache_addr_width_p(vcache_addr_width_p) - ,.vcache_data_width_p(vcache_data_width_p) - ,.vcache_dma_data_width_p(vcache_dma_data_width_p) - ,.vcache_block_size_in_words_p(vcache_block_size_in_words_p) + // This code is preserved from the previous mapping algorithm. For this specific + // testbench, only "no_concentration_p" is used + // if (no_concentration_p) begin + // assign send_cache_id = header_flit_in.src_cord[lg_wh_ruche_factor_lp+:lg_num_vcaches_lp]; + // + // end + // else begin + // if (num_pods_x_p == 1) begin + // // For pod 1x1, there are 1 HBM on each side of west and east. + // // Left half of top and bottom vcaches (16 total) maps to ch0 of HBM2 on west. + // // Right half of top and bottom vcaches (16 total) maps to ch0 of HBM2 on east. + // assign send_cache_id = { + // (1)'(header_flit_in.cid/wh_ruche_factor_p), + // header_flit_in.src_cord[lg_num_tiles_x_lp-2:0] + // }; + // end + // else begin + // // The left half of the pod array maps to HBM2 on the left side, and the right half on the right. + // // HBM2 channels are allocated to pods starting from the top left corner. + // // Within a pod, a row of vcaches (16) is allocated to a channel, so that there is one-to-one mapping from + // // vcache to HBM2 bank. + // // + // // + // // For pod 4x4 + // // + // // [dev0-ch0] [dev0-ch2] [dev2-ch0] [dev2-ch2] + // // [ m c ] [ mc ] [ m c ] [ mc ] + // // [dev0-ch1] [dev0-ch3] [dev2-ch1] [dev2-ch3] + // // + // // [dev0-ch4] [dev0-ch6] [dev2-ch4] [dev2-ch6] + // // [ m c ] [ mc ] [ m c ] [ mc ] + // // [dev0-ch5] [dev0-ch7] [dev2-ch5] [dev2-ch7] + // // + // // [dev1-ch0] [dev1-ch2] [dev3-ch0] [dev3-ch2] + // // [ m c ] [ mc ] [ m c ] [ mc ] + // // [dev1-ch1] [dev0-ch3] [dev3-ch1] [dev3-ch3] + // // + // // [dev1-ch4] [dev1-ch6] [dev3-ch4] [dev3-ch6] + // // [ m c ] [ mc ] [ m c ] [ mc ] + // // [dev1-ch5] [dev1-ch7] [dev3-ch5] [dev3-ch7] + // // + // assign send_cache_id = { + // (lg_num_pods_x_lp-1)'((header_flit_in.src_cord[wh_cord_width_p-1:lg_num_tiles_x_lp] - pod_start_x_p)%(num_pods_x_p/2)), + // (1)'(header_flit_in.cid/wh_ruche_factor_p), + // header_flit_in.src_cord[lg_num_tiles_x_lp-1:0] + // }; + // end + + `declare_bsg_cache_wh_header_flit_s(wh_flit_width_p,wh_cord_width_p,wh_len_width_p,wh_cid_width_p); + bsg_cache_wh_header_flit_s header_flit_in; + assign header_flit_in = buffered_wh_link_sif_lo[i][j][k][n][r]; + + wire [lg_num_vcaches_per_link_lp-1:0] dma_id_li = + header_flit_in.src_cord[lg_wh_ruche_factor_lp+:lg_num_vcaches_per_link_lp]; + + bsg_wormhole_to_cache_dma_fanout#( + .num_dma_p(num_vcaches_per_link_lp) + ,.dma_addr_width_p(vcache_addr_width_p) + ,.dma_burst_len_p(vcache_block_size_in_words_p*vcache_data_width_p/vcache_dma_data_width_p) - ,.no_concentration_p(1) - ,.num_pods_x_p(num_pods_x_p) - ,.pod_start_x_p(1) - ,.num_tiles_x_p(num_tiles_x_p) + ,.wh_flit_width_p(wh_flit_width_p) + ,.wh_cid_width_p(wh_cid_width_p) + ,.wh_len_width_p(wh_len_width_p) + ,.wh_cord_width_p(wh_cord_width_p) ) wh_to_dma ( .clk_i(clk_i) ,.reset_i(reset_r) ,.wh_link_sif_i (buffered_wh_link_sif_lo[i][j][k][n][r]) + ,.wh_dma_id_i (dma_id_li) ,.wh_link_sif_o (buffered_wh_link_sif_li[i][j][k][n][r]) ,.dma_pkt_o (dma_pkt_lo[i][j][k][n][r]) @@ -393,7 +444,7 @@ module bsg_nonsynth_manycore_testbench ,.dma_data_i (dma_data_li[i][j][k][n][r]) ,.dma_data_v_i (dma_data_v_li[i][j][k][n][r]) - ,.dma_data_ready_o (dma_data_ready_lo[i][j][k][n][r]) + ,.dma_data_ready_and_o (dma_data_ready_lo[i][j][k][n][r]) ,.dma_data_o (dma_data_lo[i][j][k][n][r]) ,.dma_data_v_o (dma_data_v_lo[i][j][k][n][r]) diff --git a/testbenches/common/v/bsg_nonsynth_wormhole_test_mem.v b/testbenches/common/v/bsg_nonsynth_wormhole_test_mem.v index 4c6fa7b9c..32321d284 100644 --- a/testbenches/common/v/bsg_nonsynth_wormhole_test_mem.v +++ b/testbenches/common/v/bsg_nonsynth_wormhole_test_mem.v @@ -75,9 +75,9 @@ module bsg_nonsynth_wormhole_test_mem assign wh_link_sif_o = wh_link_sif_out; - `declare_bsg_manycore_vcache_wh_header_flit_s(wh_flit_width_p,wh_cord_width_p,wh_len_width_p,wh_cid_width_p); + `declare_bsg_cache_wh_header_flit_s(wh_flit_width_p,wh_cord_width_p,wh_len_width_p,wh_cid_width_p); - bsg_manycore_vcache_wh_header_flit_s header_flit_in; + bsg_cache_wh_header_flit_s header_flit_in; assign header_flit_in = wh_link_sif_in.data; logic clear_li; @@ -109,15 +109,16 @@ module bsg_nonsynth_wormhole_test_mem logic write_not_read_r, write_not_read_n; logic [wh_flit_width_p-1:0] addr_r, addr_n; logic [wh_cord_width_p-1:0] src_cord_r, src_cord_n; - logic [wh_cid_width_p-1:0] cid_r, cid_n; + logic [wh_cid_width_p-1:0] src_cid_r, src_cid_n; - bsg_manycore_vcache_wh_header_flit_s header_flit_out; + bsg_cache_wh_header_flit_s header_flit_out; assign header_flit_out.unused = '0; assign header_flit_out.write_not_read = '0; // dont care assign header_flit_out.src_cord = '0; // dont care - assign header_flit_out.cid = cid_r; + assign header_flit_out.src_cid = '0; // dont care + assign header_flit_out.cid = src_cid_r; assign header_flit_out.len = wh_len_width_p'(data_len_lp); - assign header_flit_out.dest_cord = src_cord_r; + assign header_flit_out.cord = src_cord_r; always_comb begin wh_link_sif_out = '0; @@ -127,7 +128,7 @@ module bsg_nonsynth_wormhole_test_mem write_not_read_n = write_not_read_r; addr_n = addr_r; src_cord_n = src_cord_r; - cid_n = cid_r; + src_cid_n = src_cid_r; mem_state_n = mem_state_r; mem_we = 1'b0; @@ -145,7 +146,7 @@ module bsg_nonsynth_wormhole_test_mem if (wh_link_sif_in.v) begin write_not_read_n = header_flit_in.write_not_read; src_cord_n = header_flit_in.src_cord; - cid_n = header_flit_in.cid; + src_cid_n = header_flit_in.src_cid; mem_state_n = RECV_ADDR; end end @@ -214,7 +215,7 @@ module bsg_nonsynth_wormhole_test_mem else begin // wh ruche links coming from top and bottom caches are concentrated into one link. assign mem_addr = { - (1)'(cid_r/wh_ruche_factor_p), // determine north or south vcache + (1)'(src_cid_r/wh_ruche_factor_p), // determine north or south vcache src_cord_r[0+:(lg_num_vcaches_lp-1)], addr_r[block_offset_width_lp+:mem_addr_width_lp-lg_num_vcaches_lp-count_width_lp], count_lo @@ -230,14 +231,14 @@ module bsg_nonsynth_wormhole_test_mem mem_state_r <= RESET; write_not_read_r <= 1'b0; src_cord_r <= '0; - cid_r <= '0; + src_cid_r <= '0; addr_r <= '0; end else begin mem_state_r <= mem_state_n; write_not_read_r <= write_not_read_n; src_cord_r <= src_cord_n; - cid_r <= cid_n; + src_cid_r <= src_cid_n; addr_r <= addr_n; end end diff --git a/testbenches/common/v/bsg_nonsynth_wormhole_test_mem_with_dma.v b/testbenches/common/v/bsg_nonsynth_wormhole_test_mem_with_dma.v index c2d46c5fd..494e9e163 100644 --- a/testbenches/common/v/bsg_nonsynth_wormhole_test_mem_with_dma.v +++ b/testbenches/common/v/bsg_nonsynth_wormhole_test_mem_with_dma.v @@ -85,12 +85,11 @@ module bsg_nonsynth_wormhole_test_mem_with_dma assign wh_link_sif_o = wh_link_sif_out; - `declare_bsg_manycore_vcache_wh_header_flit_s(wh_flit_width_p,wh_cord_width_p,wh_len_width_p,wh_cid_width_p); + `declare_bsg_cache_wh_header_flit_s(wh_flit_width_p,wh_cord_width_p,wh_len_width_p,wh_cid_width_p); - bsg_manycore_vcache_wh_header_flit_s header_flit_in; + bsg_cache_wh_header_flit_s header_flit_in; assign header_flit_in = wh_link_sif_in.data; - // dma mem read data -> wormhole read data logic piso_ready_lo; @@ -178,13 +177,14 @@ module bsg_nonsynth_wormhole_test_mem_with_dma logic write_not_read_r, write_not_read_n; logic [wh_flit_width_p-1:0] addr_r, addr_n; logic [wh_cord_width_p-1:0] src_cord_r, src_cord_n; - logic [wh_cid_width_p-1:0] cid_r, cid_n; + logic [wh_cid_width_p-1:0] src_cid_r, src_cid_n; - bsg_manycore_vcache_wh_header_flit_s header_flit_out; + bsg_cache_wh_header_flit_s header_flit_out; assign header_flit_out.unused = '0; assign header_flit_out.write_not_read = '0; // dont care assign header_flit_out.src_cord = '0; // dont care - assign header_flit_out.cid = cid_r; + assign header_flit_out.src_cid = '0; // dont care + assign header_flit_out.cid = src_cid_r; assign header_flit_out.len = wh_len_width_p'(data_len_lp); assign header_flit_out.dest_cord = src_cord_r; @@ -196,7 +196,7 @@ module bsg_nonsynth_wormhole_test_mem_with_dma write_not_read_n = write_not_read_r; addr_n = addr_r; src_cord_n = src_cord_r; - cid_n = cid_r; + src_cid_n = src_cid_r; mem_state_n = mem_state_r; dma_mem_v_n = '0; @@ -219,7 +219,7 @@ module bsg_nonsynth_wormhole_test_mem_with_dma if (wh_link_sif_in.v) begin write_not_read_n = header_flit_in.write_not_read; src_cord_n = header_flit_in.src_cord; - cid_n = header_flit_in.cid; + src_cid_n = header_flit_in.src_cid; mem_state_n = RECV_ADDR; end end @@ -305,7 +305,7 @@ module bsg_nonsynth_wormhole_test_mem_with_dma else begin // wh ruche links coming from top and bottom caches are concentrated into one link. assign dma_mem_addr = { - (1)'(cid_r/wh_ruche_factor_p), + (1)'(src_cid_r/wh_ruche_factor_p), cord[0+:(lg_num_vcaches_lp-1)], addr_r[block_offset_width_lp+:dma_mem_addr_width_lp-lg_num_vcaches_lp] }; @@ -318,7 +318,7 @@ module bsg_nonsynth_wormhole_test_mem_with_dma mem_state_r <= RESET; write_not_read_r <= 1'b0; src_cord_r <= '0; - cid_r <= '0; + src_cid_r <= '0; addr_r <= '0; dma_mem_v_r <= '0; dma_mem_w_r <= '0; @@ -330,7 +330,7 @@ module bsg_nonsynth_wormhole_test_mem_with_dma mem_state_r <= mem_state_n; write_not_read_r <= write_not_read_n; src_cord_r <= src_cord_n; - cid_r <= cid_n; + src_cid_r <= src_cid_n; addr_r <= addr_n; dma_mem_v_r <= dma_mem_v_n; dma_mem_w_r <= dma_mem_w_n; @@ -352,11 +352,11 @@ module bsg_nonsynth_wormhole_test_mem_with_dma end RECV_EVICT_DATA: begin $display("[DEBUG] WH MEM: id = %d: %s: cid = %d, addr_r = %08x, src_cord_r = %04x, cord = %04x, dma_mem_addr = %08x, wh_data_n = %08x, count_lo = %d, wh_data_v_n = %b, wh_data_v_r = %08x", - id_p, mem_state_r.name(), cid_r, addr_r, src_cord_r, cord, dma_mem_addr, wh_data_n, count_lo, wh_data_v_n, wh_data_v_r); + id_p, mem_state_r.name(), src_cid_r, addr_r, src_cord_r, cord, dma_mem_addr, wh_data_n, count_lo, wh_data_v_n, wh_data_v_r); end default: begin $display("[DEBUG] WH MEM: id = %d: %s: cid = %d, addr_r = %08x, src_cord_r = %04x, cord = %04x, dma_mem_addr = %08x, piso_data_lo = %08x, count_lo = %d, dma_mem_v_r = %b, dma_data_v_r = %b", - id_p, mem_state_r.name(), cid_r, addr_r, src_cord_r,cord, dma_mem_addr, piso_data_lo, count_lo, dma_mem_v_r, dma_data_v_r); + id_p, mem_state_r.name(), src_cid_r, addr_r, src_cord_r,cord, dma_mem_addr, piso_data_lo, count_lo, dma_mem_v_r, dma_data_v_r); end endcase // case (mem_state_r) end // if (~reset_i) diff --git a/v/bsg_cache_dma_to_wormhole.v b/v/bsg_cache_dma_to_wormhole.v deleted file mode 100644 index 26a64bb07..000000000 --- a/v/bsg_cache_dma_to_wormhole.v +++ /dev/null @@ -1,292 +0,0 @@ -/** - * bsg_cache_dma_to_wormhole.v - * - * this module interfaces cache DMA to wormhole router. - * when this module receives a write dma packet from the cache, it sends - * write header flit with evict data following. - * for read dma packets, it sends the read header flit, and receives the fill data asynchronously. - */ - -`include "bsg_manycore_defines.vh" -`include "bsg_cache.vh" - -module bsg_cache_dma_to_wormhole - import bsg_cache_pkg::*; - import bsg_manycore_pkg::*; - #(`BSG_INV_PARAM(vcache_addr_width_p) - , `BSG_INV_PARAM(vcache_data_width_p) - , `BSG_INV_PARAM(vcache_dma_data_width_p) - , `BSG_INV_PARAM(vcache_block_size_in_words_p) - - // flit width should match the vcache dma width. - , `BSG_INV_PARAM(wh_flit_width_p) - , `BSG_INV_PARAM(wh_cid_width_p) - , `BSG_INV_PARAM(wh_len_width_p) - , `BSG_INV_PARAM(wh_cord_width_p) - - , localparam data_len_lp = (vcache_data_width_p*vcache_block_size_in_words_p/vcache_dma_data_width_p) - - , dma_pkt_width_lp=`bsg_cache_dma_pkt_width(vcache_addr_width_p) - , wh_link_sif_width_lp = - `bsg_ready_and_link_sif_width(wh_flit_width_p) - ) - ( - input clk_i - , input reset_i - - , input [dma_pkt_width_lp-1:0] dma_pkt_i - , input dma_pkt_v_i - , output dma_pkt_yumi_o - - , output logic [vcache_dma_data_width_p-1:0] dma_data_o - , output logic dma_data_v_o - , input dma_data_ready_i - - , input [vcache_dma_data_width_p-1:0] dma_data_i - , input dma_data_v_i - , output logic dma_data_yumi_o - - , input [wh_link_sif_width_lp-1:0] wh_link_sif_i - , output [wh_link_sif_width_lp-1:0] wh_link_sif_o - - , input [wh_cord_width_p-1:0] my_wh_cord_i - , input [wh_cord_width_p-1:0] dest_wh_cord_i - , input [wh_cid_width_p-1:0] my_wh_cid_i - ); - - - `declare_bsg_cache_dma_pkt_s(vcache_addr_width_p); - `declare_bsg_ready_and_link_sif_s(wh_flit_width_p, wh_link_sif_s); - wh_link_sif_s wh_link_sif_in; - wh_link_sif_s wh_link_sif_out; - assign wh_link_sif_in = wh_link_sif_i; - assign wh_link_sif_o = wh_link_sif_out; - - // dma pkt fifo - logic dma_pkt_ready_lo; - logic dma_pkt_v_lo; - logic dma_pkt_yumi_li; - bsg_cache_dma_pkt_s dma_pkt_lo; - - bsg_two_fifo #( - .width_p(dma_pkt_width_lp) - ) dma_pkt_fifo ( - .clk_i(clk_i) - ,.reset_i(reset_i) - - ,.v_i(dma_pkt_v_i) - ,.data_i(dma_pkt_i) - ,.ready_o(dma_pkt_ready_lo) - - ,.v_o(dma_pkt_v_lo) - ,.data_o(dma_pkt_lo) - ,.yumi_i(dma_pkt_yumi_li) - ); - - assign dma_pkt_yumi_o = dma_pkt_ready_lo & dma_pkt_v_i; - - // FIFO for wormhole flits coming back to vcache. - logic return_fifo_v_lo; - logic [wh_flit_width_p-1:0] return_fifo_data_lo; - logic return_fifo_yumi_li; - - bsg_two_fifo #( - .width_p(wh_flit_width_p) - ) return_fifo ( - .clk_i (clk_i) - ,.reset_i (reset_i) - - ,.v_i (wh_link_sif_in.v) - ,.data_i (wh_link_sif_in.data) - ,.ready_o (wh_link_sif_out.ready_and_rev) - - ,.v_o (return_fifo_v_lo) - ,.data_o (return_fifo_data_lo) - ,.yumi_i (return_fifo_yumi_li) - ); - - - // counter - localparam count_width_lp = `BSG_SAFE_CLOG2(data_len_lp); - logic send_clear_li; - logic send_up_li; - logic [count_width_lp-1:0] send_count_lo; - - bsg_counter_clear_up #( - .max_val_p(data_len_lp-1) - ,.init_val_p(0) - ) send_count ( - .clk_i(clk_i) - ,.reset_i(reset_i) - ,.clear_i(send_clear_li) - ,.up_i(send_up_li) - ,.count_o(send_count_lo) - ); - - // send FSM - typedef enum logic [1:0] { - SEND_RESET - , SEND_READY - , SEND_ADDR - , SEND_DATA - } send_state_e; - - - send_state_e send_state_r, send_state_n; - logic wh_flit_valid; - logic [wh_flit_width_p-1:0] wh_flit_out; - assign wh_link_sif_out.v = wh_flit_valid; - assign wh_link_sif_out.data = wh_flit_out; - - `declare_bsg_manycore_vcache_wh_header_flit_s(wh_flit_width_p,wh_cord_width_p,wh_len_width_p,wh_cid_width_p); - - bsg_manycore_vcache_wh_header_flit_s header_flit; - assign header_flit.unused = '0; - assign header_flit.write_not_read = dma_pkt_lo.write_not_read; - assign header_flit.src_cord = my_wh_cord_i; - assign header_flit.cid = my_wh_cid_i; - assign header_flit.len = dma_pkt_lo.write_not_read - ? wh_len_width_p'(1+data_len_lp) // header + addr + data - : wh_len_width_p'(1); // header + addr - assign header_flit.dest_cord = dest_wh_cord_i; - - - always_comb begin - - send_state_n = send_state_r; - dma_pkt_yumi_li = 1'b0; - send_clear_li = 1'b0; - send_up_li = 1'b0; - wh_flit_valid = 1'b0; - wh_flit_out = dma_data_i; - dma_data_yumi_o = 1'b0; - - case (send_state_r) - SEND_RESET: begin - send_state_n = SEND_READY; - end - - SEND_READY: begin - wh_flit_out = header_flit; - if (dma_pkt_v_lo) begin - wh_flit_valid = 1'b1; - send_state_n = wh_link_sif_in.ready_and_rev - ? SEND_ADDR - : SEND_READY; - end - end - - SEND_ADDR: begin - wh_flit_out = wh_flit_width_p'(dma_pkt_lo.addr); - if (dma_pkt_v_lo) begin - wh_flit_valid = 1'b1; - dma_pkt_yumi_li = wh_link_sif_in.ready_and_rev; - send_state_n = wh_link_sif_in.ready_and_rev - ? (dma_pkt_lo.write_not_read ? SEND_DATA : SEND_READY) - : SEND_ADDR; - end - end - - SEND_DATA: begin - wh_flit_out = dma_data_i; - if (dma_data_v_i) begin - wh_flit_valid = 1'b1; - send_up_li = (send_count_lo != data_len_lp-1) & wh_link_sif_in.ready_and_rev; - send_clear_li = (send_count_lo == data_len_lp-1) & wh_link_sif_in.ready_and_rev; - dma_data_yumi_o = wh_link_sif_in.ready_and_rev; - send_state_n = (send_count_lo == data_len_lp-1) & wh_link_sif_in.ready_and_rev - ? SEND_READY - : SEND_DATA; - end - end - - // should never happen - default: begin - send_state_n = SEND_READY; - end - endcase - end - - - - - // receiver FSM - logic recv_clear_li; - logic recv_up_li; - logic [count_width_lp-1:0] recv_count_lo; - - bsg_counter_clear_up #( - .max_val_p(data_len_lp-1) - ,.init_val_p(0) - ) recv_count ( - .clk_i(clk_i) - ,.reset_i(reset_i) - ,.clear_i(recv_clear_li) - ,.up_i(recv_up_li) - ,.count_o(recv_count_lo) - ); - - typedef enum logic [1:0] { - RECV_RESET - , RECV_READY - , RECV_DATA - } recv_state_e; - - recv_state_e recv_state_r, recv_state_n; - - - always_comb begin - recv_state_n = recv_state_r; - recv_clear_li = 1'b0; - recv_up_li = 1'b0; - return_fifo_yumi_li = 1'b0; - dma_data_v_o = 1'b0; - dma_data_o = return_fifo_data_lo; - - case (recv_state_r) - RECV_RESET: begin - recv_state_n = RECV_READY; - end - - RECV_READY: begin - return_fifo_yumi_li = return_fifo_v_lo; - recv_state_n = return_fifo_v_lo - ? RECV_DATA - : RECV_READY; - end - - RECV_DATA: begin - return_fifo_yumi_li = return_fifo_v_lo & dma_data_ready_i; - dma_data_v_o = return_fifo_v_lo; - recv_clear_li = return_fifo_v_lo & dma_data_ready_i & (recv_count_lo == data_len_lp-1); - recv_up_li = return_fifo_v_lo & dma_data_ready_i & (recv_count_lo != data_len_lp-1); - recv_state_n = return_fifo_v_lo & dma_data_ready_i & (recv_count_lo == data_len_lp-1) - ? RECV_READY - : RECV_DATA; - end - - default: begin - recv_state_n = RECV_READY; - end - endcase - end - - - - - // sequential logic - always_ff @ (posedge clk_i) begin - if (reset_i) begin - send_state_r <= SEND_RESET; - recv_state_r <= RECV_RESET; - end - else begin - send_state_r <= send_state_n; - recv_state_r <= recv_state_n; - end - end - - -endmodule - -`BSG_ABSTRACT_MODULE(bsg_cache_dma_to_wormhole) diff --git a/v/bsg_manycore_defines.vh b/v/bsg_manycore_defines.vh index e2ed39404..fbe5007b5 100644 --- a/v/bsg_manycore_defines.vh +++ b/v/bsg_manycore_defines.vh @@ -151,17 +151,5 @@ link_rev[x_cord_width_mp-1:0] \ } - - // vcache DMA wormhole header flit format - `define declare_bsg_manycore_vcache_wh_header_flit_s(wh_flit_width_mp,wh_cord_width_mp,wh_len_width_mp,wh_cid_width_mp) \ - typedef struct packed { \ - logic [wh_flit_width_mp-(wh_cord_width_mp*2)-1-wh_len_width_mp-wh_cid_width_mp-1:0] unused; \ - logic write_not_read; \ - logic [wh_cord_width_mp-1:0] src_cord; \ - logic [wh_cid_width_mp-1:0] cid; \ - logic [wh_len_width_mp-1:0] len; \ - logic [wh_cord_width_mp-1:0] dest_cord; \ - } bsg_manycore_vcache_wh_header_flit_s - `endif diff --git a/v/bsg_manycore_tile_vcache.v b/v/bsg_manycore_tile_vcache.v index f70ac9e9c..8dcfa1336 100644 --- a/v/bsg_manycore_tile_vcache.v +++ b/v/bsg_manycore_tile_vcache.v @@ -254,10 +254,8 @@ module bsg_manycore_tile_vcache wh_link_sif_s cache_wh_link_lo; bsg_cache_dma_to_wormhole #( - .vcache_addr_width_p(vcache_addr_width_p) - ,.vcache_data_width_p(vcache_data_width_p) - ,.vcache_dma_data_width_p(vcache_dma_data_width_p) - ,.vcache_block_size_in_words_p(vcache_block_size_in_words_p) + .dma_addr_width_p(vcache_addr_width_p) + ,.dma_burst_len_p(vcache_block_size_in_words_p*vcache_data_width_p/vcache_dma_data_width_p) ,.wh_flit_width_p(wh_flit_width_p) ,.wh_cid_width_p(wh_cid_width_p) @@ -273,7 +271,7 @@ module bsg_manycore_tile_vcache ,.dma_data_o(dma_data_li) ,.dma_data_v_o(dma_data_v_li) - ,.dma_data_ready_i(dma_data_ready_lo) + ,.dma_data_ready_and_i(dma_data_ready_lo) ,.dma_data_i(dma_data_lo) ,.dma_data_v_i(dma_data_v_lo) @@ -284,6 +282,7 @@ module bsg_manycore_tile_vcache ,.my_wh_cord_i(global_x_r) ,.dest_wh_cord_i({wh_cord_width_p{wh_dest_east_not_west_lo}}) + ,.dest_wh_cid_i('0) // concentrator id // lower bits come from lower bits of global_x // upper bits come from whether its north or south vc.