31static inline uint32_t snrt_dma_start_1d(uint64_t dst, uint64_t src,
33 const uint32_t channel = 0) {
34 uint32_t dst_lo = dst & 0xFFFFFFFF;
35 uint32_t dst_hi = dst >> 32;
36 uint32_t src_lo = src & 0xFFFFFFFF;
37 uint32_t src_hi = src >> 32;
41 "dmsrc %[src_lo], %[src_hi] \n"
42 "dmdst %[dst_lo], %[dst_hi] \n"
43 "dmcpyi %[txid], %[size], (%[channel] << 2) | 0b00 \n"
45 : [ src_lo ]
"r"(src_lo), [ src_hi ]
"r"(src_hi),
46 [ dst_lo ]
"r"(dst_lo), [ dst_hi ]
"r"(dst_hi), [ size ]
"r"(size),
47 [ channel ]
"i"(channel));
59static inline uint32_t snrt_dma_start_1d(
volatile void *dst,
volatile void *src,
61 const uint32_t channel = 0) {
62 return snrt_dma_start_1d((uint64_t)dst, (uint64_t)src, size, channel);
70 asm volatile(
"dmuser %[mask], zero \n" : : [ mask ]
"r"(mask));
86static inline uint32_t snrt_dma_start_1d_mcast(uint64_t dst, uint64_t src,
87 size_t size, uint32_t mask,
88 const uint32_t channel = 0) {
90 uint32_t txid = snrt_dma_start_1d(dst, src, size, channel);
104static inline uint32_t snrt_dma_start_1d_mcast(
volatile void *dst,
105 volatile void *src,
size_t size,
107 const uint32_t channel = 0) {
108 return snrt_dma_start_1d_mcast((uint64_t)dst, (uint64_t)src, size, mask,
130static inline snrt_dma_txid_t snrt_dma_start_2d(uint64_t dst, uint64_t src,
131 size_t size,
size_t dst_stride,
134 const uint32_t channel = 0) {
135 uint32_t dst_lo = dst & 0xFFFFFFFF;
136 uint32_t dst_hi = dst >> 32;
137 uint32_t src_lo = src & 0xFFFFFFFF;
138 uint32_t src_hi = src >> 32;
142 "dmsrc %[src_lo], %[src_hi] \n"
143 "dmdst %[dst_lo], %[dst_hi] \n"
144 "dmstr %[src_stride], %[dst_stride] \n"
146 "dmcpyi %[txid], %[size], (%[channel] << 2) | 0b10 \n"
147 : [ txid ]
"=r"(txid)
148 : [ src_lo ]
"r"(src_lo), [ src_hi ]
"r"(src_hi),
149 [ dst_lo ]
"r"(dst_lo), [ dst_hi ]
"r"(dst_hi),
150 [ dst_stride ]
"r"(dst_stride), [ src_stride ]
"r"(src_stride),
151 [ repeat ]
"r"(repeat), [ size ]
"r"(size), [ channel ]
"i"(channel));
164static inline uint32_t snrt_dma_start_2d(
volatile void *dst,
volatile void *src,
165 size_t size,
size_t dst_stride,
166 size_t src_stride,
size_t repeat,
167 const uint32_t channel = 0) {
168 return snrt_dma_start_2d((uint64_t)dst, (uint64_t)src, size, dst_stride,
169 src_stride, repeat, channel);
183 const uint32_t channel = 0) {
186 "dmstati t0, (%[channel] << 2) | 0 \n"
187 "bltu t0, %[txid], 1b \n"
189 : [ txid ]
"r"(txid), [ channel ]
"i"(channel)
202static inline void snrt_dma_wait_all(
const uint32_t channel = 0) {
206 "dmstati %[busy], (%[channel] << 2) | 2 \n"
207 "bne %[busy], zero, 1b \n"
208 : [ busy ]
"=r"(busy)
209 : [ channel ]
"i"(channel));
217 for (
int c = 0; c < num_channels; c++) {
218 snrt_dma_wait_all(c);
249 size_t n_1d_transfers = len / 64;
250 size_t use_dma = (len % 64) == 0 && len > 64;
251 uint8_t *p = (uint8_t *)ptr;
253 uint32_t nbytes = len < 64 || !use_dma ? len : 64;
259 snrt_dma_start_2d(ptr, ptr, 64, 64, 0, n_1d_transfers);
274 size_t tile_idx,
size_t tile_size,
276 size_t tile_nbytes = tile_size * prec;
277 return snrt_dma_start_1d(
278 (uint64_t)dst, (uint64_t)src + tile_idx * tile_nbytes, tile_nbytes);
295 size_t tile_nbytes = tile_size * prec;
296 return snrt_dma_start_1d_mcast((uintptr_t)dst,
297 (uintptr_t)src + tile_idx * tile_nbytes,
310 size_t size,
size_t row_size,
312 return snrt_dma_start_2d(dst, src, row_size, stride, row_size,
325 size_t size,
size_t row_size,
327 return snrt_dma_start_2d(dst, src, row_size, row_size, stride,
340 size_t tile_idx,
size_t tile_size,
342 size_t tile_nbytes = tile_size * prec;
343 return snrt_dma_start_1d((uint64_t)dst + tile_idx * tile_nbytes,
344 (uint64_t)src, tile_nbytes);
363 void *dst,
void *src,
size_t tile_x1_idx,
size_t tile_x0_idx,
364 size_t tile_x1_size,
size_t tile_x0_size,
size_t full_x0_size,
365 uint32_t prec,
size_t tile_ld) {
366 size_t src_offset = 0;
368 src_offset += tile_x0_idx * tile_x0_size;
369 src_offset += tile_x1_idx * tile_x1_size * full_x0_size;
372 return snrt_dma_start_2d((uint64_t)dst,
373 (uint64_t)src + src_offset,
382 void *dst,
void *src,
size_t tile_x1_idx,
size_t tile_x0_idx,
383 size_t tile_x1_size,
size_t tile_x0_size,
size_t full_x0_size,
386 tile_x1_size, tile_x0_size, full_x0_size, prec,
387 tile_x0_size * prec);
407 void *dst,
void *src,
size_t tile_x1_idx,
size_t tile_x0_idx,
408 size_t tile_x1_size,
size_t tile_x0_size,
size_t full_x0_size,
409 uint32_t prec,
size_t num_banks) {
411 size_t tile_x0_size_in_banks = (num_banks * SNRT_TCDM_BANK_WIDTH) / prec;
412 size_t tile_x1_size_in_banks =
413 ceil((tile_x1_size * tile_x0_size) / (
double)tile_x0_size_in_banks);
414 size_t tile_ld = SNRT_TCDM_HYPERBANK_WIDTH;
416 tile_x1_size_in_banks, tile_x0_size_in_banks,
417 full_x0_size, prec, tile_ld);
436 void *dst,
void *src,
size_t tile_x1_idx,
size_t tile_x0_idx,
437 size_t tile_x1_size,
size_t tile_x0_size,
size_t full_x0_size,
438 uint32_t prec,
size_t tile_ld) {
439 size_t dst_offset = 0;
441 dst_offset += tile_x0_idx * tile_x0_size;
442 dst_offset += tile_x1_idx * tile_x1_size * full_x0_size;
445 return snrt_dma_start_2d((uint64_t)dst + dst_offset,
455 void *dst,
void *src,
size_t tile_x1_idx,
size_t tile_x0_idx,
456 size_t tile_x1_size,
size_t tile_x0_size,
size_t full_x0_size,
459 tile_x1_size, tile_x0_size, full_x0_size,
460 prec, tile_x0_size * prec);
464 void *dst,
void *src,
size_t tile_x1_idx,
size_t tile_x0_idx,
465 size_t tile_x1_size,
size_t tile_x0_size,
size_t full_x0_size,
466 uint32_t prec,
size_t num_banks) {
468 size_t tile_x0_size_in_banks = (num_banks * SNRT_TCDM_BANK_WIDTH) / prec;
469 size_t tile_x1_size_in_banks =
470 ceil((tile_x1_size * tile_x0_size) / (
double)tile_x0_size_in_banks);
471 size_t tile_ld = SNRT_TCDM_HYPERBANK_WIDTH;
473 tile_x1_size_in_banks, tile_x0_size_in_banks,
474 full_x0_size, prec, tile_ld);
void snrt_dma_enable_mcast(uint32_t mask)
Enable multicast for successive transfers.
Definition dma.h:69
snrt_dma_txid_t snrt_dma_load_1d_tile(volatile void *dst, volatile void *src, size_t tile_idx, size_t tile_size, uint32_t prec)
Load a tile of a 1D array.
Definition dma.h:272
snrt_dma_txid_t snrt_dma_1d_to_2d(volatile void *dst, volatile void *src, size_t size, size_t row_size, size_t stride)
Transfer and reshape a 1D array into a 2D array.
Definition dma.h:309
snrt_dma_txid_t snrt_dma_mcast_load_1d_tile(void *dst, void *src, size_t tile_idx, size_t tile_size, uint32_t prec, uint32_t mcast)
Load a tile of a 1D array.
Definition dma.h:290
uint32_t snrt_dma_txid_t
A DMA transfer identifier.
Definition dma.h:15
snrt_dma_txid_t snrt_dma_store_2d_tile(void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld)
Store a 2D tile to a 2D array.
Definition dma.h:435
snrt_dma_txid_t snrt_dma_load_2d_tile_in_banks(void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t num_banks)
Load a 2D tile of a 2D array and reshape it to occupy a subset of TCDM banks.
Definition dma.h:406
snrt_dma_txid_t snrt_dma_store_1d_tile(void *dst, void *src, size_t tile_idx, size_t tile_size, uint32_t prec)
Store a tile to a 1D array.
Definition dma.h:339
snrt_dma_txid_t snrt_dma_2d_to_1d(volatile void *dst, volatile void *src, size_t size, size_t row_size, size_t stride)
Transfer and reshape a 2D array into a 1D array.
Definition dma.h:324
void snrt_dma_wait_all_channels(uint32_t num_channels)
Block until the first num_channels channels are idle.
Definition dma.h:216
void snrt_dma_memset(void *ptr, uint8_t value, uint32_t len)
Fast memset function performed by DMA.
Definition dma.h:245
snrt_dma_txid_t snrt_dma_load_2d_tile(void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld)
Load a 2D tile of a 2D array.
Definition dma.h:362
void snrt_dma_start_tracking()
Start tracking of dma performance region. Does not have any implications on the HW....
Definition dma.h:228
void snrt_dma_disable_mcast()
Disable multicast for successive transfers.
Definition dma.h:77
void snrt_dma_stop_tracking()
Stop tracking of dma performance region. Does not have any implications on the HW....
Definition dma.h:236