dma.h File Reference

This file provides functions to program the Snitch DMA. More...

#include <string.h>
#include <math.h>
#include "idma_compute.h"

Go to the source code of this file.

Functions
uint32_t	snrt_dma_start_1d (uint64_t dst, uint64_t src, size_t size, uint32_t channel)
	Start an asynchronous 1D DMA transfer with 64-bit wide pointers on a specific DMA channel.
uint32_t	snrt_dma_start_1d (volatile void *dst, volatile void *src, size_t size, uint32_t channel=0)
	Start an asynchronous 1D DMA transfer using native-size pointers.
void	snrt_dma_set_awuser (uint64_t field)
	Set AW user field of the DMA's AXI interface.
void	snrt_dma_enable_multicast (uint64_t mask)
	Enable multicast for successive transfers.
void	snrt_dma_enable_reduction (uint64_t mask, snrt_collective_opcode_t opcode)
	Enable reduction operations for successive transfers.
void	snrt_dma_disable_multicast ()
	Disable multicast for successive transfers.
void	snrt_dma_disable_reduction ()
	Disable reduction operations for successive transfers.
uint32_t	snrt_dma_start_1d_reduction (uint64_t dst, uint64_t src, size_t size, uint64_t mask, snrt_collective_opcode_t opcode, uint32_t channel=0)
	Start an asynchronous reduction 1D DMA transfer with 64-bit wide pointers.
uint32_t	snrt_dma_start_1d_reduction (uint64_t dst, uint64_t src, size_t size, snrt_comm_t comm, snrt_collective_opcode_t opcode, uint32_t channel=0)
	Start an asynchronous reduction 1D DMA transfer with 64-bit wide pointers.
uint32_t	snrt_dma_start_1d_mcast (uint64_t dst, uint64_t src, size_t size, uint64_t mask, uint32_t channel=0)
	Start an asynchronous multicast 1D DMA transfer with 64-bit wide pointers.
uint32_t	snrt_dma_start_1d_mcast (uint64_t dst, uint64_t src, size_t size, snrt_comm_t comm, uint32_t channel=0)
	Start an asynchronous multicast 1D DMA transfer with 64-bit wide pointers.
uint32_t	snrt_dma_start_1d_reduction (volatile void *dst, volatile void *src, size_t size, uint64_t mask, snrt_collective_opcode_t opcode, uint32_t channel=0)
	Start an asynchronous reduction 1D DMA transfer using native-size pointers.
uint32_t	snrt_dma_start_1d_mcast (volatile void *dst, volatile void *src, size_t size, uint64_t mask, uint32_t channel=0)
	Start an asynchronous multicast 1D DMA transfer using native-size pointers.
snrt_dma_txid_t	snrt_dma_start_2d (uint64_t dst, uint64_t src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t channel)
	Start an asynchronous 2D DMA transfer with 64-bit wide pointers.
uint32_t	snrt_dma_start_2d (volatile void *dst, volatile void *src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t channel=0)
	Start an asynchronous 2D DMA transfer using native-size pointers.
uint32_t	snrt_dma_start_2d_mcast (uint64_t dst, uint64_t src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t mask, uint32_t channel=0)
	Start an asynchronous, multicast 2D DMA transfer with 64-bit wide pointers.
uint32_t	snrt_dma_start_2d_mcast (volatile void *dst, volatile void *src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t mask, uint32_t channel=0)
	Start an asynchronous, multicast 2D DMA transfer using native-size pointers.
static uint32_t	snrt_dma_busy (const uint32_t channel)
	Read DMA busy flag.
static uint32_t	snrt_dma_would_block (const uint32_t channel)
	Read DMA would_block flag.
static void	snrt_dma_wait (snrt_dma_txid_t txid, const uint32_t channel=0)
	Block until a DMA transfer finishes on a specific DMA channel.
static void	snrt_dma_wait_all (const uint32_t channel=0)
	Block until a specific DMA channel is idle.
void	snrt_dma_wait_all_channels (uint32_t num_channels)
	Block until the first num_channels channels are idle.
void	snrt_dma_start_tracking ()
	Start tracking of dma performance region. Does not have any implications on the HW. Only injects a marker in the DMA traces that can be analyzed.
void	snrt_dma_stop_tracking ()
	Stop tracking of dma performance region. Does not have any implications on the HW. Only injects a marker in the DMA traces that can be analyzed.
void	snrt_dma_memset (void *ptr, uint8_t value, uint32_t len)
	Fast memset function performed by DMA.
snrt_dma_txid_t	snrt_dma_load_1d_tile (volatile void *dst, volatile void *src, size_t tile_idx, size_t tile_size, uint32_t prec)
	Load a tile of a 1D array.
snrt_dma_txid_t	snrt_dma_load_1d_tile_mcast (void *dst, void *src, size_t tile_idx, size_t tile_size, uint32_t prec, uint64_t mask)
	Load a tile of a 1D array.
snrt_dma_txid_t	snrt_dma_reduction_load_1d_tile (void *dst, void *src, size_t tile_idx, size_t tile_size, uint32_t prec, uint64_t mask, snrt_collective_opcode_t opcode)
	Load a tile of a 1D array.
snrt_dma_txid_t	snrt_dma_1d_to_2d (volatile void *dst, volatile void *src, size_t size, size_t row_size, size_t stride)
	Transfer and reshape a 1D array into a 2D array.
snrt_dma_txid_t	snrt_dma_2d_to_1d (volatile void *dst, volatile void *src, size_t size, size_t row_size, size_t stride)
	Transfer and reshape a 2D array into a 1D array.
snrt_dma_txid_t	snrt_dma_store_1d_tile (void *dst, void *src, size_t tile_idx, size_t tile_size, uint32_t prec)
	Store a tile to a 1D array.
snrt_dma_txid_t	snrt_dma_load_2d_tile (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld)
	Load a 2D tile of a 2D array.
snrt_dma_txid_t	snrt_dma_load_2d_tile (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec)
	Load a 2D tile of a 2D array.
snrt_dma_txid_t	snrt_dma_load_2d_tile_mcast (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld, uint32_t mask)
	Load a 2D tile of a 2D array using multicast.
snrt_dma_txid_t	snrt_dma_load_2d_tile_mcast (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, uint32_t mask)
	Load a 2D tile of a 2D array.
snrt_dma_txid_t	snrt_dma_load_2d_tile_mcast (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, snrt_comm_t comm)
	Load a 2D tile of a 2D array using multicast.
snrt_dma_txid_t	snrt_dma_load_2d_tile_in_banks (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t num_banks)
	Load a 2D tile of a 2D array and reshape it to occupy a subset of TCDM banks.
snrt_dma_txid_t	snrt_dma_store_2d_tile (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld)
	Store a 2D tile to a 2D array.
snrt_dma_txid_t	snrt_dma_store_2d_tile (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec)
	Store a 2D tile of a 2D array.
snrt_dma_txid_t	snrt_dma_store_2d_tile_from_banks (void *dst, void *src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t num_banks)
	Store a 2D tile of a 2D array from a 1D layout occupying a subset of TCDM banks.
void	snrt_dma_set_opcode_params (uint32_t opcode, uint32_t params)
	Set the on-the-fly compute configuration for subsequent transfers.
void	snrt_dma_set_opcode (uint32_t opcode)
	Set a parameterless on-the-fly compute op for subsequent transfers.
void	snrt_dma_enable_transpose (uint32_t mode, uint32_t tensor_m, uint32_t tensor_n)
	Enable the tiled transpose of a row-major tensor for successive transfers.
void	snrt_dma_disable_compute ()
	Disable on-the-fly compute for successive transfers.
uint32_t	snrt_dma_start_transpose (uint64_t dst, uint64_t src, size_t size, uint32_t mode, uint32_t tensor_m, uint32_t tensor_n, uint32_t channel=0)
	Start an asynchronous transposing DMA transfer.
uint32_t	snrt_dma_start_transpose (volatile void *dst, volatile void *src, size_t size, uint32_t mode, uint32_t tensor_m, uint32_t tensor_n, uint32_t channel=0)
	Start an asynchronous transposing DMA transfer.

Detailed Description

This file provides functions to program the Snitch DMA.

Function Documentation

snrt_dma_1d_to_2d()

snrt_dma_txid_t snrt_dma_1d_to_2d ( volatile void * dst, volatile void * src, size_t size, size_t row_size, size_t stride )

inline

Transfer and reshape a 1D array into a 2D array.

Parameters

dst	Pointer to the destination array.
src	Pointer to the source array.
size	Number of bytes to transfer.
row_size	Size of a row in the 2D array, in bytes.
stride	Stride between successive rows in the 2D array, in bytes.

                                                        {
    return snrt_dma_start_2d(dst, src, row_size, stride, row_size,
                             size / row_size);
}

snrt_dma_2d_to_1d()

snrt_dma_txid_t snrt_dma_2d_to_1d ( volatile void * dst, volatile void * src, size_t size, size_t row_size, size_t stride )

inline

Transfer and reshape a 2D array into a 1D array.

Parameters

dst	Pointer to the destination array.
src	Pointer to the source array.
size	Number of bytes to transfer.
row_size	Size of a row in the 2D array, in bytes.
stride	Stride between successive rows in the 2D array, in bytes.

                                                        {
    return snrt_dma_start_2d(dst, src, row_size, row_size, stride,
                             size / row_size);
}

snrt_dma_busy()

static uint32_t snrt_dma_busy ( const uint32_t channel )

inlinestatic

Read DMA busy flag.

Parameters

channel

The index of the channel.

Note

The function passes the channel argument as an immediate, thus this must be known at compile time. As a consequence, the function must use internal linkage (static keyword) and must be always inlined. This is true also for all functions invoking this function, and passing down an argument to channel.

                                                             {
#ifdef SNRT_SUPPORTS_DMA
    uint32_t busy;
    asm volatile("dmstati %[busy], (%[channel] << 2) | 2 \n"
                 : [ busy ] "=r"(busy)
                 : [ channel ] "i"(channel)
                 :);
    return busy;
#else
    return 0;
#endif
}

snrt_dma_disable_compute()

void snrt_dma_disable_compute ( )

inline

Disable on-the-fly compute for successive transfers.

Successive DMA transfers will be plain copies

                                       {
    snrt_dma_set_opcode(IDMA_DMOPC_OPC_PASSTHROUGH);
}

snrt_dma_disable_multicast()

void snrt_dma_disable_multicast ( )

inline

Disable multicast for successive transfers.

Successive DMA transfers will be unicast transfers

{ snrt_dma_set_awuser(0); }

snrt_dma_disable_reduction()

void snrt_dma_disable_reduction ( )

inline

Disable reduction operations for successive transfers.

Successive DMA transfers will be unicast transfers

{ snrt_dma_set_awuser(0); }

snrt_dma_enable_multicast()

void snrt_dma_enable_multicast ( uint64_t mask )

inline

Enable multicast for successive transfers.

All transfers performed after this call will be multicast to all addresses specified by the address and mask pair.

Parameters

mask	Multicast mask value

                                                     {
    snrt_collective_t op;
    op.f.opcode = SNRT_COLLECTIVE_MULTICAST;
    op.f.mask = mask;
    snrt_dma_set_awuser(op.w);
}

snrt_dma_enable_reduction()

void snrt_dma_enable_reduction ( uint64_t mask, snrt_collective_opcode_t opcode )

inline

Enable reduction operations for successive transfers.

All transfers performed after this call will be part of a reduction involving all masters identified by the mask.

Parameters

mask	Mask defines all involved members
opcode	Type of reduction operation

                                                                       {
    snrt_collective_t op;
    op.f.opcode = opcode;
    op.f.mask = mask;
    snrt_dma_set_awuser(op.w);
}

snrt_dma_enable_transpose()

void snrt_dma_enable_transpose ( uint32_t mode, uint32_t tensor_m, uint32_t tensor_n )

inline

Enable the tiled transpose of a row-major tensor for successive transfers.

Parameters

mode	Element size selector; elements are 1 << mode bytes.
tensor_m	Rows of the source tensor, in elements.
tensor_n	Columns of the source tensor, in elements.

                                                         {
    snrt_dma_set_opcode_params(
        IDMA_DMOPC_OPC_TRANSPOSE | ((mode & IDMA_DMOPC_RS1_TP_MODE_MASK)
                                    << IDMA_DMOPC_RS1_TP_MODE_SHIFT),
        ((tensor_m & IDMA_DMOPC_RS2_TP_TENSOR_M_MASK)
         << IDMA_DMOPC_RS2_TP_TENSOR_M_SHIFT) |
            ((tensor_n & IDMA_DMOPC_RS2_TP_TENSOR_N_MASK)
             << IDMA_DMOPC_RS2_TP_TENSOR_N_SHIFT));
}

snrt_dma_load_1d_tile()

snrt_dma_txid_t snrt_dma_load_1d_tile ( volatile void * dst, volatile void * src, size_t tile_idx, size_t tile_size, uint32_t prec )

inline

Load a tile of a 1D array.

Parameters

dst	Pointer to the tile destination.
src	Pointer to the source array.
tile_idx	Index of the tile in the 1D array.
tile_size	Number of elements within a tile of the 1D array.
prec	Number of bytes of each element in the 1D array.

                                                            {
    size_t tile_nbytes = tile_size * prec;
    return snrt_dma_start_1d(
        (uint64_t)dst, (uint64_t)src + tile_idx * tile_nbytes, tile_nbytes);
}

snrt_dma_load_1d_tile_mcast()

snrt_dma_txid_t snrt_dma_load_1d_tile_mcast ( void * dst, void * src, size_t tile_idx, size_t tile_size, uint32_t prec, uint64_t mask )

inline

Load a tile of a 1D array.

Parameters

dst	Pointer to the tile destination.
src	Pointer to the source array.
tile_idx	Index of the tile in the 1D array.
tile_size	Number of elements within a tile of the 1D array.
prec	Number of bytes of each element in the 1D array.
mask	Multicast mask applied on the destination address.

                                                                  {
    size_t tile_nbytes = tile_size * prec;
    return snrt_dma_start_1d_mcast((uintptr_t)dst,
                                   (uintptr_t)src + tile_idx * tile_nbytes,
                                   tile_nbytes, mask);
}

snrt_dma_load_2d_tile() [1/2]

snrt_dma_txid_t snrt_dma_load_2d_tile ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec )

inline

Load a 2D tile of a 2D array.

The stride in the destination tile is assumed to be that of a 1D tile, effectively. In other words, this is the same as snrt_dma_2d_to_1d().

See also

snrt_dma_load_2d_tile(void *, void *, size_t, size_t, size_t, size_t, size_t, uint32_t, size_t) for a detailed description of the parameters.

                   {
    return snrt_dma_load_2d_tile(dst, src, tile_x1_idx, tile_x0_idx,
                                 tile_x1_size, tile_x0_size, full_x0_size, prec,
                                 tile_x0_size * prec);
}

snrt_dma_load_2d_tile() [2/2]

snrt_dma_txid_t snrt_dma_load_2d_tile ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld )

inline

Load a 2D tile of a 2D array.

Parameters

dst	Pointer to the tile destination.
src	Pointer to the source array.
tile_x1_idx	Outermost coordinate of the tile in the 2D array.
tile_x0_idx	Innermost coordinate of the tile in the 2D array.
tile_x1_size	Number of elements in the outermost dimension of the tile.
tile_x0_size	Number of elements in the innermost dimension of the tile.
full_x0_size	Number of elements in the innermost dimension of the array.
prec	Number of bytes of each element in the 2D array.
tile_ld	Leading dimension of the tile, in bytes.

                                   {
    size_t src_offset = 0;
    // Advance src array in x0 and x1 dimensions, and convert to byte offset
    src_offset += tile_x0_idx * tile_x0_size;
    src_offset += tile_x1_idx * tile_x1_size * full_x0_size;
    src_offset *= prec;
    // Initiate transfer
    return snrt_dma_start_2d((uint64_t)dst,               // dst
                             (uint64_t)src + src_offset,  // src
                             tile_x0_size * prec,         // size
                             tile_ld,                     // dst_stride
                             full_x0_size * prec,         // src_stride
                             tile_x1_size                 // repeat
    );
}

snrt_dma_load_2d_tile_in_banks()

snrt_dma_txid_t snrt_dma_load_2d_tile_in_banks ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t num_banks )

inline

Load a 2D tile of a 2D array and reshape it to occupy a subset of TCDM banks.

Parameters

dst	Pointer to the tile destination.
src	Pointer to the source array.
tile_x1_idx	Outermost coordinate of the tile in the 2D array.
tile_x0_idx	Innermost coordinate of the tile in the 2D array.
tile_x1_size	Number of elements in the outermost dimension of the tile.
tile_x0_size	Number of elements in the innermost dimension of the tile.
full_x0_size	Number of elements in the innermost dimension of the array.
prec	Number of bytes of each element in the 2D array.
num_banks	Number of banks to reshape the tile into.

                                     {
    // Calculate new tile size after reshaping the tile in the selected banks
    size_t tile_x0_size_in_banks = (num_banks * SNRT_TCDM_BANK_WIDTH) / prec;
    size_t tile_x1_size_in_banks =
        ceil((tile_x1_size * tile_x0_size) / (double)tile_x0_size_in_banks);
    size_t tile_ld = SNRT_TCDM_HYPERBANK_WIDTH;
    return snrt_dma_load_2d_tile(dst, src, tile_x1_idx, tile_x0_idx,
                                 tile_x1_size_in_banks, tile_x0_size_in_banks,
                                 full_x0_size, prec, tile_ld);
}

snrt_dma_load_2d_tile_mcast() [1/3]

snrt_dma_txid_t snrt_dma_load_2d_tile_mcast ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld, uint32_t mask )

inline

Load a 2D tile of a 2D array using multicast.

Parameters

mask	Multicast mask.

See also

snrt_dma_load_2d_tile(void *, void *, size_t, size_t, size_t, size_t, size_t, uint32_t, size_t) for a description of the other parameters.

                                                  {
    size_t src_offset = 0;
    // Advance src array in x0 and x1 dimensions, and convert to byte offset
    src_offset += tile_x0_idx * tile_x0_size;
    src_offset += tile_x1_idx * tile_x1_size * full_x0_size;
    src_offset *= prec;
    // Initiate transfer
    return snrt_dma_start_2d_mcast((uint64_t)dst,               // dst
                                   (uint64_t)src + src_offset,  // src
                                   tile_x0_size * prec,         // size
                                   tile_ld,                     // dst_stride
                                   full_x0_size * prec,         // src_stride
                                   tile_x1_size,                // repeat
                                   mask                         // mask
    );
}

snrt_dma_load_2d_tile_mcast() [2/3]

snrt_dma_txid_t snrt_dma_load_2d_tile_mcast ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, snrt_comm_t comm )

inline

Load a 2D tile of a 2D array using multicast.

Parameters

comm	Communicator specifying which clusters to multicast to.

The stride in the destination tile is assumed to be that of a 1D tile, effectively. In other words, this is similar to snrt_dma_2d_to_1d().

See also

snrt_dma_load_2d_tile_mcast(void *, void *, size_t, size_t, size_t, size_t, size_t, uint32_t, size_t, uint32_t) for a detailed description of the parameters.

                                     {
    uint64_t mask = snrt_get_collective_mask(comm);
    return snrt_dma_load_2d_tile_mcast(dst, src, tile_x1_idx, tile_x0_idx,
                                       tile_x1_size, tile_x0_size, full_x0_size,
                                       prec, mask);
}

snrt_dma_load_2d_tile_mcast() [3/3]

snrt_dma_txid_t snrt_dma_load_2d_tile_mcast ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, uint32_t mask )

inline

Load a 2D tile of a 2D array.

The stride in the destination tile is assumed to be that of a 1D tile, effectively. In other words, this is similar to snrt_dma_2d_to_1d().

See also

snrt_dma_load_2d_tile_mcast(void *, void *, size_t, size_t, size_t, size_t, size_t, uint32_t, size_t, uint32_t) for a detailed description of the parameters.

                                  {
    return snrt_dma_load_2d_tile_mcast(dst, src, tile_x1_idx, tile_x0_idx,
                                       tile_x1_size, tile_x0_size, full_x0_size,
                                       prec, tile_x0_size * prec, mask);
}

snrt_dma_memset()

void snrt_dma_memset ( void * ptr, uint8_t value, uint32_t len )

inline

Fast memset function performed by DMA.

Parameters

ptr	Pointer to the start of the region.
value	Value to set.
len	Number of bytes, must be a multiple of the DMA bus width to use the DMA.

                                                                    {
#ifdef SNRT_SUPPORTS_DMA
    // We set the first 64 bytes to the value, and then we use the DMA to copy
    // these into the remaining memory region. DMA is used only if len is
    // larger than 64 bytes, and an integer multiple of 64 bytes.
    size_t n_1d_transfers = len / 64;
    size_t use_dma = (len % 64) == 0 && len > 64;
    uint8_t *p = (uint8_t *)ptr;
 
    uint32_t nbytes = len < 64 || !use_dma ? len : 64;
    while (nbytes--) {
        *p++ = value;
    }
 
    if (use_dma) {
        snrt_dma_start_2d(ptr, ptr, 64, 64, 0, n_1d_transfers);
        snrt_dma_wait_all();
    }
#else
    memset(ptr, (int)value, len);
#endif
}

snrt_dma_reduction_load_1d_tile()

snrt_dma_txid_t snrt_dma_reduction_load_1d_tile ( void * dst, void * src, size_t tile_idx, size_t tile_size, uint32_t prec, uint64_t mask, snrt_collective_opcode_t opcode )

inline

Load a tile of a 1D array.

Parameters

dst	Pointer to the tile destination.
src	Pointer to the source array.
tile_idx	Index of the tile in the 1D array.
tile_size	Number of elements within a tile of the 1D array.
prec	Number of bytes of each element in the 1D array.
mask	Mask for reduction operation.
opcode	Reduction operation.

                                                    {
    size_t tile_nbytes = tile_size * prec;
    return snrt_dma_start_1d_reduction((uintptr_t)dst,
                                       (uintptr_t)src + tile_idx * tile_nbytes,
                                       tile_nbytes, mask, opcode);
}

snrt_dma_set_awuser()

void snrt_dma_set_awuser ( uint64_t field )

inline

Set AW user field of the DMA's AXI interface.

All DMA transfers performed after this call are equipped with the given AW user field

Parameters

field

Defines the AW user field for the AXI transfer

                                                {
#ifdef SNRT_SUPPORTS_DMA
    uint32_t user_low = (uint32_t)(field);
    uint32_t user_high = (uint32_t)(field >> 32);
    asm volatile("dmuser %[user_low], %[user_high] \n"
                 :
                 : [ user_low ] "r"(user_low), [ user_high ] "r"(user_high));
#endif
}

snrt_dma_set_opcode()

void snrt_dma_set_opcode ( uint32_t opcode )

inline

Set a parameterless on-the-fly compute op for subsequent transfers.

Parameters

opcode

Opcode byte; one of the IDMA_DMOPC_OPC_* defines.

                                                 {
    snrt_dma_set_opcode_params(opcode, 0u);
}

snrt_dma_set_opcode_params()

void snrt_dma_set_opcode_params ( uint32_t opcode, uint32_t params )

inline

Set the on-the-fly compute configuration for subsequent transfers.

Parameters

opcode	Opcode word for rs1: an IDMA_DMOPC_OPC_* byte, plus the transpose element-size mode at IDMA_DMOPC_RS1_TP_MODE_SHIFT.
params	Op-parameter word for rs2; 0 for ops that take none.

Latched at the DMOPC handshake and applied until the next DMOPC.

                                                                         {
#ifdef SNRT_SUPPORTS_DMA_COMPUTE
    asm volatile("dmopc %[opcode], %[params] \n"
                 :
                 : [ opcode ] "r"(opcode), [ params ] "r"(params)
                 : "memory");
#endif
}

snrt_dma_start_1d() [1/2]

uint32_t snrt_dma_start_1d ( uint64_t dst, uint64_t src, size_t size, uint32_t channel )

inline

Start an asynchronous 1D DMA transfer with 64-bit wide pointers on a specific DMA channel.

Parameters

dst	The destination address.
src	The source address.
size	The size of the transfer in bytes.
channel	The index of the channel.

Returns

The DMA transfer ID.

                                                    {
#ifdef SNRT_SUPPORTS_DMA
    uint32_t dst_lo = dst & 0xFFFFFFFF;
    uint32_t dst_hi = dst >> 32;
    uint32_t src_lo = src & 0xFFFFFFFF;
    uint32_t src_hi = src >> 32;
    uint32_t cfg = (channel << 2) | 0b00;
    uint32_t txid;
 
    asm volatile(
        "dmsrc %[src_lo], %[src_hi] \n"
        "dmdst %[dst_lo], %[dst_hi] \n"
        "dmcpy %[txid], %[size], %[cfg] \n"
        : [ txid ] "=r"(txid)
        : [ src_lo ] "r"(src_lo), [ src_hi ] "r"(src_hi),
          [ dst_lo ] "r"(dst_lo), [ dst_hi ] "r"(dst_hi), [ size ] "r"(size),
          [ cfg ] "r"(cfg));
    return txid;
#else
    memcpy((void *)dst, (const void *)src, size);
    return 0;
#endif
}

snrt_dma_start_1d() [2/2]

uint32_t snrt_dma_start_1d ( volatile void * dst, volatile void * src, size_t size, uint32_t channel = 0 )

inline

Start an asynchronous 1D DMA transfer using native-size pointers.

This is a convenience overload of snrt_dma_start_1d(uint64_t, uint64_t, size_t, uint32_t) using void* pointers.

                                                                     {
    return snrt_dma_start_1d((uint64_t)dst, (uint64_t)src, size, channel);
}

snrt_dma_start_1d_mcast() [1/3]

uint32_t snrt_dma_start_1d_mcast ( uint64_t dst, uint64_t src, size_t size, snrt_comm_t comm, uint32_t channel = 0 )

inline

Start an asynchronous multicast 1D DMA transfer with 64-bit wide pointers.

Parameters

comm	The communicator for the multicast operation

See also

snrt_dma_start_1d(uint64_t, uint64_t, size_t, uint32_t) for a description of the other parameters.

                                                              {
    uint64_t mask = snrt_get_collective_mask(comm);
    uint32_t txid = snrt_dma_start_1d_mcast(dst, src, size, mask, channel);
    return txid;
}

snrt_dma_start_1d_mcast() [2/3]

uint32_t snrt_dma_start_1d_mcast ( uint64_t dst, uint64_t src, size_t size, uint64_t mask, uint32_t channel = 0 )

inline

Start an asynchronous multicast 1D DMA transfer with 64-bit wide pointers.

Parameters

mask	The mask for the multicast operation

See also

snrt_dma_start_1d(uint64_t, uint64_t, size_t, uint32_t) for a description of the other parameters.

                                                                             {
    snrt_dma_enable_multicast(mask);
    uint32_t txid = snrt_dma_start_1d(dst, src, size, channel);
    snrt_dma_disable_multicast();
    return txid;
}

snrt_dma_start_1d_mcast() [3/3]

uint32_t snrt_dma_start_1d_mcast ( volatile void * dst, volatile void * src, size_t size, uint64_t mask, uint32_t channel = 0 )

inline

Start an asynchronous multicast 1D DMA transfer using native-size pointers.

This is a convenience overload of snrt_dma_start_1d_mcast(uint64_t, uint64_t, size_t, uint64_t, uint32_t) using void* pointers.

                                                              {
    return snrt_dma_start_1d_mcast((uint64_t)dst, (uint64_t)src, size, mask,
                                   channel);
}

snrt_dma_start_1d_reduction() [1/3]

uint32_t snrt_dma_start_1d_reduction ( uint64_t dst, uint64_t src, size_t size, snrt_comm_t comm, snrt_collective_opcode_t opcode, uint32_t channel = 0 )

inline

Start an asynchronous reduction 1D DMA transfer with 64-bit wide pointers.

Parameters

comm	The communicator for the reduction operation
opcode	Reduction operation

See also

snrt_dma_start_1d(uint64_t, uint64_t, size_t, uint32_t) for a description of the other parameters.

                                                                  {
    uint64_t mask = snrt_get_collective_mask(comm);
    uint32_t txid =
        snrt_dma_start_1d_reduction(dst, src, size, mask, opcode, channel);
    return txid;
}

snrt_dma_start_1d_reduction() [2/3]

uint32_t snrt_dma_start_1d_reduction ( uint64_t dst, uint64_t src, size_t size, uint64_t mask, snrt_collective_opcode_t opcode, uint32_t channel = 0 )

inline

Start an asynchronous reduction 1D DMA transfer with 64-bit wide pointers.

Parameters

mask	Mask defines all involved members
opcode	Reduction operation

See also

snrt_dma_start_1d(uint64_t, uint64_t, size_t, uint32_t) for a description of the other parameters.

                                                                  {
    snrt_dma_enable_reduction(mask, opcode);
    uint32_t txid = snrt_dma_start_1d(dst, src, size, channel);
    snrt_dma_disable_reduction();
    return txid;
}

snrt_dma_start_1d_reduction() [3/3]

uint32_t snrt_dma_start_1d_reduction ( volatile void * dst, volatile void * src, size_t size, uint64_t mask, snrt_collective_opcode_t opcode, uint32_t channel = 0 )

inline

Start an asynchronous reduction 1D DMA transfer using native-size pointers.

This is a convenience overload of snrt_dma_start_1d_reduction(uint64_t, uint64_t, size_t, uint64_t, uint32_t, uint32_t) using void* pointers.

                                                                  {
    return snrt_dma_start_1d_reduction((uint64_t)dst, (uint64_t)src, size, mask,
                                       opcode, channel);
}

snrt_dma_start_2d() [1/2]

snrt_dma_txid_t snrt_dma_start_2d ( uint64_t dst, uint64_t src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t channel )

inline

Start an asynchronous 2D DMA transfer with 64-bit wide pointers.

Parameters

dst	The destination address.
src	The source address.
size	The size of every 1D transfer within the 2D transfer in bytes.
dst_stride	The offset between consecutive 1D transfers at the destination, in bytes.
src_stride	The offset between consecutive 1D transfers at the source, in bytes.
repeat	The number of 1D transfers composing the 2D transfer.
channel	The index of the channel.

Returns

The DMA transfer ID.

                                                           {
#ifdef SNRT_SUPPORTS_DMA
    uint32_t dst_lo = dst & 0xFFFFFFFF;
    uint32_t dst_hi = dst >> 32;
    uint32_t src_lo = src & 0xFFFFFFFF;
    uint32_t src_hi = src >> 32;
    uint32_t cfg = (channel << 2) | 0b10;
    uint32_t txid;
 
    asm volatile(
        "dmsrc %[src_lo], %[src_hi] \n"
        "dmdst %[dst_lo], %[dst_hi] \n"
        "dmstr %[src_stride], %[dst_stride] \n"
        "dmrep %[repeat] \n"
        "dmcpy %[txid], %[size], %[cfg] \n"
        : [ txid ] "=r"(txid)
        : [ src_lo ] "r"(src_lo), [ src_hi ] "r"(src_hi),
          [ dst_lo ] "r"(dst_lo), [ dst_hi ] "r"(dst_hi),
          [ dst_stride ] "r"(dst_stride), [ src_stride ] "r"(src_stride),
          [ repeat ] "r"(repeat), [ size ] "r"(size), [ cfg ] "r"(cfg));
 
    return txid;
#else
    // TODO(colluca): we can implement this as a series of memcpy calls
    return 0;
#endif
}

snrt_dma_start_2d() [2/2]

uint32_t snrt_dma_start_2d ( volatile void * dst, volatile void * src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t channel = 0 )

inline

Start an asynchronous 2D DMA transfer using native-size pointers.

This is a convenience overload of snrt_dma_start_2d(uint64_t, uint64_t, size_t, size_t, size_t, size_t, uint32_t) using void* pointers.

                                                        {
    return snrt_dma_start_2d((uint64_t)dst, (uint64_t)src, size, dst_stride,
                             src_stride, repeat, channel);
}

snrt_dma_start_2d_mcast() [1/2]

uint32_t snrt_dma_start_2d_mcast ( uint64_t dst, uint64_t src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t mask, uint32_t channel = 0 )

inline

Start an asynchronous, multicast 2D DMA transfer with 64-bit wide pointers.

Parameters

mask	Multicast mask.

See also

snrt_dma_start_2d(uint64_t, uint64_t, size_t, size_t, size_t, size_t, uint32_t) for a description of the other parameters.

                                                              {
    snrt_dma_enable_multicast(mask);
    uint32_t txid = snrt_dma_start_2d(dst, src, size, dst_stride, src_stride,
                                      repeat, channel);
    snrt_dma_disable_multicast();
    return txid;
}

snrt_dma_start_2d_mcast() [2/2]

uint32_t snrt_dma_start_2d_mcast ( volatile void * dst, volatile void * src, size_t size, size_t dst_stride, size_t src_stride, size_t repeat, uint32_t mask, uint32_t channel = 0 )

inline

Start an asynchronous, multicast 2D DMA transfer using native-size pointers.

This is a convenience overload of snrt_dma_start_2d_mcast(uint64_t, uint64_t, size_t, size_t, size_t, size_t, uint32_t, uint32_t) using void* pointers.

                                                                             {
    return snrt_dma_start_2d_mcast((uint64_t)dst, (uint64_t)src, size,
                                   dst_stride, src_stride, repeat, mask,
                                   channel);
}

snrt_dma_start_tracking()

void snrt_dma_start_tracking ( )

inline

Start tracking of dma performance region. Does not have any implications on the HW. Only injects a marker in the DMA traces that can be analyzed.

Deprecated

                                      {
#ifdef SNRT_SUPPORTS_DMA
    asm volatile("dmstati zero, 0 \n");
#endif
}

snrt_dma_start_transpose() [1/2]

uint32_t snrt_dma_start_transpose ( uint64_t dst, uint64_t src, size_t size, uint32_t mode, uint32_t tensor_m, uint32_t tensor_n, uint32_t channel = 0 )

inline

Start an asynchronous transposing DMA transfer.

Parameters

mode	Element size selector; elements are 1 << mode bytes.
tensor_m	Rows of the source tensor, in elements.
tensor_n	Columns of the source tensor, in elements.

See also

snrt_dma_start_1d(uint64_t, uint64_t, size_t, uint32_t) for a description of the other parameters.

                                                               {
    snrt_dma_enable_transpose(mode, tensor_m, tensor_n);
    uint32_t txid = snrt_dma_start_1d(dst, src, size, channel);
    snrt_dma_disable_compute();
    return txid;
}

snrt_dma_start_transpose() [2/2]

uint32_t snrt_dma_start_transpose ( volatile void * dst, volatile void * src, size_t size, uint32_t mode, uint32_t tensor_m, uint32_t tensor_n, uint32_t channel = 0 )

inline

Start an asynchronous transposing DMA transfer.

See also

snrt_dma_start_transpose(uint64_t, uint64_t, size_t, uint32_t, uint32_t, uint32_t, uint32_t)

                                                               {
    return snrt_dma_start_transpose((uint64_t)dst, (uint64_t)src, size, mode,
                                    tensor_m, tensor_n, channel);
}

snrt_dma_stop_tracking()

void snrt_dma_stop_tracking ( )

inline

Stop tracking of dma performance region. Does not have any implications on the HW. Only injects a marker in the DMA traces that can be analyzed.

Deprecated

                                     {
#ifdef SNRT_SUPPORTS_DMA
    asm volatile("dmstati zero, 0 \n");
#endif
}

snrt_dma_store_1d_tile()

snrt_dma_txid_t snrt_dma_store_1d_tile ( void * dst, void * src, size_t tile_idx, size_t tile_size, uint32_t prec )

inline

Store a tile to a 1D array.

Parameters

dst	Pointer to the destination array.
src	Pointer to the source tile.
tile_idx	Index of the tile in the 1D array.
tile_size	Number of elements within a tile of the 1D array.
prec	Number of bytes of each element in the 1D array.

                                                             {
    size_t tile_nbytes = tile_size * prec;
    return snrt_dma_start_1d((uint64_t)dst + tile_idx * tile_nbytes,
                             (uint64_t)src, tile_nbytes);
}

snrt_dma_store_2d_tile() [1/2]

snrt_dma_txid_t snrt_dma_store_2d_tile ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec )

inline

Store a 2D tile of a 2D array.

The stride in the source tile is assumed to be that of a 1D tile, effectively. In other words, this is the same as snrt_dma_1d_to_2d().

See also

snrt_dma_store_2d_tile(void *, void *, size_t, size_t, size_t, size_t, size_t, uint32_t, size_t) for a detailed description of the parameters.

                   {
    return snrt_dma_store_2d_tile(dst, src, tile_x1_idx, tile_x0_idx,
                                  tile_x1_size, tile_x0_size, full_x0_size,
                                  prec, tile_x0_size * prec);
}

snrt_dma_store_2d_tile() [2/2]

snrt_dma_txid_t snrt_dma_store_2d_tile ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t tile_ld )

inline

Store a 2D tile to a 2D array.

Parameters

dst	Pointer to the destination array.
src	Pointer to the source tile.
tile_x1_idx	Outermost coordinate of the tile in the 2D array.
tile_x0_idx	Innermost coordinate of the tile in the 2D array.
tile_x1_size	Number of elements in the outermost dimension of the tile.
tile_x0_size	Number of elements in the innermost dimension of the tile.
full_x0_size	Number of elements in the innermost dimension of the array.
prec	Number of bytes of each element in the 2D array.
tile_ld	Leading dimension of the tile, in bytes.

                                   {
    size_t dst_offset = 0;
    // Advance dst array in x0 and x1 dimensions, and convert to byte offset
    dst_offset += tile_x0_idx * tile_x0_size;
    dst_offset += tile_x1_idx * tile_x1_size * full_x0_size;
    dst_offset *= prec;
    // Initiate transfer
    return snrt_dma_start_2d((uint64_t)dst + dst_offset,  // dst
                             (uint64_t)src,               // src
                             tile_x0_size * prec,         // size
                             full_x0_size * prec,         // dst_stride
                             tile_ld,                     // src_stride
                             tile_x1_size                 // repeat
    );
}

snrt_dma_store_2d_tile_from_banks()

snrt_dma_txid_t snrt_dma_store_2d_tile_from_banks ( void * dst, void * src, size_t tile_x1_idx, size_t tile_x0_idx, size_t tile_x1_size, size_t tile_x0_size, size_t full_x0_size, uint32_t prec, size_t num_banks )

inline

Store a 2D tile of a 2D array from a 1D layout occupying a subset of TCDM banks.

Parameters

dst	Pointer to the destination array.
src	Pointer to the source tile.
tile_x1_idx	Outermost coordinate of the tile in the 2D array.
tile_x0_idx	Innermost coordinate of the tile in the 2D array.
tile_x1_size	Number of elements in the outermost dimension of the tile.
tile_x0_size	Number of elements in the innermost dimension of the tile.
full_x0_size	Number of elements in the innermost dimension of the array.
prec	Number of bytes of each element in the 2D array.
num_banks	Number of banks the tile is stored in.

                                     {
    // Calculate new tile size after reshaping the tile in the selected banks
    size_t tile_x0_size_in_banks = (num_banks * SNRT_TCDM_BANK_WIDTH) / prec;
    size_t tile_x1_size_in_banks =
        ceil((tile_x1_size * tile_x0_size) / (double)tile_x0_size_in_banks);
    size_t tile_ld = SNRT_TCDM_HYPERBANK_WIDTH;
    return snrt_dma_store_2d_tile(dst, src, tile_x1_idx, tile_x0_idx,
                                  tile_x1_size_in_banks, tile_x0_size_in_banks,
                                  full_x0_size, prec, tile_ld);
}

snrt_dma_wait()

static void snrt_dma_wait ( snrt_dma_txid_t txid, const uint32_t channel = 0 )

inlinestatic

Block until a DMA transfer finishes on a specific DMA channel.

Parameters

txid	The DMA transfer's ID.
channel	The index of the channel.

Note

The function passes the channel argument as an immediate, thus this must be known at compile time. As a consequence, the function must use internal linkage (static keyword) and must be always inlined. This is true also for all functions invoking this function, and passing down an argument to channel.

                                                             {
#ifdef SNRT_SUPPORTS_DMA
    asm volatile(
        "1: \n"
        "dmstati t0, (%[channel] << 2) | 0 \n"
        "bltu t0, %[txid], 1b \n"
        :
        : [ txid ] "r"(txid), [ channel ] "i"(channel)
        : "t0");
#endif
}

snrt_dma_wait_all()

static void snrt_dma_wait_all ( const uint32_t channel = 0 )

inlinestatic

Block until a specific DMA channel is idle.

Parameters

channel

The index of the channel.

Note

The function passes the channel argument as an immediate, thus this must be known at compile time. As a consequence, the function must use internal linkage (static keyword) and must be always inlined. This is true also for all functions invoking this function, and passing down an argument to channel.

                                                                 {
#ifdef SNRT_SUPPORTS_DMA
    while (snrt_dma_busy(channel))
        ;
#endif
}

snrt_dma_wait_all_channels()

void snrt_dma_wait_all_channels ( uint32_t num_channels )

inline

Block until the first num_channels channels are idle.

Parameters

num_channels

The number of channels to wait on.

                                                              {
    for (int c = 0; c < num_channels; c++) {
        snrt_dma_wait_all(c);
    }
}

snrt_dma_would_block()

static uint32_t snrt_dma_would_block ( const uint32_t channel )

inlinestatic

Read DMA would_block flag.

Parameters

channel

The index of the channel.

Note

The function passes the channel argument as an immediate, thus this must be known at compile time. As a consequence, the function must use internal linkage (static keyword) and must be always inlined. This is true also for all functions invoking this function, and passing down an argument to channel.

                                                                    {
#ifdef SNRT_SUPPORTS_DMA
    uint32_t would_block;
    asm volatile("dmstati %[would_block], (%[channel] << 2) | 3 \n"
                 : [ would_block ] "=r"(would_block)
                 : [ channel ] "i"(channel)
                 :);
    return would_block;
#else
    return 0;
#endif
}