sync.h

Go to the documentation of this file.

// Copyright 2023 ETH Zurich and University of Bologna.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0
//
// Luca Colagrande <colluca@iis.ee.ethz.ch>
// Viviane Potocnik <vivianep@iis.ee.ethz.ch>
 
#pragma once
 
#include "../../deps/riscv-opcodes/encoding.h"
 
#include <math.h>
 
//================================================================================
// Communicator functions
//================================================================================
 
extern __thread snrt_comm_info_t snrt_comm_world_info;
extern __thread snrt_comm_t snrt_comm_world;
 
inline void snrt_comm_init() {
    // Point to default-initialized communicator struct, with barrier pointer
    // in L3.
    snrt_comm_world = &snrt_comm_world_info;
 
    // Allocate barrier counter in L1. This allows us to perform global
    // hardware barriers, as reductions are currently not supported in L3.
    // All clusters allocate a barrier counter because we want to keep all
    // clusters' L1 allocators aligned, but only the zero-th cluster's is
    // actually used. So all clusters allocate one, but only the zero-th
    // cluster's is initialized. A global barrier is then used to ensure
    // all cores "see" the initialized value. This global barrier uses the
    // default-initialized barrier pointer in L3. It must thus be a software
    // barrier, as we currently do not support hardware reductions in L3.
    void *barrier_ptr = snrt_l1_alloc_cluster_local(sizeof(uint32_t));
    barrier_ptr = snrt_remote_l1_ptr(barrier_ptr, snrt_cluster_idx(), 0);
    if (snrt_global_core_idx() == 0) {
        *(uint32_t *)barrier_ptr = 0;
        // TODO(colluca): this is a workaround that shouldn't be necessary.
        // It seems some tests expect the next pointer at the start of the
        // user application to be aligned to the hyperbank.
        // > Should we get rid of the alloc_v1 API altogether and fix these?
        snrt_l1_update_next(snrt_l1_next_aligned_hyperbank());
    }
    snrt_global_sw_barrier();
 
    // Update the communicator struct, pointing to the barrier pointer in L1.
    // This whole workaround is required because we cannot statically allocate
    // variables in L1.
    snrt_comm_world->barrier_ptr = (uint32_t *)barrier_ptr;
}
 
inline void snrt_comm_create(uint32_t size, snrt_comm_t *communicator) {
    // Allocate communicator struct in L1 and point to it.
    *communicator =
        (snrt_comm_t)snrt_l1_alloc_cluster_local(sizeof(snrt_comm_info_t));
 
    // Allocate barrier counter in L1. This allows us to perform global
    // hardware barriers, as reductions are currently not supported in L3.
    // All clusters allocate a barrier counter because we want to keep all
    // clusters' L1 allocators aligned, but only the zero-th cluster's is
    // actually used. So all clusters allocate one, but only the zero-th
    // cluster's is initialized. A global barrier is then used to ensure
    // all cores "see" the initialized value.
    void *barrier_ptr = snrt_l1_alloc_cluster_local(sizeof(uint32_t));
    barrier_ptr = snrt_remote_l1_ptr(barrier_ptr, snrt_cluster_idx(), 0);
    if (snrt_global_core_idx() == 0) *(uint32_t *)barrier_ptr = 0;
    snrt_global_barrier();
 
    // Initialize communicator, pointing to the newly-allocated barrier
    // counter in L3.
    (*communicator)->size = size;
    (*communicator)->base = 0;
    (*communicator)->mask = size - 1;
    (*communicator)->barrier_ptr = (uint32_t *)barrier_ptr;
    (*communicator)->is_participant = snrt_cluster_idx() < size;
}
 
//================================================================================
// Mutex functions
//================================================================================
 
inline volatile uint32_t *snrt_mutex() { return &_snrt_mutex; }
 
inline void snrt_mutex_acquire(volatile uint32_t *pmtx) {
    asm volatile(
        "li            t0,1          # t0 = 1\n"
        "1:\n"
        "  amoswap.w.aq  t0,t0,(%0)   # t0 = oldlock & lock = 1\n"
        "  bnez          t0,1b      # Retry if previously set)\n"
        : "+r"(pmtx)
        :
        : "t0");
}
 
inline void snrt_mutex_ttas_acquire(volatile uint32_t *pmtx) {
    asm volatile(
        "1:\n"
        "  lw t0, 0(%0)\n"
        "  bnez t0, 1b\n"
        "  li t0,1          # t0 = 1\n"
        "2:\n"
        "  amoswap.w.aq  t0,t0,(%0)   # t0 = oldlock & lock = 1\n"
        "  bnez          t0,2b      # Retry if previously set)\n"
        : "+r"(pmtx)
        :
        : "t0");
}
 
inline void snrt_mutex_release(volatile uint32_t *pmtx) {
    asm volatile("amoswap.w.rl  x0,x0,(%0)   # Release lock by storing 0\n"
                 : "+r"(pmtx));
}
 
//================================================================================
// Barrier functions
//================================================================================
 
inline void snrt_wake_clusters(uint32_t core_mask, snrt_comm_t comm = NULL) {
    // If no communicator is given, world communicator is used as default.
    if (comm == NULL) comm = snrt_comm_world;
 
#ifdef SNRT_SUPPORTS_NARROW_MULTICAST
    // Multicast cluster interrupt to every other cluster's core
    if (snrt_cluster_num() > 0) {
        volatile snitch_cluster_t *cluster = snrt_cluster(0);
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Waddress-of-packed-member"
        uint32_t *addr = (uint32_t *)&(cluster->peripheral_reg.cl_clint_set.w);
#pragma clang diagnostic pop
        uint32_t mcast_mask = snrt_get_collective_mask(comm);
        snrt_enable_multicast(mcast_mask);
        *addr = core_mask;
        snrt_disable_multicast();
    }
#else
    // Wake clusters sequentially.
    // We find all clusters represented by the (base, mask) encoding through
    // submask enumeration (https://codeforces.com/blog/entry/108942).
    uint32_t mask = comm->mask;
    uint32_t fixed = comm->base & ~mask;
    uint32_t submask = 0;
    do {
        uint32_t i = fixed | submask;
        if (snrt_cluster_idx() != i) snrt_int_cluster_set(core_mask, i);
        submask = (submask - 1) & mask;
    } while (submask != 0);
#endif
}
 
inline void snrt_cluster_hw_barrier() {
    asm volatile("csrr x0, barrier" ::: "memory");
}
 
static inline void snrt_inter_cluster_sw_barrier(snrt_comm_t comm = NULL) {
    // If no communicator is given, world communicator is used as default.
    if (comm == NULL) comm = snrt_comm_world;
 
    // If the current cluster is not a participant, return immediately.
    if (!comm->is_participant) return;
 
    // Clusters participating in the barrier increment a shared counter.
    uint32_t cnt = __atomic_add_fetch(comm->barrier_ptr, 1, __ATOMIC_RELAXED);
 
    // All but the last cluster arriving on the barrier enter WFI. The last
    // cluster resets the counter for the next barrier (if any) and multicasts
    // an interrupt to wake up the other clusters.
    if (cnt == comm->size) {
        *(comm->barrier_ptr) = 0;
        snrt_fence();
        snrt_wake_clusters(1 << snrt_cluster_core_idx(), comm);
    } else {
        snrt_wfi();
    }
    // Clear interrupt for next barrier (interrupt arrives also at sender)
    snrt_int_clr_mcip();
}
 
static inline void snrt_inter_cluster_barrier(snrt_comm_t comm = NULL) {
    // If no communicator is given, world communicator is used as default.
    if (comm == NULL) comm = snrt_comm_world;
 
    // If the current cluster is not a participant, return immediately.
    if (!comm->is_participant) return;
 
#ifdef SNRT_SUPPORTS_NARROW_REDUCTION
    // Fetch the address for the reduction
    volatile uint32_t *addr = comm->barrier_ptr;
 
    // Compose collective mask
    uint64_t mask = snrt_get_collective_mask(comm);
 
    // Launch the reduction
    snrt_enable_reduction(mask, SNRT_REDUCTION_BARRIER);
    *addr = 0;
    snrt_disable_reduction();
 
    // Fence to wait until the reduction is finished
    snrt_fence();
#else
    snrt_inter_cluster_sw_barrier(comm);
#endif
}
 
inline void snrt_global_sw_barrier(snrt_comm_t comm) {
    // Synchronize cores in a cluster with the HW barrier
    snrt_cluster_hw_barrier();
 
    // Synchronize all clusters
    if (snrt_is_dm_core()) {
        snrt_inter_cluster_sw_barrier(comm);
    }
 
    // Synchronize cores in a cluster with the HW barrier
    snrt_cluster_hw_barrier();
}
 
inline void snrt_global_barrier(snrt_comm_t comm) {
    // Synchronize cores in a cluster with the HW barrier
    snrt_cluster_hw_barrier();
 
    // Synchronize all clusters
    if (snrt_is_dm_core()) {
        snrt_inter_cluster_barrier(comm);
    }
 
    // Synchronize cores in a cluster with the HW barrier
    snrt_cluster_hw_barrier();
}
 
inline void snrt_partial_barrier(snrt_barrier_t *barr, uint32_t n) {
    // Remember previous iteration
    uint32_t prev_it = barr->iteration;
    uint32_t cnt = __atomic_add_fetch(&barr->cnt, 1, __ATOMIC_RELAXED);
 
    // Increment the barrier counter
    if (cnt == n) {
        barr->cnt = 0;
        __atomic_add_fetch(&barr->iteration, 1, __ATOMIC_RELAXED);
    } else {
        // Some threads have not reached the barrier --> Let's wait
        while (prev_it == barr->iteration)
            ;
    }
}
 
//================================================================================
// Reduction functions
//================================================================================
 
inline uint32_t snrt_global_all_to_all_reduction(uint32_t value) {
    // Reduce cores within cluster in TCDM
    uint32_t *cluster_result = &(snrt_cls()->reduction);
    uint32_t tmp = __atomic_fetch_add(cluster_result, value, __ATOMIC_RELAXED);
 
    // Wait for writeback to ensure AMO is seen by all cores after barrier
    snrt_wait_writeback(tmp);
    snrt_cluster_hw_barrier();
 
    // Reduce DM cores across clusters in global memory
    if (snrt_is_dm_core()) {
        __atomic_add_fetch(&_reduction_result, *cluster_result,
                           __ATOMIC_RELAXED);
        snrt_inter_cluster_barrier();
        *cluster_result = _reduction_result;
    }
    snrt_cluster_hw_barrier();
    return *cluster_result;
}
 
template <typename T>
inline void snrt_global_reduction_dma(T *dst_buffer, T *src_buffer, size_t len,
                                      snrt_comm_t comm = NULL) {
    // If no communicator is given, world communicator is used as default.
    if (comm == NULL) comm = snrt_comm_world;
 
    // If we have a single cluster, no reduction has to be done
    if (comm->size > 1) {
        // DMA core will send compute cores' data, so it must wait on it
        // to be available
        snrt_fpu_fence();
        snrt_cluster_hw_barrier();
 
        // Iterate levels in the binary reduction tree
        int num_levels = ceil(log2(comm->size));
        for (unsigned int level = 0; level < num_levels; level++) {
            // Determine whether the current cluster is an active cluster.
            // An active cluster is a cluster that participates in the current
            // level of the reduction tree. Every second cluster among the
            // active ones is a sender.
            uint32_t is_active = (snrt_cluster_idx() % (1 << level)) == 0;
            uint32_t is_sender = (snrt_cluster_idx() % (1 << (level + 1))) != 0;
 
            // If the cluster is a sender, it sends the data in its source
            // buffer to the respective receiver's destination buffer
            if (is_active && is_sender) {
                if (!snrt_is_compute_core()) {
                    uint64_t dst = (uint64_t)dst_buffer -
                                   (1 << level) * SNRT_CLUSTER_OFFSET;
                    snrt_dma_start_1d(dst, (uint64_t)src_buffer,
                                      len * sizeof(T));
                    snrt_dma_wait_all();
                }
            }
 
            // Synchronize senders and receivers
            snrt_global_barrier(comm);
 
            // Every cluster which is not a sender performs the reduction
            if (is_active && !is_sender) {
                // Computation is parallelized over the compute cores
                if (snrt_is_compute_core()) {
                    uint32_t items_per_core =
                        len / snrt_cluster_compute_core_num();
                    uint32_t core_offset =
                        snrt_cluster_core_idx() * items_per_core;
                    for (uint32_t i = 0; i < items_per_core; i++) {
                        uint32_t abs_i = core_offset + i;
                        src_buffer[abs_i] += dst_buffer[abs_i];
                    }
                }
            }
 
            // Synchronize compute and DM cores for next tree level
            snrt_fpu_fence();
            snrt_cluster_hw_barrier();
        }
    }
}
 
//================================================================================
// Memory consistency
//================================================================================
 
inline void snrt_wait_writeback(uint32_t val) {
    asm volatile("mv %0, %0" : "+r"(val)::);
}
 
//================================================================================
// User functions
//================================================================================
 
inline void snrt_set_awuser(uint64_t field) {
    write_csr(user_low, (uint32_t)(field));
    write_csr(user_high, (uint32_t)(field >> 32));
}
 
inline void snrt_set_awuser_low(uint32_t field) {
    write_csr(user_low, (uint32_t)(field));
}
 
inline uint64_t snrt_get_collective_mask(snrt_comm_t comm) {
    return comm->mask * SNRT_CLUSTER_OFFSET;
}
 
//================================================================================
// Multicast functions
//================================================================================
 
inline void snrt_enable_multicast(uint64_t mask) {
    snrt_collective_t op;
    op.f.opcode = SNRT_COLLECTIVE_MULTICAST;
    op.f.mask = mask;
    snrt_set_awuser(op.w);
}
 
inline void snrt_disable_multicast() { snrt_set_awuser(0); }
 
//================================================================================
// Reduction functions
//================================================================================
 
inline void snrt_enable_reduction(uint64_t mask,
                                  snrt_collective_opcode_t opcode) {
    snrt_collective_t op;
    op.f.opcode = opcode;
    op.f.mask = mask;
    snrt_set_awuser(op.w);
}
 
inline void snrt_disable_reduction() { snrt_set_awuser(0); }