84ccc54fda
Driver did not allow to execute any invalidate all operation. This operation should not be allowed for data cache as it will lead to undefined behavior but it is ok to invalidate instruction cache. Signed-off-by: Krzysztof Chruściński <krzysztof.chruscinski@nordicsemi.no>
411 lines
7.4 KiB
C
411 lines
7.4 KiB
C
/*
|
|
* Copyright (c) 2024 Nordic Semiconductor ASA
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
#include <zephyr/kernel.h>
|
|
#include <zephyr/drivers/cache.h>
|
|
#include <zephyr/sys/barrier.h>
|
|
#include <hal/nrf_cache.h>
|
|
#include <zephyr/logging/log.h>
|
|
|
|
LOG_MODULE_REGISTER(cache_nrfx, CONFIG_CACHE_LOG_LEVEL);
|
|
|
|
#if !defined(NRF_ICACHE) && defined(NRF_CACHE)
|
|
#define NRF_ICACHE NRF_CACHE
|
|
#endif
|
|
|
|
#define CACHE_BUSY_RETRY_INTERVAL_US 10
|
|
|
|
|
|
enum k_nrf_cache_op {
|
|
/*
|
|
* Sequentially loop through all dirty lines and write those data units to
|
|
* memory.
|
|
*
|
|
* This is FLUSH in Zephyr nomenclature.
|
|
*/
|
|
K_NRF_CACHE_CLEAN,
|
|
|
|
/*
|
|
* Mark all lines as invalid, ignoring any dirty data.
|
|
*
|
|
* This is INVALIDATE in Zephyr nomenclature.
|
|
*/
|
|
K_NRF_CACHE_INVD,
|
|
|
|
/*
|
|
* Clean followed by invalidate
|
|
*
|
|
* This is FLUSH_AND_INVALIDATE in Zephyr nomenclature.
|
|
*/
|
|
K_NRF_CACHE_FLUSH,
|
|
};
|
|
|
|
static inline bool is_cache_busy(NRF_CACHE_Type *cache)
|
|
{
|
|
#if NRF_CACHE_HAS_STATUS
|
|
return nrf_cache_busy_check(cache);
|
|
#else
|
|
return false;
|
|
#endif
|
|
}
|
|
|
|
static inline void wait_for_cache(NRF_CACHE_Type *cache)
|
|
{
|
|
while (is_cache_busy(cache)) {
|
|
}
|
|
}
|
|
|
|
static inline int _cache_all(NRF_CACHE_Type *cache, enum k_nrf_cache_op op)
|
|
{
|
|
wait_for_cache(cache);
|
|
|
|
barrier_dsync_fence_full();
|
|
|
|
switch (op) {
|
|
|
|
#if NRF_CACHE_HAS_TASK_CLEAN
|
|
case K_NRF_CACHE_CLEAN:
|
|
nrf_cache_task_trigger(cache, NRF_CACHE_TASK_CLEANCACHE);
|
|
break;
|
|
#endif
|
|
|
|
case K_NRF_CACHE_INVD:
|
|
nrf_cache_task_trigger(cache, NRF_CACHE_TASK_INVALIDATECACHE);
|
|
break;
|
|
|
|
#if NRF_CACHE_HAS_TASK_FLUSH
|
|
case K_NRF_CACHE_FLUSH:
|
|
nrf_cache_task_trigger(cache, NRF_CACHE_TASK_FLUSHCACHE);
|
|
break;
|
|
#endif
|
|
|
|
default:
|
|
break;
|
|
}
|
|
|
|
wait_for_cache(cache);
|
|
|
|
return 0;
|
|
}
|
|
|
|
#if NRF_CACHE_HAS_LINEADDR
|
|
static inline void _cache_line(NRF_CACHE_Type *cache, enum k_nrf_cache_op op, uintptr_t line_addr)
|
|
{
|
|
do {
|
|
wait_for_cache(cache);
|
|
|
|
nrf_cache_lineaddr_set(cache, line_addr);
|
|
|
|
barrier_dsync_fence_full();
|
|
|
|
switch (op) {
|
|
|
|
#if NRF_CACHE_HAS_TASK_CLEAN
|
|
case K_NRF_CACHE_CLEAN:
|
|
nrf_cache_task_trigger(cache, NRF_CACHE_TASK_CLEANLINE);
|
|
break;
|
|
#endif
|
|
|
|
case K_NRF_CACHE_INVD:
|
|
nrf_cache_task_trigger(cache, NRF_CACHE_TASK_INVALIDATELINE);
|
|
break;
|
|
|
|
#if NRF_CACHE_HAS_TASK_FLUSH
|
|
case K_NRF_CACHE_FLUSH:
|
|
nrf_cache_task_trigger(cache, NRF_CACHE_TASK_FLUSHLINE);
|
|
break;
|
|
#endif
|
|
|
|
default:
|
|
break;
|
|
}
|
|
} while (nrf_cache_lineaddr_get(cache) != line_addr);
|
|
}
|
|
|
|
static inline int _cache_range(NRF_CACHE_Type *cache, enum k_nrf_cache_op op, void *addr,
|
|
size_t size)
|
|
{
|
|
uintptr_t line_addr = (uintptr_t)addr;
|
|
uintptr_t end_addr;
|
|
|
|
/* Some SOCs has a bug that requires to set 28th bit in the address on
|
|
* Trustzone secure builds.
|
|
*/
|
|
if (IS_ENABLED(CONFIG_CACHE_NRF_PATCH_LINEADDR) &&
|
|
!IS_ENABLED(CONFIG_TRUSTED_EXECUTION_NONSECURE)) {
|
|
line_addr |= BIT(28);
|
|
}
|
|
|
|
end_addr = line_addr + size;
|
|
|
|
/*
|
|
* Align address to line size
|
|
*/
|
|
line_addr &= ~(CONFIG_DCACHE_LINE_SIZE - 1);
|
|
|
|
do {
|
|
_cache_line(cache, op, line_addr);
|
|
line_addr += CONFIG_DCACHE_LINE_SIZE;
|
|
} while (line_addr < end_addr);
|
|
|
|
wait_for_cache(cache);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static inline int _cache_checks(NRF_CACHE_Type *cache, enum k_nrf_cache_op op, void *addr,
|
|
size_t size, bool is_range)
|
|
{
|
|
/* Check if the cache is enabled */
|
|
if (!nrf_cache_enable_check(cache)) {
|
|
return -EAGAIN;
|
|
}
|
|
|
|
if (!is_range) {
|
|
return _cache_all(cache, op);
|
|
}
|
|
|
|
/* Check for invalid address or size */
|
|
if ((!addr) || (!size)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return _cache_range(cache, op, addr, size);
|
|
}
|
|
#else
|
|
static inline int _cache_all_checks(NRF_CACHE_Type *cache, enum k_nrf_cache_op op)
|
|
{
|
|
/* Check if the cache is enabled */
|
|
if (!nrf_cache_enable_check(cache)) {
|
|
return -EAGAIN;
|
|
}
|
|
return _cache_all(cache, op);
|
|
}
|
|
#endif /* NRF_CACHE_HAS_LINEADDR */
|
|
|
|
#if defined(NRF_DCACHE) && NRF_CACHE_HAS_TASKS
|
|
|
|
void cache_data_enable(void)
|
|
{
|
|
nrf_cache_enable(NRF_DCACHE);
|
|
}
|
|
|
|
int cache_data_flush_all(void)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_CLEAN
|
|
return _cache_checks(NRF_DCACHE, K_NRF_CACHE_CLEAN, NULL, 0, false);
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
void cache_data_disable(void)
|
|
{
|
|
if (nrf_cache_enable_check(NRF_DCACHE)) {
|
|
(void)cache_data_flush_all();
|
|
}
|
|
nrf_cache_disable(NRF_DCACHE);
|
|
}
|
|
|
|
int cache_data_invd_all(void)
|
|
{
|
|
/* We really do not want to invalidate the whole data cache. */
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_data_flush_and_invd_all(void)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_FLUSH
|
|
return _cache_checks(NRF_DCACHE, K_NRF_CACHE_FLUSH, NULL, 0, false);
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
int cache_data_flush_range(void *addr, size_t size)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_CLEAN
|
|
return _cache_checks(NRF_DCACHE, K_NRF_CACHE_CLEAN, addr, size, true);
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
int cache_data_invd_range(void *addr, size_t size)
|
|
{
|
|
return _cache_checks(NRF_DCACHE, K_NRF_CACHE_INVD, addr, size, true);
|
|
}
|
|
|
|
int cache_data_flush_and_invd_range(void *addr, size_t size)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_FLUSH
|
|
return _cache_checks(NRF_DCACHE, K_NRF_CACHE_FLUSH, addr, size, true);
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
#else
|
|
|
|
void cache_data_enable(void)
|
|
{
|
|
/* Nothing */
|
|
}
|
|
|
|
void cache_data_disable(void)
|
|
{
|
|
/* Nothing */
|
|
}
|
|
|
|
int cache_data_flush_all(void)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_data_invd_all(void)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_data_flush_and_invd_all(void)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_data_flush_range(void *addr, size_t size)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_data_invd_range(void *addr, size_t size)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_data_flush_and_invd_range(void *addr, size_t size)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
#endif /* NRF_DCACHE */
|
|
|
|
#if defined(NRF_ICACHE) && NRF_CACHE_HAS_TASKS
|
|
|
|
void cache_instr_enable(void)
|
|
{
|
|
nrf_cache_enable(NRF_ICACHE);
|
|
}
|
|
|
|
void cache_instr_disable(void)
|
|
{
|
|
nrf_cache_disable(NRF_ICACHE);
|
|
}
|
|
|
|
int cache_instr_flush_all(void)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_CLEAN
|
|
#if NRF_CACHE_HAS_LINEADDR
|
|
return _cache_checks(NRF_ICACHE, K_NRF_CACHE_CLEAN, NULL, 0, false);
|
|
#else
|
|
return _cache_all_checks(NRF_ICACHE, K_NRF_CACHE_CLEAN);
|
|
#endif
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
int cache_instr_invd_all(void)
|
|
{
|
|
#if NRF_CACHE_HAS_LINEADDR
|
|
return _cache_checks(NRF_ICACHE, K_NRF_CACHE_INVD, NULL, 0, false);
|
|
#else
|
|
return _cache_all_checks(NRF_ICACHE, K_NRF_CACHE_INVD);
|
|
#endif
|
|
}
|
|
|
|
int cache_instr_flush_and_invd_all(void)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_FLUSH
|
|
#if NRF_CACHE_HAS_LINEADDR
|
|
return _cache_checks(NRF_ICACHE, K_NRF_CACHE_FLUSH, NULL, 0, false);
|
|
#else
|
|
return _cache_all_checks(NRF_ICACHE, K_NRF_CACHE_FLUSH);
|
|
#endif
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
int cache_instr_flush_range(void *addr, size_t size)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_CLEAN && NRF_CACHE_HAS_LINEADDR
|
|
return _cache_checks(NRF_ICACHE, K_NRF_CACHE_CLEAN, addr, size, true);
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
int cache_instr_invd_range(void *addr, size_t size)
|
|
{
|
|
#if NRF_CACHE_HAS_LINEADDR
|
|
return _cache_checks(NRF_ICACHE, K_NRF_CACHE_INVD, addr, size, true);
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
int cache_instr_flush_and_invd_range(void *addr, size_t size)
|
|
{
|
|
#if NRF_CACHE_HAS_TASK_FLUSH && NRF_CACHE_HAS_LINEADDR
|
|
return _cache_checks(NRF_ICACHE, K_NRF_CACHE_FLUSH, addr, size, true);
|
|
#else
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
#else
|
|
|
|
void cache_instr_enable(void)
|
|
{
|
|
/* Nothing */
|
|
}
|
|
|
|
void cache_instr_disable(void)
|
|
{
|
|
/* Nothing */
|
|
}
|
|
|
|
int cache_instr_flush_all(void)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_instr_invd_all(void)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_instr_flush_and_invd_all(void)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_instr_flush_range(void *addr, size_t size)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_instr_invd_range(void *addr, size_t size)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
int cache_instr_flush_and_invd_range(void *addr, size_t size)
|
|
{
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
#endif /* NRF_ICACHE */
|