diff --git a/src/arch/riscv/core/zicbom.c b/src/arch/riscv/core/zicbom.c new file mode 100644 index 000000000..306b6c459 --- /dev/null +++ b/src/arch/riscv/core/zicbom.c @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2025 Michael Brown . + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** @file + * + * Cache-block management operations (Zicbom) + * + * We support explicit cache management operations on I/O buffers. + * These are guaranteed to be aligned on their own size and at least + * as large as a (reasonable) cacheline, and therefore cannot cross a + * cacheline boundary. + */ + +#include +#include +#include +#include +#include + +/** Minimum supported cacheline size + * + * We assume that cache management operations will ignore the least + * significant address bits, and so we are safe to assume a cacheline + * size that is smaller than the size actually used by the CPU. + * + * Cache clean and invalidate loops could be made faster by detecting + * the actual cacheline size. + */ +#define CACHE_STRIDE 32 + +/** A cache management extension */ +struct cache_extension { + /** + * Clean data cache (i.e. write cached content back to memory) + * + * @v first First byte + * @v last Last byte + */ + void ( * clean ) ( const void *first, const void *last ); + /** + * Invalidate data cache (i.e. discard any cached content) + * + * @v first First byte + * @v last Last byte + */ + void ( * invalidate ) ( void *first, void *last ); +}; + +/** Define an operation to clean the data cache */ +#define CACHE_CLEAN( extension, insn ) \ + static void extension ## _clean ( const void *first, \ + const void *last ) { \ + \ + __asm__ __volatile__ ( ".option arch, +" #extension "\n\t" \ + "\n1:\n\t" \ + insn "\n\t" \ + "addi %0, %0, %2\n\t" \ + "bltu %0, %1, 1b\n\t" \ + : "+r" ( first ) \ + : "r" ( last ), "i" ( CACHE_STRIDE ) ); \ + } + +/** Define an operation to invalidate the data cache */ +#define CACHE_INVALIDATE( extension, insn ) \ + static void extension ## _invalidate ( void *first, \ + void *last ) { \ + \ + __asm__ __volatile__ ( ".option arch, +" #extension "\n\t" \ + "\n1:\n\t" \ + insn "\n\t" \ + "addi %0, %0, %2\n\t" \ + "bltu %0, %1, 1b\n\t" \ + : "+r" ( first ) \ + : "r" ( last ), "i" ( CACHE_STRIDE ) \ + : "memory" ); \ + } + +/** Define a cache management extension */ +#define CACHE_EXTENSION( extension, clean_insn, invalidate_insn ) \ + CACHE_CLEAN ( extension, clean_insn ); \ + CACHE_INVALIDATE ( extension, invalidate_insn ); \ + static struct cache_extension extension = { \ + .clean = extension ## _clean, \ + .invalidate = extension ## _invalidate, \ + }; + +/** The standard Zicbom extension */ +CACHE_EXTENSION ( zicbom, "cbo.clean (%0)", "cbo.inval (%0)" ); + +/** The T-Head cache management extension */ +CACHE_EXTENSION ( xtheadcmo, "th.dcache.cva %0", "th.dcache.iva %0" ); + +/** + * Clean data cache (with fully coherent memory) + * + * @v first First byte + * @v last Last byte + */ +static void cache_coherent_clean ( const void *first __unused, + const void *last __unused ) { + /* Nothing to do */ +} + +/** + * Invalidate data cache (with fully coherent memory) + * + * @v first First byte + * @v last Last byte + */ +static void cache_coherent_invalidate ( void *first __unused, + void *last __unused ) { + /* Nothing to do */ +} + +/** Dummy cache management extension for fully coherent memory */ +static struct cache_extension cache_coherent = { + .clean = cache_coherent_clean, + .invalidate = cache_coherent_invalidate, +}; + +static void cache_auto_detect ( void ); +static void cache_auto_clean ( const void *first, const void *last ); +static void cache_auto_invalidate ( void *first, void *last ); + +/** The autodetect cache management extension */ +static struct cache_extension cache_auto = { + .clean = cache_auto_clean, + .invalidate = cache_auto_invalidate, +}; + +/** Active cache management extension */ +static struct cache_extension *cache_extension = &cache_auto; + +/** + * Clean data cache (i.e. write cached content back to memory) + * + * @v start Start address + * @v len Length + */ +void cache_clean ( struct io_buffer *iobuf ) { + const void *first; + const void *last; + + /* Do nothing for zero-length buffers */ + if ( ! iob_len ( iobuf ) ) + return; + + /* Construct address range */ + first = ( ( const void * ) + ( ( ( intptr_t ) iobuf->data ) & ~( CACHE_STRIDE - 1 ) ) ); + last = ( iobuf->tail - 1 ); + + /* Clean cache lines */ + cache_extension->clean ( first, last ); +} + +/** + * Invalidate data cache (i.e. discard any cached content) + * + * @v start Start address + * @v len Length + */ +void cache_invalidate ( struct io_buffer *iobuf ) { + void *first; + void *last; + + /* Do nothing for zero-length buffers */ + if ( ! iob_len ( iobuf ) ) + return; + + /* Construct address range */ + first = ( ( void * ) + ( ( ( intptr_t ) iobuf->data ) & ~( CACHE_STRIDE - 1 ) ) ); + last = ( iobuf->tail - 1 ); + + /* Invalidate cache lines */ + cache_extension->invalidate ( first, last ); +} + +/** + * Autodetect and clean data cache + * + * @v first First byte + * @v last Last byte + */ +static void cache_auto_clean ( const void *first, const void *last ) { + + /* Detect cache extension */ + cache_auto_detect(); + + /* Clean data cache */ + cache_extension->clean ( first, last ); +} + +/** + * Autodetect and invalidate data cache + * + * @v first First byte + * @v last Last byte + */ +static void cache_auto_invalidate ( void *first, void *last ) { + + /* Detect cache extension */ + cache_auto_detect(); + + /* Clean data cache */ + cache_extension->invalidate ( first, last ); +} + +/** + * Autodetect cache + * + */ +static void cache_auto_detect ( void ) { + int rc; + + /* Check for standard Zicbom extension */ + if ( ( rc = hart_supported ( "_zicbom" ) ) == 0 ) { + DBGC ( &cache_extension, "CACHE detected Zicbom\n" ); + cache_extension = &zicbom; + return; + } + + /* Check for T-Head cache management extension */ + if ( xthead_supported ( THEAD_SXSTATUS_THEADISAEE ) ) { + DBGC ( &cache_extension, "CACHE detected XTheadCmo\n" ); + cache_extension = &xtheadcmo; + return; + } + + /* Assume coherent memory if no supported extension detected */ + DBGC ( &cache_extension, "CACHE assuming coherent memory\n" ); + cache_extension = &cache_coherent; +} diff --git a/src/arch/riscv/include/ipxe/zicbom.h b/src/arch/riscv/include/ipxe/zicbom.h new file mode 100644 index 000000000..0aeba1e68 --- /dev/null +++ b/src/arch/riscv/include/ipxe/zicbom.h @@ -0,0 +1,17 @@ +#ifndef _IPXE_ZICBOM_H +#define _IPXE_ZICBOM_H + +/** @file + * + * Cache-block management operations (Zicbom) + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include + +extern void cache_clean ( struct io_buffer *iobuf ); +extern void cache_invalidate ( struct io_buffer *iobuf ); + +#endif /* _IPXE_ZICBOM_H */