[riscv] Support explicit cache management operations on I/O buffers

On platforms where DMA devices are not in the same coherency domain as
the CPU cache, it is necessary to be able to explicitly clean the
cache (i.e. force data to be written back to main memory) and
invalidate the cache (i.e. discard any cached data and force a
subsequent read from main memory).

Add support for cache management via the standard Zicbom extension or
the T-Head cache management operations extension, with the supported
extension detected on first use.

Support cache management operations only on I/O buffers, since these
are guaranteed to not share cachelines with other data.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/arch/riscv/core/zicbom.c

@@ -0,0 +1,256 @@
+/*
+ * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Cache-block management operations (Zicbom)
+ *
+ * We support explicit cache management operations on I/O buffers.
+ * These are guaranteed to be aligned on their own size and at least
+ * as large as a (reasonable) cacheline, and therefore cannot cross a
+ * cacheline boundary.
+ */
+
+#include <stdint.h>
+#include <ipxe/hart.h>
+#include <ipxe/xthead.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/zicbom.h>
+
+/** Minimum supported cacheline size
+ *
+ * We assume that cache management operations will ignore the least
+ * significant address bits, and so we are safe to assume a cacheline
+ * size that is smaller than the size actually used by the CPU.
+ *
+ * Cache clean and invalidate loops could be made faster by detecting
+ * the actual cacheline size.
+ */
+#define CACHE_STRIDE 32
+
+/** A cache management extension */
+struct cache_extension {
+	/**
+	 * Clean data cache (i.e. write cached content back to memory)
+	 *
+	 * @v first		First byte
+	 * @v last		Last byte
+	 */
+	void ( * clean ) ( const void *first, const void *last );
+	/**
+	 * Invalidate data cache (i.e. discard any cached content)
+	 *
+	 * @v first		First byte
+	 * @v last		Last byte
+	 */
+	void ( * invalidate ) ( void *first, void *last );
+};
+
+/** Define an operation to clean the data cache */
+#define CACHE_CLEAN( extension, insn )					\
+	static void extension ## _clean ( const void *first,		\
+					  const void *last ) {		\
+									\
+	__asm__ __volatile__ ( ".option arch, +" #extension "\n\t"	\
+			       "\n1:\n\t"				\
+			       insn "\n\t"				\
+			       "addi %0, %0, %2\n\t"			\
+			       "bltu %0, %1, 1b\n\t"			\
+			       : "+r" ( first )				\
+			       : "r" ( last ), "i" ( CACHE_STRIDE ) );	\
+	}
+
+/** Define an operation to invalidate the data cache */
+#define CACHE_INVALIDATE( extension, insn )				\
+	static void extension ## _invalidate ( void *first,		\
+					       void *last ) {		\
+									\
+	__asm__ __volatile__ ( ".option arch, +" #extension "\n\t"	\
+			       "\n1:\n\t"				\
+			       insn "\n\t"				\
+			       "addi %0, %0, %2\n\t"			\
+			       "bltu %0, %1, 1b\n\t"			\
+			       : "+r" ( first )				\
+			       : "r" ( last ), "i" ( CACHE_STRIDE )	\
+			       : "memory" );				\
+	}
+
+/** Define a cache management extension */
+#define CACHE_EXTENSION( extension, clean_insn, invalidate_insn )	\
+	CACHE_CLEAN ( extension, clean_insn );				\
+	CACHE_INVALIDATE ( extension, invalidate_insn );		\
+	static struct cache_extension extension = {			\
+		.clean = extension ## _clean,				\
+		.invalidate = extension ## _invalidate,			\
+	};
+
+/** The standard Zicbom extension */
+CACHE_EXTENSION ( zicbom, "cbo.clean (%0)", "cbo.inval (%0)" );
+
+/** The T-Head cache management extension */
+CACHE_EXTENSION ( xtheadcmo, "th.dcache.cva %0", "th.dcache.iva %0" );
+
+/**
+ * Clean data cache (with fully coherent memory)
+ *
+ * @v first		First byte
+ * @v last		Last byte
+ */
+static void cache_coherent_clean ( const void *first __unused,
+				   const void *last __unused ) {
+	/* Nothing to do */
+}
+
+/**
+ * Invalidate data cache (with fully coherent memory)
+ *
+ * @v first		First byte
+ * @v last		Last byte
+ */
+static void cache_coherent_invalidate ( void *first __unused,
+					void *last __unused ) {
+	/* Nothing to do */
+}
+
+/** Dummy cache management extension for fully coherent memory */
+static struct cache_extension cache_coherent = {
+	.clean = cache_coherent_clean,
+	.invalidate = cache_coherent_invalidate,
+};
+
+static void cache_auto_detect ( void );
+static void cache_auto_clean ( const void *first, const void *last );
+static void cache_auto_invalidate ( void *first, void *last );
+
+/** The autodetect cache management extension */
+static struct cache_extension cache_auto = {
+	.clean = cache_auto_clean,
+	.invalidate = cache_auto_invalidate,
+};
+
+/** Active cache management extension */
+static struct cache_extension *cache_extension = &cache_auto;
+
+/**
+ * Clean data cache (i.e. write cached content back to memory)
+ *
+ * @v start		Start address
+ * @v len		Length
+ */
+void cache_clean ( struct io_buffer *iobuf ) {
+	const void *first;
+	const void *last;
+
+	/* Do nothing for zero-length buffers */
+	if ( ! iob_len ( iobuf ) )
+		return;
+
+	/* Construct address range */
+	first = ( ( const void * )
+		  ( ( ( intptr_t ) iobuf->data ) & ~( CACHE_STRIDE - 1 ) ) );
+	last = ( iobuf->tail - 1 );
+
+	/* Clean cache lines */
+	cache_extension->clean ( first, last );
+}
+
+/**
+ * Invalidate data cache (i.e. discard any cached content)
+ *
+ * @v start		Start address
+ * @v len		Length
+ */
+void cache_invalidate ( struct io_buffer *iobuf ) {
+	void *first;
+	void *last;
+
+	/* Do nothing for zero-length buffers */
+	if ( ! iob_len ( iobuf ) )
+		return;
+
+	/* Construct address range */
+	first = ( ( void * )
+		  ( ( ( intptr_t ) iobuf->data ) & ~( CACHE_STRIDE - 1 ) ) );
+	last = ( iobuf->tail - 1 );
+
+	/* Invalidate cache lines */
+	cache_extension->invalidate ( first, last );
+}
+
+/**
+ * Autodetect and clean data cache
+ *
+ * @v first		First byte
+ * @v last		Last byte
+ */
+static void cache_auto_clean ( const void *first, const void *last ) {
+
+	/* Detect cache extension */
+	cache_auto_detect();
+
+	/* Clean data cache */
+	cache_extension->clean ( first, last );
+}
+
+/**
+ * Autodetect and invalidate data cache
+ *
+ * @v first		First byte
+ * @v last		Last byte
+ */
+static void cache_auto_invalidate ( void *first, void *last ) {
+
+	/* Detect cache extension */
+	cache_auto_detect();
+
+	/* Clean data cache */
+	cache_extension->invalidate ( first, last );
+}
+
+/**
+ * Autodetect cache
+ *
+ */
+static void cache_auto_detect ( void ) {
+	int rc;
+
+	/* Check for standard Zicbom extension */
+	if ( ( rc = hart_supported ( "_zicbom" ) ) == 0 ) {
+		DBGC ( &cache_extension, "CACHE detected Zicbom\n" );
+		cache_extension = &zicbom;
+		return;
+	}
+
+	/* Check for T-Head cache management extension */
+	if ( xthead_supported ( THEAD_SXSTATUS_THEADISAEE ) ) {
+		DBGC ( &cache_extension, "CACHE detected XTheadCmo\n" );
+		cache_extension = &xtheadcmo;
+		return;
+	}
+
+	/* Assume coherent memory if no supported extension detected */
+	DBGC ( &cache_extension, "CACHE assuming coherent memory\n" );
+	cache_extension = &cache_coherent;
+}

src/arch/riscv/include/ipxe/zicbom.h

@@ -0,0 +1,17 @@
+#ifndef _IPXE_ZICBOM_H
+#define _IPXE_ZICBOM_H
+
+/** @file
+ *
+ * Cache-block management operations (Zicbom)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/iobuf.h>
+
+extern void cache_clean ( struct io_buffer *iobuf );
+extern void cache_invalidate ( struct io_buffer *iobuf );
+
+#endif /* _IPXE_ZICBOM_H */