An MMC/SD/SDIO stack using CAM

Implement the MMC/SD/SDIO protocol within a CAM framework. CAM's
flexible queueing will make it easier to write non-storage drivers
than the legacy stack. SDIO drivers from both the kernel and as
userland daemons are possible, though much of that functionality will
come later.

Some of the CAM integration isn't complete (there are sleeps in the
device probe state machine, for example), but those minor issues can
be improved in-tree more easily than out of tree and shouldn't gate
progress on other fronts. Appologies to reviews if specific items
have been overlooked.

Submitted by: Ilya Bakulin
Reviewed by: emaste, imp, mav, adrian, ian
Differential Review: https://reviews.freebsd.org/D4761

merge with first commit, various compile hacks.

etc/mtree/BSD.include.dist

@@ -90,6 +90,8 @@
     cam
         ata
         ..
+        mmc
+        ..
         nvme
         ..
         scsi

include/Makefile

@@ -42,7 +42,7 @@ LHDRS=	aio.h errno.h fcntl.h linker_set.h poll.h stdatomic.h stdint.h \
 LDIRS=	bsm cam geom net net80211 netgraph netinet netinet6 \
 	netipsec netsmb nfs nfsclient nfsserver sys vm
 
-LSUBDIRS=	cam/ata cam/nvme cam/scsi \
+LSUBDIRS=	cam/ata cam/mmc cam/nvme cam/scsi \
 	dev/acpica dev/agp dev/an dev/bktr dev/ciss dev/filemon dev/firewire \
 	dev/hwpmc dev/hyperv \
 	dev/ic dev/iicbus dev/io dev/lmc dev/mfi dev/mmc dev/nvme \

lib/libcam/Makefile

@@ -38,6 +38,7 @@ MLINKS+=	cam.3 cam_open_device.3 \
 
 .PATH:		${SRCTOP}/sys/cam \
 		${SRCTOP}/sys/cam/ata \
+		${SRCTOP}/sys/cam/mmc \
 		${SRCTOP}/sys/cam/scsi
 
 CFLAGS+=	-I${.CURDIR} -I${SRCTOP}/sys

sys/amd64/conf/MMCCAM

@@ -0,0 +1,36 @@
+# MMCCAM is the kernel config for doing MMC on CAM development
+# and testing on bhyve
+# $FreeBSD$
+
+include         MINIMAL
+
+ident		MMCCAM
+
+# Access GPT-formatted and labeled root volume
+options         GEOM_PART_GPT
+options         GEOM_LABEL
+
+# UART -- for bhyve console
+device          uart
+
+# kgdb stub
+device          bvmdebug
+
+# VirtIO support, needed for bhyve
+device		virtio			# Generic VirtIO bus (required)
+device		virtio_pci		# VirtIO PCI device
+device		vtnet			# VirtIO Ethernet device
+device		virtio_blk		# VirtIO Block device
+device		virtio_scsi		# VirtIO SCSI device
+device		virtio_balloon		# VirtIO Memory Balloon device
+
+# CAM-specific stuff
+device		pass
+device		scbus
+device		da
+device		mmccam
+
+options	MMCCAM
+# Add CAMDEBUG stuff
+options CAMDEBUG
+options CAM_DEBUG_FLAGS=(CAM_DEBUG_INFO|CAM_DEBUG_PROBE|CAM_DEBUG_PERIPH|CAM_DEBUG_TRACE)

sys/arm/broadcom/bcm2835/bcm2835_sdhci.c

@@ -52,6 +52,8 @@ __FBSDID("$FreeBSD$");
 #include "mmcbr_if.h"
 #include "sdhci_if.h"
 
+#include "opt_mmccam.h"
+
 #include "bcm2835_dma.h"
 #include <arm/broadcom/bcm2835/bcm2835_mbox_prop.h>
 #include "bcm2835_vcbus.h"
@@ -253,7 +255,11 @@ bcm_sdhci_attach(device_t dev)
 	bus_generic_probe(dev);
 	bus_generic_attach(dev);
 
+#ifdef MMCCAM
+	sdhci_cam_start_slot(&sc->sc_slot);
+#else
 	sdhci_start_slot(&sc->sc_slot);
+#endif
 
 	return (0);
 

sys/arm/conf/BEAGLEBONE-MMCCAM

@@ -0,0 +1,21 @@
+#
+# BEAGLEBONE-MMCCAM
+#
+# Custom kernel for Beaglebone plus MMCCAM as opposed to the prior MMC stack. It is
+# present to keep it building in tree since it wouldn't work in LINT.
+#
+# $FreeBSD$
+
+include		BEAGLEBONE
+
+# Add CAMDEBUG stuff
+options 	CAMDEBUG
+options 	CAM_DEBUG_FLAGS=(CAM_DEBUG_INFO|CAM_DEBUG_PROBE|CAM_DEBUG_PERIPH|CAM_DEBUG_TRACE)
+
+# pass(4) device
+device		pass
+device		mmccam
+options		MMCCAM
+
+nodevice	mmc
+nodevice	mmcsd

sys/arm/ti/ti_sdhci.c

@@ -39,6 +39,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
@@ -60,6 +62,8 @@ __FBSDID("$FreeBSD$");
 #include <arm/ti/ti_hwmods.h>
 #include "gpio_if.h"
 
+#include "opt_mmccam.h"
+
 struct ti_sdhci_softc {
 	device_t		dev;
 	struct sdhci_fdt_gpio * gpio;
@@ -122,6 +126,11 @@ static struct ofw_compat_data compat_data[] = {
 #define	  MMCHS_SD_CAPA_VS30		  (1 << 25)
 #define	  MMCHS_SD_CAPA_VS33		  (1 << 24)
 
+/* Forward declarations, CAM-relataed */
+// static void ti_sdhci_cam_poll(struct cam_sim *);
+// static void ti_sdhci_cam_action(struct cam_sim *, union ccb *);
+// static int ti_sdhci_cam_settran_settings(struct ti_sdhci_softc *sc, union ccb *);
+
 static inline uint32_t
 ti_mmchs_read_4(struct ti_sdhci_softc *sc, bus_size_t off)
 {
@@ -241,6 +250,22 @@ ti_sdhci_write_1(device_t dev, struct sdhci_slot *slot, bus_size_t off,
 	struct ti_sdhci_softc *sc = device_get_softc(dev);
 	uint32_t val32;
 
+#ifdef MMCCAM
+	uint32_t newval32;
+	if (off == SDHCI_HOST_CONTROL) {
+		val32 = ti_mmchs_read_4(sc, MMCHS_CON);
+		newval32  = val32;
+		if (val & SDHCI_CTRL_8BITBUS) {
+			device_printf(dev, "Custom-enabling 8-bit bus\n");
+			newval32 |= MMCHS_CON_DW8;
+		} else {
+			device_printf(dev, "Custom-disabling 8-bit bus\n");
+			newval32 &= ~MMCHS_CON_DW8;
+		}
+		if (newval32 != val32)
+			ti_mmchs_write_4(sc, MMCHS_CON, newval32);
+	}
+#endif
 	val32 = RD4(sc, off & ~3);
 	val32 &= ~(0xff << (off & 3) * 8);
 	val32 |= (val << (off & 3) * 8);
@@ -658,8 +683,11 @@ ti_sdhci_attach(device_t dev)
 	bus_generic_probe(dev);
 	bus_generic_attach(dev);
 
+#ifdef MMCCAM
+	sdhci_cam_start_slot(&sc->slot);
+#else
 	sdhci_start_slot(&sc->slot);
-
+#endif
 	return (0);
 
 fail:
@@ -730,4 +758,7 @@ static driver_t ti_sdhci_driver = {
 DRIVER_MODULE(sdhci_ti, simplebus, ti_sdhci_driver, ti_sdhci_devclass, NULL,
     NULL);
 MODULE_DEPEND(sdhci_ti, sdhci, 1, 1, 1);
+
+#ifndef MMCCAM
 MMC_DECLARE_BRIDGE(sdhci_ti);
+#endif

sys/cam/cam_ccb.h

@@ -42,6 +42,7 @@
 #include <cam/scsi/scsi_all.h>
 #include <cam/ata/ata_all.h>
 #include <cam/nvme/nvme_all.h>
+#include <cam/mmc/mmc_all.h>
 
 /* General allocation length definitions for CCB structures */
 #define	IOCDBLEN	CAM_MAX_CDBLEN	/* Space for CDB bytes/pointer */
@@ -208,10 +209,10 @@ typedef enum {
 	XPT_NVME_IO		= 0x1c | XPT_FC_DEV_QUEUED,
 				/* Execiute the requestred NVMe I/O operation */
 
-	XPT_MMCSD_IO		= 0x1d | XPT_FC_DEV_QUEUED,
+	XPT_MMC_IO		= 0x1d | XPT_FC_DEV_QUEUED,
 				/* Placeholder for MMC / SD / SDIO I/O stuff */
 
-	XPT_SCAN_TGT		= 0x1E | XPT_FC_QUEUED | XPT_FC_USER_CCB
+	XPT_SCAN_TGT		= 0x1e | XPT_FC_QUEUED | XPT_FC_USER_CCB
 				       | XPT_FC_XPT_ONLY,
 				/* Scan Target */
 
@@ -267,6 +268,7 @@ typedef enum {
 	PROTO_SATAPM,	/* SATA Port Multiplier */
 	PROTO_SEMB,	/* SATA Enclosure Management Bridge */
 	PROTO_NVME,	/* NVME */
+	PROTO_MMCSD,	/* MMC, SD, SDIO */
 } cam_proto;
 
 typedef enum {
@@ -283,6 +285,7 @@ typedef enum {
 	XPORT_ISCSI,	/* iSCSI */
 	XPORT_SRP,	/* SCSI RDMA Protocol */
 	XPORT_NVME,	/* NVMe over PCIe */
+	XPORT_MMCSD,	/* MMC, SD, SDIO card */
 } cam_xport;
 
 #define XPORT_IS_NVME(t)	((t) == XPORT_NVME)
@@ -498,6 +501,7 @@ struct device_match_result {
 	cam_proto			protocol;
 	struct scsi_inquiry_data	inq_data;
 	struct ata_params		ident_data;
+        struct mmc_params               mmc_ident_data;
 	dev_result_flags		flags;
 };
 
@@ -773,6 +777,16 @@ struct ccb_ataio {
 	uint32_t   unused;
 };
 
+/*
+ * MMC I/O Request CCB used for the XPT_MMC_IO function code.
+ */
+struct ccb_mmcio {
+	struct	   ccb_hdr ccb_h;
+	union	   ccb *next_ccb;	/* Ptr for next CCB for action */
+	struct mmc_command cmd;
+        struct mmc_command stop;
+};
+
 struct ccb_accept_tio {
 	struct	   ccb_hdr ccb_h;
 	cdb_t	   cdb_io;		/* Union for CDB bytes/pointer */
@@ -1005,7 +1019,28 @@ struct ccb_trans_settings_nvme
 	u_int		max_xfer;	/* Max transfer size (0 -> unlimited */
 	u_int		caps;
 };
-
+
+#include <cam/mmc/mmc_bus.h>
+struct ccb_trans_settings_mmc {
+	struct mmc_ios ios;
+#define MMC_CLK		(1 << 1)
+#define MMC_VDD		(1 << 2)
+#define MMC_CS		(1 << 3)
+#define MMC_BW		(1 << 4)
+#define MMC_PM		(1 << 5)
+#define MMC_BT		(1 << 6)
+#define MMC_BM		(1 << 7)
+	uint32_t ios_valid;
+/* The folowing is used only for GET_TRAN_SETTINGS */
+	uint32_t	host_ocr;
+	int host_f_min;
+	int host_f_max;
+#define MMC_CAP_4_BIT_DATA	(1 << 0) /* Can do 4-bit data transfers */
+#define MMC_CAP_8_BIT_DATA	(1 << 1) /* Can do 8-bit data transfers */
+#define MMC_CAP_HSPEED		(1 << 2) /* Can do High Speed transfers */
+	uint32_t host_caps;
+};
+
 /* Get/Set transfer rate/width/disconnection/tag queueing settings */
 struct ccb_trans_settings {
 	struct	  ccb_hdr ccb_h;
@@ -1019,6 +1054,7 @@ struct ccb_trans_settings {
 		struct ccb_trans_settings_ata ata;
 		struct ccb_trans_settings_scsi scsi;
 		struct ccb_trans_settings_nvme nvme;
+		struct ccb_trans_settings_mmc mmc;
 	} proto_specific;
 	union {
 		u_int  valid;	/* Which fields to honor */
@@ -1284,6 +1320,7 @@ union ccb {
 	struct	ccb_dev_advinfo		cdai;
 	struct	ccb_async		casync;
 	struct	ccb_nvmeio		nvmeio;
+	struct	ccb_mmcio		mmcio;
 };
 
 #define CCB_CLEAR_ALL_EXCEPT_HDR(ccbp)			\
@@ -1326,6 +1363,13 @@ cam_fill_smpio(struct ccb_smpio *smpio, uint32_t retries,
 	       uint8_t *smp_response, int smp_response_len,
 	       uint32_t timeout);
 
+static __inline void
+cam_fill_mmcio(struct ccb_mmcio *mmcio, uint32_t retries,
+	       void (*cbfcnp)(struct cam_periph *, union ccb *), uint32_t flags,
+               uint32_t mmc_opcode, uint32_t mmc_arg, uint32_t mmc_flags,
+	       struct mmc_data *mmc_d,
+	       uint32_t timeout);
+
 static __inline void
 cam_fill_csio(struct ccb_scsiio *csio, u_int32_t retries,
 	      void (*cbfcnp)(struct cam_periph *, union ccb *),
@@ -1414,6 +1458,34 @@ cam_fill_smpio(struct ccb_smpio *smpio, uint32_t retries,
 	smpio->smp_response_len = smp_response_len;
 }
 
+static __inline void
+cam_fill_mmcio(struct ccb_mmcio *mmcio, uint32_t retries,
+	       void (*cbfcnp)(struct cam_periph *, union ccb *), uint32_t flags,
+	       uint32_t mmc_opcode, uint32_t mmc_arg, uint32_t mmc_flags,
+	       struct mmc_data *mmc_d,
+	       uint32_t timeout)
+{
+	mmcio->ccb_h.func_code = XPT_MMC_IO;
+	mmcio->ccb_h.flags = flags;
+	mmcio->ccb_h.retry_count = retries;
+	mmcio->ccb_h.cbfcnp = cbfcnp;
+	mmcio->ccb_h.timeout = timeout;
+	mmcio->cmd.opcode = mmc_opcode;
+	mmcio->cmd.arg = mmc_arg;
+	mmcio->cmd.flags = mmc_flags;
+	mmcio->stop.opcode = 0;
+	mmcio->stop.arg = 0;
+	mmcio->stop.flags = 0;
+	if (mmc_d != NULL) {
+		mmcio->cmd.data = mmc_d;
+	} else
+		mmcio->cmd.data = NULL;
+	mmcio->cmd.resp[0] = 0;
+	mmcio->cmd.resp[1] = 0;
+	mmcio->cmd.resp[2] = 0;
+	mmcio->cmd.resp[3] = 0;
+}
+
 static __inline void
 cam_set_ccbstatus(union ccb *ccb, cam_status status)
 {

sys/cam/cam_periph.c

@@ -827,6 +827,18 @@ cam_periph_mapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo,
 		dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK;
 		numbufs = 1;
 		break;
+	case XPT_MMC_IO:
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE)
+			return(0);
+		/* Two mappings: one for cmd->data and one for cmd->data->data */
+		data_ptrs[0] = (unsigned char **)&ccb->mmcio.cmd.data;
+		lengths[0] = sizeof(struct mmc_data *);
+		dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK;
+		data_ptrs[1] = (unsigned char **)&ccb->mmcio.cmd.data->data;
+		lengths[1] = ccb->mmcio.cmd.data->len;
+		dirs[1] = ccb->ccb_h.flags & CAM_DIR_MASK;
+		numbufs = 2;
+		break;
 	case XPT_SMP_IO:
 		data_ptrs[0] = &ccb->smpio.smp_request;
 		lengths[0] = ccb->smpio.smp_request_len;