From: Kieran Mansley <kmansley@solarflare.com>
Subject: enable access to Falcon's external SRAM
References: bnc#489105
Patch-mainline: n/a

Include ability to reference external SRAM on Solarflare Falcon NICs to
allow event queues to be accessed by virtualised guests.  

Acked-by: jbeulich@novell.com

--- head-2009-07-28.orig/drivers/net/sfc/falcon.c	2009-07-28 10:05:40.000000000 +0200
+++ head-2009-07-28/drivers/net/sfc/falcon.c	2009-07-28 10:06:53.000000000 +0200
@@ -36,6 +36,9 @@
 
 /**
  * struct falcon_nic_data - Falcon NIC state
+ * @sram_cfg: SRAM configuration value
+ * @tx_dc_base: Base address in SRAM of TX queue descriptor caches
+ * @rx_dc_base: Base address in SRAM of RX queue descriptor caches
  * @next_buffer_table: First available buffer table id
  * @resources: Resource information for driverlink client
  * @pci_dev2: The secondary PCI device if present
@@ -44,6 +47,9 @@
  * @int_error_expire: Time at which error count will be expired
  */
 struct falcon_nic_data {
+	int sram_cfg;
+	unsigned tx_dc_base;
+	unsigned rx_dc_base;
 #ifndef CONFIG_SFC_DRIVERLINK
 	unsigned next_buffer_table;
 #else
@@ -74,11 +80,11 @@ static int disable_dma_stats;
  */
 #define TX_DC_ENTRIES 16
 #define TX_DC_ENTRIES_ORDER 0
-#define TX_DC_BASE 0x130000
+#define TX_DC_INTERNAL_BASE 0x130000
 
 #define RX_DC_ENTRIES 64
 #define RX_DC_ENTRIES_ORDER 2
-#define RX_DC_BASE 0x100000
+#define RX_DC_INTERNAL_BASE 0x100000
 
 static const unsigned int
 /* "Large" EEPROM device: Atmel AT25640 or similar
@@ -468,9 +474,17 @@ void falcon_push_buffers(struct efx_tx_q
 int falcon_probe_tx(struct efx_tx_queue *tx_queue)
 {
 	struct efx_nic *efx = tx_queue->efx;
-	return falcon_alloc_special_buffer(efx, &tx_queue->txd,
-					   FALCON_TXD_RING_SIZE *
-					   sizeof(efx_qword_t));
+	int rc = falcon_alloc_special_buffer(efx, &tx_queue->txd,
+					    FALCON_TXD_RING_SIZE *
+					    sizeof(efx_qword_t));
+#ifdef CONFIG_SFC_DRIVERLINK
+	if (rc == 0) {
+		struct falcon_nic_data *nic_data = efx->nic_data;
+		nic_data->resources.txq_min = max(nic_data->resources.txq_min,
+						  (unsigned)tx_queue->queue + 1);
+	}
+#endif
+	return rc;
 }
 
 void falcon_init_tx(struct efx_tx_queue *tx_queue)
@@ -610,9 +624,17 @@ void falcon_notify_rx_desc(struct efx_rx
 int falcon_probe_rx(struct efx_rx_queue *rx_queue)
 {
 	struct efx_nic *efx = rx_queue->efx;
-	return falcon_alloc_special_buffer(efx, &rx_queue->rxd,
-					   FALCON_RXD_RING_SIZE *
-					   sizeof(efx_qword_t));
+	int rc = falcon_alloc_special_buffer(efx, &rx_queue->rxd,
+					    FALCON_RXD_RING_SIZE *
+					    sizeof(efx_qword_t));
+#ifdef CONFIG_SFC_DRIVERLINK
+	if (rc == 0) {
+		struct falcon_nic_data *nic_data = efx->nic_data;
+		nic_data->resources.rxq_min = max(nic_data->resources.rxq_min,
+						  (unsigned)rx_queue->queue + 1);
+	}
+#endif
+	return rc;
 }
 
 void falcon_init_rx(struct efx_rx_queue *rx_queue)
@@ -1120,9 +1142,18 @@ int falcon_probe_eventq(struct efx_chann
 {
 	struct efx_nic *efx = channel->efx;
 	unsigned int evq_size;
+	int rc;
 
 	evq_size = FALCON_EVQ_SIZE * sizeof(efx_qword_t);
-	return falcon_alloc_special_buffer(efx, &channel->eventq, evq_size);
+	rc = falcon_alloc_special_buffer(efx, &channel->eventq, evq_size);
+#ifdef CONFIG_SFC_DRIVERLINK
+	if (rc == 0) {
+		struct falcon_nic_data *nic_data = efx->nic_data;
+		nic_data->resources.evq_int_min = max(nic_data->resources.evq_int_min,
+						      (unsigned)channel->channel + 1);
+	}
+#endif
+	return rc;
 }
 
 void falcon_init_eventq(struct efx_channel *channel)
@@ -2618,19 +2649,22 @@ fail5:
  */
 static int falcon_reset_sram(struct efx_nic *efx)
 {
+	struct falcon_nic_data *nic_data = efx->nic_data;
 	efx_oword_t srm_cfg_reg_ker, gpio_cfg_reg_ker;
-	int count;
+	int count, onchip, sram_cfg_val;
 
 	/* Set the SRAM wake/sleep GPIO appropriately. */
+	onchip = (nic_data->sram_cfg == SRM_NB_BSZ_ONCHIP_ONLY);
 	falcon_read(efx, &gpio_cfg_reg_ker, GPIO_CTL_REG_KER);
 	EFX_SET_OWORD_FIELD(gpio_cfg_reg_ker, GPIO1_OEN, 1);
-	EFX_SET_OWORD_FIELD(gpio_cfg_reg_ker, GPIO1_OUT, 1);
+	EFX_SET_OWORD_FIELD(gpio_cfg_reg_ker, GPIO1_OUT, onchip);
 	falcon_write(efx, &gpio_cfg_reg_ker, GPIO_CTL_REG_KER);
 
 	/* Initiate SRAM reset */
+	sram_cfg_val = onchip ? 0 : nic_data->sram_cfg;
 	EFX_POPULATE_OWORD_2(srm_cfg_reg_ker,
 			     SRAM_OOB_BT_INIT_EN, 1,
-			     SRM_NUM_BANKS_AND_BANK_SIZE, 0);
+			     SRM_NUM_BANKS_AND_BANK_SIZE, sram_cfg_val);
 	falcon_write(efx, &srm_cfg_reg_ker, SRM_CFG_REG_KER);
 
 	/* Wait for SRAM reset to complete */
@@ -2702,8 +2736,10 @@ static void falcon_remove_spi_devices(st
 /* Extract non-volatile configuration */
 static int falcon_probe_nvconfig(struct efx_nic *efx)
 {
+	struct falcon_nic_data *nic_data = efx->nic_data;
 	struct falcon_nvconfig *nvconfig;
 	int board_rev;
+	bool onchip_sram;
 	int rc;
 
 	nvconfig = kmalloc(sizeof(*nvconfig), GFP_KERNEL);
@@ -2716,6 +2752,7 @@ static int falcon_probe_nvconfig(struct 
 		efx->phy_type = PHY_TYPE_NONE;
 		efx->mdio.prtad = MDIO_PRTAD_NONE;
 		board_rev = 0;
+		onchip_sram = true;
 		rc = 0;
 	} else if (rc) {
 		goto fail1;
@@ -2726,6 +2763,13 @@ static int falcon_probe_nvconfig(struct 
 		efx->phy_type = v2->port0_phy_type;
 		efx->mdio.prtad = v2->port0_phy_addr;
 		board_rev = le16_to_cpu(v2->board_revision);
+#ifdef CONFIG_SFC_DRIVERLINK
+		onchip_sram = EFX_OWORD_FIELD(nvconfig->nic_stat_reg,
+					      ONCHIP_SRAM);
+#else
+		/* We have no use for external SRAM */
+		onchip_sram = true;
+#endif
 
 		if (le16_to_cpu(nvconfig->board_struct_ver) >= 3) {
 			__le32 fl = v3->spi_device_type[EE_SPI_FLASH];
@@ -2750,6 +2794,21 @@ static int falcon_probe_nvconfig(struct 
 
 	efx_set_board_info(efx, board_rev);
 
+	/* Read the SRAM configuration.  The register is initialised
+	 * automatically but might may been reset since boot.
+	 */
+	if (onchip_sram) {
+		nic_data->sram_cfg = SRM_NB_BSZ_ONCHIP_ONLY;
+	} else {
+		nic_data->sram_cfg =
+			EFX_OWORD_FIELD(nvconfig->srm_cfg_reg,
+					SRM_NUM_BANKS_AND_BANK_SIZE);
+		WARN_ON(nic_data->sram_cfg == SRM_NB_BSZ_RESERVED);
+		/* Replace invalid setting with the smallest defaults */
+		if (nic_data->sram_cfg == SRM_NB_BSZ_DEFAULT)
+			nic_data->sram_cfg = SRM_NB_BSZ_1BANKS_2M;
+	}
+
 	kfree(nvconfig);
 	return 0;
 
@@ -2765,9 +2824,9 @@ static int falcon_probe_nvconfig(struct 
  * should live. */
 static int falcon_dimension_resources(struct efx_nic *efx)
 {
+	struct falcon_nic_data *nic_data = efx->nic_data;
 #ifdef CONFIG_SFC_DRIVERLINK
 	unsigned internal_dcs_entries;
-	struct falcon_nic_data *nic_data = efx->nic_data;
 	struct efx_dl_falcon_resources *res = &nic_data->resources;
 
 	/* Fill out the driverlink resource list */
@@ -2800,16 +2859,64 @@ static int falcon_dimension_resources(st
 		break;
 	}
 
-	/* Internal SRAM only for now */
-	res->rxq_lim = internal_dcs_entries / RX_DC_ENTRIES;
-	res->txq_lim = internal_dcs_entries / TX_DC_ENTRIES;
-	res->buffer_table_lim = 8192;
+	if (nic_data->sram_cfg == SRM_NB_BSZ_ONCHIP_ONLY) {
+		res->rxq_lim = internal_dcs_entries / RX_DC_ENTRIES;
+		res->txq_lim = internal_dcs_entries / TX_DC_ENTRIES;
+		res->buffer_table_lim = 8192;
+		nic_data->tx_dc_base = TX_DC_INTERNAL_BASE;
+		nic_data->rx_dc_base = RX_DC_INTERNAL_BASE;
+	} else {
+		unsigned sram_bytes, vnic_bytes, max_vnics, n_vnics, dcs;
+
+		/* Determine how much SRAM we have to play with.  We have
+		 * to fit buffer table and descriptor caches in.
+		 */
+		switch (nic_data->sram_cfg) {
+		case SRM_NB_BSZ_1BANKS_2M:
+		default:
+			sram_bytes = 2 * 1024 * 1024;
+			break;
+		case SRM_NB_BSZ_1BANKS_4M:
+		case SRM_NB_BSZ_2BANKS_4M:
+			sram_bytes = 4 * 1024 * 1024;
+			break;
+		case SRM_NB_BSZ_1BANKS_8M:
+		case SRM_NB_BSZ_2BANKS_8M:
+			sram_bytes = 8 * 1024 * 1024;
+			break;
+		case SRM_NB_BSZ_2BANKS_16M:
+			sram_bytes = 16 * 1024 * 1024;
+			break;
+		}
+		/* For each VNIC allow at least 512 buffer table entries
+		 * and descriptor cache for an rxq and txq.  Buffer table
+		 * space for evqs and dmaqs is relatively trivial, so not
+		 * considered in this calculation.
+		 */
+		vnic_bytes = 512 * 8 + RX_DC_ENTRIES * 8 + TX_DC_ENTRIES * 8;
+		max_vnics = sram_bytes / vnic_bytes;
+		for (n_vnics = 1; n_vnics < res->evq_timer_min + max_vnics;)
+			n_vnics *= 2;
+		res->rxq_lim = n_vnics;
+		res->txq_lim = n_vnics;
+
+		dcs = n_vnics * TX_DC_ENTRIES * 8;
+		nic_data->tx_dc_base = sram_bytes - dcs;
+		dcs = n_vnics * RX_DC_ENTRIES * 8;
+		nic_data->rx_dc_base = nic_data->tx_dc_base - dcs;
+		res->buffer_table_lim = nic_data->rx_dc_base / 8;
+	}
 
 	if (FALCON_IS_DUAL_FUNC(efx))
 		res->flags |= EFX_DL_FALCON_DUAL_FUNC;
 
 	if (EFX_INT_MODE_USE_MSI(efx))
 		res->flags |= EFX_DL_FALCON_USE_MSI;
+#else
+	/* We ignore external SRAM */
+	EFX_BUG_ON_PARANOID(nic_data->sram_cfg != SRM_NB_BSZ_ONCHIP_ONLY);
+	nic_data->tx_dc_base = TX_DC_INTERNAL_BASE;
+	nic_data->rx_dc_base = RX_DC_INTERNAL_BASE;
 #endif
 
 	return 0;
@@ -2998,13 +3105,15 @@ int falcon_probe_nic(struct efx_nic *efx
  */
 int falcon_init_nic(struct efx_nic *efx)
 {
+	struct falcon_nic_data *nic_data = efx->nic_data;
 	efx_oword_t temp;
 	unsigned thresh;
 	int rc;
 
-	/* Use on-chip SRAM */
+	/* Use on-chip SRAM if wanted. */
 	falcon_read(efx, &temp, NIC_STAT_REG);
-	EFX_SET_OWORD_FIELD(temp, ONCHIP_SRAM, 1);
+	EFX_SET_OWORD_FIELD(temp, ONCHIP_SRAM,
+			    nic_data->sram_cfg == SRM_NB_BSZ_ONCHIP_ONLY);
 	falcon_write(efx, &temp, NIC_STAT_REG);
 
 	/* Set the source of the GMAC clock */
@@ -3023,9 +3132,9 @@ int falcon_init_nic(struct efx_nic *efx)
 		return rc;
 
 	/* Set positions of descriptor caches in SRAM. */
-	EFX_POPULATE_OWORD_1(temp, SRM_TX_DC_BASE_ADR, TX_DC_BASE / 8);
+	EFX_POPULATE_OWORD_1(temp, SRM_TX_DC_BASE_ADR, nic_data->tx_dc_base / 8);
 	falcon_write(efx, &temp, SRM_TX_DC_CFG_REG_KER);
-	EFX_POPULATE_OWORD_1(temp, SRM_RX_DC_BASE_ADR, RX_DC_BASE / 8);
+	EFX_POPULATE_OWORD_1(temp, SRM_RX_DC_BASE_ADR, nic_data->rx_dc_base / 8);
 	falcon_write(efx, &temp, SRM_RX_DC_CFG_REG_KER);
 
 	/* Set TX descriptor cache size. */
