QSFP x32 FPGA driver, server multiple device from fpga-server (#626)

This diff implements the following: - Allow a `fpga-server` task to server requests for multiple FPGA devices - Add a driver for hardware found on the QSFP x32 board, where multiple ECP5 devices are controlled using a mux'd SPI link and an I2C GPIO expander
oxidecomputer · Jun 29, 2022 · 4c2b922 · 4c2b922
1 parent 68cff7d
commit 4c2b922
Showing 12 changed files with 728 additions and 108 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/app/sidecar/app.toml b/app/sidecar/app.toml
@@ -59,6 +59,20 @@ max-sizes = {flash = 2048, ram = 1024}
 uses = ["rcc", "gpios1", "gpios2", "gpios3"]
 start = true
 
+[tasks.spi1_driver]
+name = "drv-stm32h7-spi-server"
+priority = 2
+max-sizes = {flash = 16384, ram = 2048}
+features = ["h753", "spi1"]
+uses = ["spi1"]
+start = true
+interrupts = {"spi1.irq" = 1}
+stacksize = 880
+task-slots = ["sys"]
+
+[tasks.spi1_driver.config.spi]
+global_config = "spi1"
+
 [tasks.spi2_driver]
 name = "drv-stm32h7-spi-server"
 priority = 2
@@ -180,12 +194,22 @@ start = true
 
 [tasks.ecp5_mainboard]
 name = "drv-fpga-server"
+features = ["mainboard"]
 priority = 3
 max-sizes = {flash = 32768, ram = 4096}
 stacksize = 2048
 start = true
 task-slots = ["sys", {spi_driver = "spi5_driver"}]
 
+[tasks.ecp5_front_io]
+name = "drv-fpga-server"
+features = ["front_io"]
+priority = 3
+max-sizes = {flash = 32768, ram = 4096}
+stacksize = 2048
+start = true
+task-slots = ["sys", "i2c_driver", {spi_driver = "spi1_driver"}]
+
 [tasks.sequencer]
 name = "drv-sidecar-seq-server"
 features = []
@@ -266,7 +290,7 @@ controller = 2
 # I2C_FRONT_IO0_SDA
 #
 [config.i2c.controllers.ports.F]
-name = "frontio"
+name = "front_io"
 description = "Front I/O Board"
 pins = [ { pins = [ 0, 1 ], af = 4 } ]
 
@@ -613,18 +637,50 @@ refdes = "U65"
 # device = "at24csw080"
 # description = ""
 
-# [[config.i2c.devices]]
-# bus = "south2"
-# address = 0b1010_0**
-# device = "at24csw080"
-# description = ""
+[[config.i2c.devices]]
+bus = "south2"
+address = 0b1010_000
+device = "at24csw080"
+description = "Mainboard FRUID"
+refdes = "U91"
 
 # [[config.i2c.devices]]
 # device = "tf2"
 # bus = "ECP5->S?"
 # address = 0b1011_011
 # description = ""
 
+[[config.i2c.devices]]
+bus = "front_io"
+address = 0b1010_000
+device = "at24csw080"
+description = "Front IO board FRUID"
+removable = true
+
+[[config.i2c.devices]]
+bus = "front_io"
+address = 0b1110_011
+device = "pca9538"
+description = "Front IO GPIO expander"
+removable = true
+
+[config.spi.spi1]
+controller = 1
+
+[config.spi.spi1.mux_options.port_adg]
+outputs = [
+    {port = "A", pins = [5], af = 5}, # FRONT_IO_SCK
+    {port = "D", pins = [7], af = 5}, # FRONT_IO_COPI
+]
+input = {port = "G", pin = 9, af = 5} # FRONT_IO_CIPO
+
+[config.spi.spi1.devices.ecp5_front_io_fpga]
+mux = "port_adg"
+cs = [{port = "G", pin = 10}] # FRONT_IO_CS0
+
+[config.spi.spi1.devices.ecp5_front_io_user_design]
+mux = "port_adg"
+cs = [{port = "A", pin = 15}] # FRONT_IO_CS1
 
 [config.spi.spi2]
 controller = 2

diff --git a/drv/fpga-api/src/lib.rs b/drv/fpga-api/src/lib.rs
@@ -19,6 +19,7 @@ pub enum FpgaError {
     InvalidState,
     InvalidValue,
     PortDisabled,
+    BadDevice,
     NotLocked,
     AlreadyLocked,
 }
@@ -33,8 +34,9 @@ impl From<FpgaError> for u16 {
             FpgaError::InvalidState => 0x0300,
             FpgaError::InvalidValue => 0x0301,
             FpgaError::PortDisabled => 0x0400,
-            FpgaError::NotLocked => 0x0500,
-            FpgaError::AlreadyLocked => 0x0501,
+            FpgaError::BadDevice => 0x0500,
+            FpgaError::NotLocked => 0x0501,
+            FpgaError::AlreadyLocked => 0x0502,
         }
     }
 }
@@ -62,8 +64,9 @@ impl core::convert::TryFrom<u16> for FpgaError {
                 0x0300 => Ok(FpgaError::InvalidState),
                 0x0301 => Ok(FpgaError::InvalidValue),
                 0x0400 => Ok(FpgaError::PortDisabled),
-                0x0500 => Ok(FpgaError::NotLocked),
-                0x0501 => Ok(FpgaError::AlreadyLocked),
+                0x0500 => Ok(FpgaError::BadDevice),
+                0x0501 => Ok(FpgaError::NotLocked),
+                0x0502 => Ok(FpgaError::AlreadyLocked),
                 _ => Err(()),
             },
         }
@@ -122,13 +125,16 @@ impl From<bool> for WriteOp {
     }
 }
 
-pub struct FpgaLock(idl::Fpga);
+pub struct FpgaLock {
+    server: idl::Fpga,
+    device_index: u8,
+}
 
 impl Deref for FpgaLock {
     type Target = idl::Fpga;
 
     fn deref(&self) -> &Self::Target {
-        &self.0
+        &self.server
     }
 }
 
@@ -140,45 +146,55 @@ impl Drop for FpgaLock {
     }
 }
 
-pub struct Fpga(idl::Fpga);
+pub struct Fpga {
+    server: idl::Fpga,
+    device_index: u8,
+}
 
 impl Fpga {
-    pub fn new(task_id: userlib::TaskId) -> Self {
-        Self(idl::Fpga::from(task_id))
+    pub fn new(task_id: userlib::TaskId, device_index: u8) -> Self {
+        Self {
+            server: idl::Fpga::from(task_id),
+            device_index,
+        }
     }
 
     pub fn enabled(&self) -> Result<bool, FpgaError> {
-        self.0.device_enabled()
+        self.server.device_enabled(self.device_index)
     }
 
     pub fn set_enabled(&mut self, enabled: bool) -> Result<(), FpgaError> {
-        self.0.set_device_enabled(enabled)
+        self.server.set_device_enabled(self.device_index, enabled)
     }
 
     pub fn reset(&mut self) -> Result<(), FpgaError> {
-        self.0.reset_device()
+        self.server.reset_device(self.device_index)
     }
 
     pub fn state(&self) -> Result<DeviceState, FpgaError> {
-        self.0.device_state()
+        self.server.device_state(self.device_index)
     }
 
     pub fn id(&self) -> Result<u32, FpgaError> {
-        self.0.device_id()
+        self.server.device_id(self.device_index)
     }
 
     pub fn start_bitstream_load(
         &mut self,
         bitstream_type: BitstreamType,
     ) -> Result<Bitstream, FpgaError> {
         let lock = self.lock()?;
-        lock.0.start_bitstream_load(bitstream_type)?;
+        lock.server
+            .start_bitstream_load(lock.device_index, bitstream_type)?;
         Ok(Bitstream(lock))
     }
 
     pub fn lock(&mut self) -> Result<FpgaLock, FpgaError> {
-        self.0.lock()?;
-        Ok(FpgaLock(self.0.clone()))
+        self.server.lock(self.device_index)?;
+        Ok(FpgaLock {
+            server: self.server.clone(),
+            device_index: self.device_index,
+        })
     }
 }
 
@@ -194,31 +210,42 @@ impl Bitstream {
     }
 }
 
-pub struct FpgaUserDesign(idl::Fpga);
+pub struct FpgaUserDesign {
+    server: idl::Fpga,
+    device_index: u8,
+}
 
 impl FpgaUserDesign {
-    pub fn new(task_id: userlib::TaskId) -> Self {
-        Self(idl::Fpga::from(task_id))
+    pub fn new(task_id: userlib::TaskId, device_index: u8) -> Self {
+        Self {
+            server: idl::Fpga::from(task_id),
+            device_index,
+        }
     }
 
     pub fn enabled(&self) -> Result<bool, FpgaError> {
-        self.0.user_design_enabled()
+        self.server.user_design_enabled(self.device_index)
     }
 
     pub fn set_enabled(&mut self, enabled: bool) -> Result<(), FpgaError> {
-        self.0.set_user_design_enabled(enabled)
+        self.server
+            .set_user_design_enabled(self.device_index, enabled)
     }
 
     pub fn reset(&mut self) -> Result<(), FpgaError> {
-        self.0.reset_user_design()
+        self.server.reset_user_design(self.device_index)
     }
 
     pub fn read<T>(&self, addr: impl Into<u16>) -> Result<T, FpgaError>
     where
         T: AsBytes + Default + FromBytes,
     {
         let mut v = T::default();
-        self.0.user_design_read(addr.into(), v.as_bytes_mut())?;
+        self.server.user_design_read(
+            self.device_index,
+            addr.into(),
+            v.as_bytes_mut(),
+        )?;
         Ok(v)
     }
 
@@ -231,7 +258,12 @@ impl FpgaUserDesign {
     where
         T: AsBytes + FromBytes,
     {
-        self.0.user_design_write(op, addr.into(), value.as_bytes())
+        self.server.user_design_write(
+            self.device_index,
+            op,
+            addr.into(),
+            value.as_bytes(),
+        )
     }
 }
 

diff --git a/drv/fpga-devices/Cargo.toml b/drv/fpga-devices/Cargo.toml
@@ -6,10 +6,12 @@ edition = "2021"
 [dependencies]
 userlib = {path = "../../sys/userlib"}
 ringbuf = {path = "../../lib/ringbuf"}
+drv-i2c-api = {path = "../../drv/i2c-api"}
+drv-i2c-devices = {path = "../../drv/i2c-devices"}
+drv-fpga-api = {path = "../../drv/fpga-api"}
 drv-spi-api = {path = "../../drv/spi-api"}
 drv-stm32xx-sys-api = {path = "../stm32xx-sys-api"}
-drv-fpga-api = {path = "../../drv/fpga-api"}
-num-traits = { version = "0.2.12", default-features = false }
+num-traits = {version = "0.2.12", default-features = false}
 cfg-if = "1"
 bitfield = "0.13"
 zerocopy = "0.6.1"