From 004b61825ecaeeb1aaa57faf2c031a473bc91d44 Mon Sep 17 00:00:00 2001
From: Alex Forencich <alex@alexforencich.com>
Date: Fri, 23 Jun 2023 18:18:34 -0700
Subject: [PATCH] Handle CRS during enumeration

Signed-off-by: Alex Forencich <alex@alexforencich.com>
---
 cocotbext/pcie/core/pci.py      | 27 +++++++++++++++++++++++++++
 tests/pcie_us/test_pcie_us.py   |  7 +++++++
 tests/pcie_usp/test_pcie_usp.py |  7 +++++++
 3 files changed, 41 insertions(+)

diff --git a/cocotbext/pcie/core/pci.py b/cocotbext/pcie/core/pci.py
index 882ed38..37b871a 100644
--- a/cocotbext/pcie/core/pci.py
+++ b/cocotbext/pcie/core/pci.py
@@ -24,6 +24,8 @@
 
 import struct
 
+from cocotb.triggers import Timer
+
 from .caps import PciCapId, PciExtCapId
 from .utils import PcieId, align
 
@@ -119,6 +121,27 @@ def only_one_child(self):
             return True
         return False
 
+    async def wait_crs(self, dev_id, timeout=1000, timeout_unit='ns'):
+        delay = 10
+        val = 0xffff0001
+        while val == 0xffff0001:
+            if delay > 10000:
+                self.rc.log.warning("pci %s: not ready after %d us; giving up", dev_id, delay)
+                return False
+
+            if delay > 1000:
+                self.rc.log.info("pci %s: not ready after %d us; waiting", dev_id, delay)
+
+            await Timer(delay, 'us')
+            delay *= 2
+
+            val = await self.rc.config_read_dword(dev_id, 0x000, 'little', timeout, timeout_unit)
+
+        if delay > 1000:
+            self.rc.log.info("pci %s: ready after %d us", dev_id, delay)
+
+        return True
+
     async def scan(self, available_buses=0, timeout=1000, timeout_unit='ns'):
         first_bus = self.bus_num
         last_bus = first_bus
@@ -138,6 +161,10 @@ async def scan(self, available_buses=0, timeout=1000, timeout_unit='ns'):
             if val in {0, 0xffffffff, 0xffff0000, 0x0000ffff}:
                 continue
 
+            if val == 0xffff0001:
+                if not await self.wait_crs(dev_id, timeout, timeout_unit):
+                    continue
+
             # valid vendor ID
             self.rc.log.info("Found device at %s", dev_id)
 
diff --git a/tests/pcie_us/test_pcie_us.py b/tests/pcie_us/test_pcie_us.py
index bbbcd24..fada27a 100644
--- a/tests/pcie_us/test_pcie_us.py
+++ b/tests/pcie_us/test_pcie_us.py
@@ -965,6 +965,13 @@ async def run_test_crs(dut, idle_inserter=None, backpressure_inserter=None):
     await FallingEdge(dut.user_reset)
     await Timer(100, 'ns')
 
+    async def toggle_config_space_enable(dut):
+        dut.cfg_config_space_enable.setimmediatevalue(0)
+        await Timer(100, 'us')
+        dut.cfg_config_space_enable.setimmediatevalue(1)
+
+    cocotb.start_soon(toggle_config_space_enable(dut))
+
     await tb.rc.enumerate()
 
     dev = tb.rc.find_device(tb.dev.functions[0].pcie_id)
diff --git a/tests/pcie_usp/test_pcie_usp.py b/tests/pcie_usp/test_pcie_usp.py
index 9de3feb..1c2e9ff 100644
--- a/tests/pcie_usp/test_pcie_usp.py
+++ b/tests/pcie_usp/test_pcie_usp.py
@@ -1000,6 +1000,13 @@ async def run_test_crs(dut, idle_inserter=None, backpressure_inserter=None):
     await FallingEdge(dut.user_reset)
     await Timer(100, 'ns')
 
+    async def toggle_config_space_enable(dut):
+        dut.cfg_config_space_enable.setimmediatevalue(0)
+        await Timer(100, 'us')
+        dut.cfg_config_space_enable.setimmediatevalue(1)
+
+    cocotb.start_soon(toggle_config_space_enable(dut))
+
     await tb.rc.enumerate()
 
     dev = tb.rc.find_device(tb.dev.functions[0].pcie_id)