1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
From patchwork Wed Jan 24 08:02:33 2018
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-Patchwork-Submitter: geoff@hostfission.com
X-Patchwork-Id: 10181903
Return-Path: <linux-pci-owner@kernel.org>
Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org
[172.30.200.125])
by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id
CA59260233 for <patchwork-linux-pci@patchwork.kernel.org>;
Wed, 24 Jan 2018 08:11:47 +0000 (UTC)
Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1])
by mail.wl.linuxfoundation.org (Postfix) with ESMTP id AF181203B9
for <patchwork-linux-pci@patchwork.kernel.org>;
Wed, 24 Jan 2018 08:11:47 +0000 (UTC)
Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486)
id A3A3F28528; Wed, 24 Jan 2018 08:11:47 +0000 (UTC)
X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on
pdx-wl-mail.web.codeaurora.org
X-Spam-Level:
X-Spam-Status: No, score=-7.0 required=2.0 tests=BAYES_00,DKIM_SIGNED,
DKIM_VALID, DKIM_VALID_AU,
RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1
Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
by mail.wl.linuxfoundation.org (Postfix) with ESMTP id E213C203B9
for <patchwork-linux-pci@patchwork.kernel.org>;
Wed, 24 Jan 2018 08:11:46 +0000 (UTC)
Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
id S1752572AbeAXILq (ORCPT
<rfc822;patchwork-linux-pci@patchwork.kernel.org>);
Wed, 24 Jan 2018 03:11:46 -0500
Received: from mail1.hostfission.com ([139.99.139.48]:55918 "EHLO
mail1.hostfission.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
with ESMTP id S1752314AbeAXILp (ORCPT
<rfc822; linux-pci@vger.kernel.org>); Wed, 24 Jan 2018 03:11:45 -0500
X-Greylist: delayed 550 seconds by postgrey-1.27 at vger.kernel.org;
Wed, 24 Jan 2018 03:11:45 EST
Received: from www1.hostfission.com (www1.hostfission.com [139.99.139.52])
by mail1.hostfission.com (Postfix) with ESMTP id B36D943BA8
for <linux-pci@vger.kernel.org>;
Wed, 24 Jan 2018 19:02:33 +1100 (AEDT)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=hostfission.com;
s=mail; t=1516780953;
bh=ugYMw18vRWgXdwnU4crdim5QvxbFUZxY6MfQZMJBccI=;
h=To:Subject:Date:From:From;
b=hFZdAe9U0H0i1vULq25/bkxFGhgZcc8uQPnhdEonlx2y8iYTW7xybJD8L2G+I79Xy
OWWqG3HfHLTBLDNSbdAOs/qJYT1JmcAgVQQm9WB6W9mEiSmXfXzC88oY9h7dB63s6r
TLXSiJ3+JxTYrjH/KjUExOvYoOEmS6UopNicluoE=
Received: by www1.hostfission.com (Postfix, from userid 1000)
id AB16B80745; Wed, 24 Jan 2018 19:02:33 +1100 (AEDT)
To: linux-pci@vger.kernel.org
Subject: [PATCH] Restore PCI bridge configuration space on bridge reset
X-PHP-Originating-Script: 0:rcube.php
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8;
format=flowed
Date: Wed, 24 Jan 2018 19:02:33 +1100
From: geoff@hostfission.com
Message-ID: <0986ad77b71f3b8e0a17f79e238d1ebc@hostfission.com>
X-Sender: geoff@hostfission.com
User-Agent: Roundcube Webmail/1.2.3
Sender: linux-pci-owner@vger.kernel.org
Precedence: bulk
List-ID: <linux-pci.vger.kernel.org>
X-Mailing-List: linux-pci@vger.kernel.org
X-Virus-Scanned: ClamAV using ClamSMTP
According to PCI-to-PCI Bridge Architecture Specification 3.2.5.17
> The bridge’s secondary bus interface and any buffers between
> the two interfaces (primary and secondary) must be initialized
> back to their default state whenever this bit is set.
Failure to observe this causes inability to access devices on the
secondary bus
on the AMD Threadripper platform after device reset when the device is
being
used for PCI passthrough with KVM.
The following patch corrects this by saving the pci state and restoring
it after
the bus has been reset.
--Signed-off-by: Geoffrey McRae<geoff@hostfission.com>
--- ./drivers/pci/pci.c.orig 2018-01-24 18:30:23.913953332 +1100
+++ ./drivers/pci/pci.c 2018-01-24 18:46:31.752819451 +1100
@@ -1112,12 +1112,12 @@ int pci_save_state(struct pci_dev *dev)
EXPORT_SYMBOL(pci_save_state);
static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
- u32 saved_val, int retry)
+ u32 saved_val, int retry, int force)
{
u32 val;
pci_read_config_dword(pdev, offset, &val);
- if (val == saved_val)
+ if (!force && val == saved_val)
return;
for (;;) {
@@ -1136,33 +1136,29 @@ static void pci_restore_config_dword(str
}
static void pci_restore_config_space_range(struct pci_dev *pdev,
- int start, int end, int retry)
+ int start, int end, int retry, int force)
{
int index;
for (index = end; index >= start; index--)
pci_restore_config_dword(pdev, 4 * index,
pdev->saved_config_space[index],
- retry);
+ retry, force);
}
-static void pci_restore_config_space(struct pci_dev *pdev)
+static void pci_restore_config_space(struct pci_dev *pdev, int force)
{
if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
- pci_restore_config_space_range(pdev, 10, 15, 0);
+ pci_restore_config_space_range(pdev, 10, 15, 0, force);
/* Restore BARs before the command register. */
- pci_restore_config_space_range(pdev, 4, 9, 10);
- pci_restore_config_space_range(pdev, 0, 3, 0);
+ pci_restore_config_space_range(pdev, 4, 9, 10, force);
+ pci_restore_config_space_range(pdev, 0, 3, 0, force);
} else {
- pci_restore_config_space_range(pdev, 0, 15, 0);
+ pci_restore_config_space_range(pdev, 0, 15, 0, force);
}
}
-/**
- * pci_restore_state - Restore the saved state of a PCI device
- * @dev: - PCI device that we're dealing with
- */
-void pci_restore_state(struct pci_dev *dev)
+static void _pci_restore_state(struct pci_dev *dev, int force)
{
if (!dev->state_saved)
return;
@@ -1176,7 +1172,7 @@ void pci_restore_state(struct pci_dev *d
pci_cleanup_aer_error_status_regs(dev);
- pci_restore_config_space(dev);
+ pci_restore_config_space(dev, force);
pci_restore_pcix_state(dev);
pci_restore_msi_state(dev);
@@ -1187,6 +1183,15 @@ void pci_restore_state(struct pci_dev *d
dev->state_saved = false;
}
+
+/**
+ * pci_restore_state - Restore the saved state of a PCI device
+ * @dev: - PCI device that we're dealing with
+ */
+void pci_restore_state(struct pci_dev *dev)
+{
+ _pci_restore_state(dev, 0);
+}
EXPORT_SYMBOL(pci_restore_state);
struct pci_saved_state {
@@ -4083,6 +4088,8 @@ void pci_reset_secondary_bus(struct pci_
{
u16 ctrl;
+ pci_save_state(dev);
+
pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
@@ -4092,10 +4099,23 @@ void pci_reset_secondary_bus(struct pci_
*/
msleep(2);
+ pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
/*
+ * According to PCI-to-PCI Bridge Architecture Specification 3.2.5.17
+ *
+ * "The bridge’s secondary bus interface and any buffers between
+ * the two interfaces (primary and secondary) must be initialized
+ * back to their default state whenever this bit is set."
+ *
+ * Failure to observe this causes inability to access devices on the
+ * secondary bus on the AMD Threadripper platform.
+ */
+ _pci_restore_state(dev, 1);
+
+ /*
* Trhfa for conventional PCI is 2^25 clock cycles.
* Assuming a minimum 33MHz clock this results in a 1s
* delay before we can consider subordinate devices to
|