blob: c435bbba851c774727b152c2eed875aab56e78d6 [file] [log] [blame]
Thomas Gleixner1802d0b2019-05-27 08:55:21 +02001// SPDX-License-Identifier: GPL-2.0-only
Christian König7bf60c52019-04-01 17:50:55 +08002/*
3 * fence-chain: chain fences together in a timeline
4 *
5 * Copyright (C) 2018 Advanced Micro Devices, Inc.
6 * Authors:
7 * Christian König <christian.koenig@amd.com>
Christian König7bf60c52019-04-01 17:50:55 +08008 */
9
10#include <linux/dma-fence-chain.h>
11
12static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
13
14/**
15 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
16 * @chain: chain node to get the previous node from
17 *
18 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
19 * chain node.
20 */
21static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
22{
23 struct dma_fence *prev;
24
25 rcu_read_lock();
26 prev = dma_fence_get_rcu_safe(&chain->prev);
27 rcu_read_unlock();
28 return prev;
29}
30
31/**
32 * dma_fence_chain_walk - chain walking function
33 * @fence: current chain node
34 *
35 * Walk the chain to the next node. Returns the next fence or NULL if we are at
36 * the end of the chain. Garbage collects chain nodes which are already
37 * signaled.
38 */
39struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
40{
41 struct dma_fence_chain *chain, *prev_chain;
42 struct dma_fence *prev, *replacement, *tmp;
43
44 chain = to_dma_fence_chain(fence);
45 if (!chain) {
46 dma_fence_put(fence);
47 return NULL;
48 }
49
50 while ((prev = dma_fence_chain_get_prev(chain))) {
51
52 prev_chain = to_dma_fence_chain(prev);
53 if (prev_chain) {
54 if (!dma_fence_is_signaled(prev_chain->fence))
55 break;
56
57 replacement = dma_fence_chain_get_prev(prev_chain);
58 } else {
59 if (!dma_fence_is_signaled(prev))
60 break;
61
62 replacement = NULL;
63 }
64
Chris Wilson29da47c2020-04-09 12:00:59 +010065 tmp = cmpxchg((struct dma_fence __force **)&chain->prev,
66 prev, replacement);
Christian König7bf60c52019-04-01 17:50:55 +080067 if (tmp == prev)
68 dma_fence_put(tmp);
69 else
70 dma_fence_put(replacement);
71 dma_fence_put(prev);
72 }
73
74 dma_fence_put(fence);
75 return prev;
76}
77EXPORT_SYMBOL(dma_fence_chain_walk);
78
79/**
80 * dma_fence_chain_find_seqno - find fence chain node by seqno
81 * @pfence: pointer to the chain node where to start
82 * @seqno: the sequence number to search for
83 *
84 * Advance the fence pointer to the chain node which will signal this sequence
85 * number. If no sequence number is provided then this is a no-op.
86 *
87 * Returns EINVAL if the fence is not a chain node or the sequence number has
88 * not yet advanced far enough.
89 */
90int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
91{
92 struct dma_fence_chain *chain;
93
94 if (!seqno)
95 return 0;
96
97 chain = to_dma_fence_chain(*pfence);
98 if (!chain || chain->base.seqno < seqno)
99 return -EINVAL;
100
101 dma_fence_chain_for_each(*pfence, &chain->base) {
Chris Wilson5de376b2020-04-09 12:01:00 +0100102 if ((*pfence)->seqno < seqno) { /* already signaled */
103 dma_fence_put(*pfence);
104 *pfence = NULL;
105 break;
106 }
107
Christian König7bf60c52019-04-01 17:50:55 +0800108 if ((*pfence)->context != chain->base.context ||
109 to_dma_fence_chain(*pfence)->prev_seqno < seqno)
110 break;
111 }
112 dma_fence_put(&chain->base);
113
114 return 0;
115}
116EXPORT_SYMBOL(dma_fence_chain_find_seqno);
117
118static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
119{
120 return "dma_fence_chain";
121}
122
123static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
124{
125 return "unbound";
126}
127
128static void dma_fence_chain_irq_work(struct irq_work *work)
129{
130 struct dma_fence_chain *chain;
131
132 chain = container_of(work, typeof(*chain), work);
133
134 /* Try to rearm the callback */
135 if (!dma_fence_chain_enable_signaling(&chain->base))
136 /* Ok, we are done. No more unsignaled fences left */
137 dma_fence_signal(&chain->base);
138 dma_fence_put(&chain->base);
139}
140
141static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
142{
143 struct dma_fence_chain *chain;
144
145 chain = container_of(cb, typeof(*chain), cb);
146 irq_work_queue(&chain->work);
147 dma_fence_put(f);
148}
149
150static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
151{
152 struct dma_fence_chain *head = to_dma_fence_chain(fence);
153
154 dma_fence_get(&head->base);
155 dma_fence_chain_for_each(fence, &head->base) {
156 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
157 struct dma_fence *f = chain ? chain->fence : fence;
158
159 dma_fence_get(f);
160 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
161 dma_fence_put(fence);
162 return true;
163 }
164 dma_fence_put(f);
165 }
166 dma_fence_put(&head->base);
167 return false;
168}
169
170static bool dma_fence_chain_signaled(struct dma_fence *fence)
171{
172 dma_fence_chain_for_each(fence, fence) {
173 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
174 struct dma_fence *f = chain ? chain->fence : fence;
175
176 if (!dma_fence_is_signaled(f)) {
177 dma_fence_put(fence);
178 return false;
179 }
180 }
181
182 return true;
183}
184
185static void dma_fence_chain_release(struct dma_fence *fence)
186{
187 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
Christian König92cb3e52019-08-01 15:11:14 +0200188 struct dma_fence *prev;
Christian König7bf60c52019-04-01 17:50:55 +0800189
Christian König92cb3e52019-08-01 15:11:14 +0200190 /* Manually unlink the chain as much as possible to avoid recursion
191 * and potential stack overflow.
192 */
193 while ((prev = rcu_dereference_protected(chain->prev, true))) {
194 struct dma_fence_chain *prev_chain;
195
196 if (kref_read(&prev->refcount) > 1)
197 break;
198
199 prev_chain = to_dma_fence_chain(prev);
200 if (!prev_chain)
201 break;
202
203 /* No need for atomic operations since we hold the last
204 * reference to prev_chain.
205 */
206 chain->prev = prev_chain->prev;
207 RCU_INIT_POINTER(prev_chain->prev, NULL);
208 dma_fence_put(prev);
209 }
210 dma_fence_put(prev);
211
Christian König7bf60c52019-04-01 17:50:55 +0800212 dma_fence_put(chain->fence);
213 dma_fence_free(fence);
214}
215
216const struct dma_fence_ops dma_fence_chain_ops = {
Christian König5e498ab2019-04-15 14:46:34 +0200217 .use_64bit_seqno = true,
Christian König7bf60c52019-04-01 17:50:55 +0800218 .get_driver_name = dma_fence_chain_get_driver_name,
219 .get_timeline_name = dma_fence_chain_get_timeline_name,
220 .enable_signaling = dma_fence_chain_enable_signaling,
221 .signaled = dma_fence_chain_signaled,
222 .release = dma_fence_chain_release,
223};
224EXPORT_SYMBOL(dma_fence_chain_ops);
225
226/**
227 * dma_fence_chain_init - initialize a fence chain
228 * @chain: the chain node to initialize
229 * @prev: the previous fence
230 * @fence: the current fence
Chris Wilson5de376b2020-04-09 12:01:00 +0100231 * @seqno: the sequence number (syncpt) of the fence within the chain
Christian König7bf60c52019-04-01 17:50:55 +0800232 *
233 * Initialize a new chain node and either start a new chain or add the node to
234 * the existing chain of the previous fence.
235 */
236void dma_fence_chain_init(struct dma_fence_chain *chain,
237 struct dma_fence *prev,
238 struct dma_fence *fence,
239 uint64_t seqno)
240{
241 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
242 uint64_t context;
243
244 spin_lock_init(&chain->lock);
245 rcu_assign_pointer(chain->prev, prev);
246 chain->fence = fence;
247 chain->prev_seqno = 0;
248 init_irq_work(&chain->work, dma_fence_chain_irq_work);
249
250 /* Try to reuse the context of the previous chain node. */
Christian König5e498ab2019-04-15 14:46:34 +0200251 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
Christian König7bf60c52019-04-01 17:50:55 +0800252 context = prev->context;
253 chain->prev_seqno = prev->seqno;
254 } else {
255 context = dma_fence_context_alloc(1);
256 /* Make sure that we always have a valid sequence number. */
257 if (prev_chain)
258 seqno = max(prev->seqno, seqno);
259 }
260
261 dma_fence_init(&chain->base, &dma_fence_chain_ops,
262 &chain->lock, context, seqno);
263}
264EXPORT_SYMBOL(dma_fence_chain_init);