1
use std::{collections::BTreeMap, pin::Pin, sync::Arc};
2

            
3
use futures::{Sink, Stream, StreamExt};
4
use jetstream_error::{Error, Result};
5
use tokio::sync::{mpsc, oneshot, Mutex};
6

            
7
use crate::{
8
    client::ClientTransport, context::Context, subscription::RDONE, Frame,
9
    Framer, Protocol, RpcCall, RpcFuture, RpcStream, TagPool,
10
};
11

            
12
pub type RxStream<P> = Pin<
13
    Box<
14
        dyn Stream<Item = Result<Frame<<P as Protocol>::Response>>>
15
            + Send
16
            + Sync,
17
    >,
18
>;
19

            
20
pub type TxSink<P> = Pin<
21
    Box<dyn Sink<Frame<<P as Protocol>::Request>, Error = Error> + Send + Sync>,
22
>;
23

            
24
pub type InFlight<P> = Arc<Mutex<BTreeMap<u16, Waiter<P>>>>;
25

            
26
/// What is waiting on a tag.
27
///
28
/// r[impl jetstream.subscription.definition]
29
/// A unary call's tag is freed by its one response. A subscription's is
30
/// held for as long as the producer may still emit, and freed by the
31
/// terminator — which is why the two cannot share a representation.
32
pub enum Waiter<P: Protocol> {
33
    /// One response, then the tag is free.
34
    Unary(oneshot::Sender<Result<Frame<P::Response>>>),
35
    /// Many responses under one tag, until the terminator.
36
    ///
37
    /// r[impl jetstream.subscription.identity]
38
    /// The `binding` is which subscription this slot belongs to, and it
39
    /// is what makes abandoning one safe: a tag is reused once its
40
    /// terminator frees it, so a stream dropped after that would
41
    /// otherwise find a *different* subscription's waiter under its old
42
    /// number and silence it. Identifiers start at one and are never
43
    /// reused, so a stale holder simply fails to match.
44
    Streaming {
45
        binding: u64,
46
        tx: mpsc::Sender<Result<Frame<P::Response>>>,
47
    },
48
    /// The caller dropped its stream. Frames are discarded, and the tag
49
    /// stays held until the terminator arrives.
50
    ///
51
    /// r[impl jetstream.subscription.identity]
52
    /// Releasing it at drop instead would let the tag be rebound while
53
    /// the producer is still emitting under it, and an in-lane item
54
    /// carries no binding identifier to tell the two apart.
55
    Abandoned,
56
}
57

            
58
/// Client Mux
59
pub struct Mux<P: Protocol> {
60
    send_queue: tokio::sync::mpsc::Sender<Frame<P::Request>>,
61
    in_flight: InFlight<P>,
62
    tag_pool: Arc<TagPool>,
63
    /// r[impl jetstream.subscription.cancel]
64
    /// Tags of subscriptions whose caller has gone, awaiting a
65
    /// cancellation on the wire.
66
    ///
67
    /// Unbounded, and that is deliberate. A bounded queue here is filled
68
    /// by dropping enough unfinished subscriptions without yielding —
69
    /// trivial on a single-threaded runtime — and the drop path cannot
70
    /// wait, so the overflow is *discarded*: those subscriptions stay
71
    /// `Abandoned`, their producers keep working, and their tags are
72
    /// never released. Teardown has to be lossless, and this is bounded
73
    /// in practice by the number of live subscriptions anyway.
74
    cancels: mpsc::UnboundedSender<u16>,
75
    /// r[impl jetstream.subscription.identity]
76
    /// Hands out the binding identifiers above. Unique, never reused,
77
    /// and never zero — the counter starts at one, so `fetch_add`
78
    /// returning the pre-increment value would hand out zero first.
79
    bindings: std::sync::atomic::AtomicU64,
80
}
81

            
82
impl<P: Protocol> Mux<P>
83
where
84
    P: 'static,
85
{
86
285
    async fn demux(
87
285
        mut rx: RxStream<P>,
88
285
        in_flight: InFlight<P>,
89
285
        tag_pool: Arc<TagPool>,
90
285
        cancels: mpsc::UnboundedSender<u16>,
91
285
    ) -> Result<()> {
92
        use futures::StreamExt;
93
        // Why the lane stopped, so the drain below can tell every
94
        // waiter something true rather than just dropping them.
95
        let reason: Option<Error> = loop {
96
2663
            let frame = match rx.next().await {
97
2378
                Some(Ok(frame)) => frame,
98
107
                Some(Err(e)) => break Some(e),
99
1
                None => break None,
100
            };
101
2378
            let frame: Frame<P::Response> = frame;
102
2378
            let tag = frame.tag;
103
            // r[impl jetstream.subscription.termination]
104
2378
            let ends_here = frame.msg.message_type() == RDONE;
105

            
106
            // Take what we need under the lock and release it before any
107
            // send: a bounded stream channel can block, and holding the
108
            // map across that would stall every other tag.
109
            let waiter = {
110
2378
                let mut map = in_flight.lock().await;
111
2378
                match map.get(&tag) {
112
                    None => {
113
                        // r[impl jetstream.rcp.multiplexing]
114
                        // A frame for a tag nobody holds means this end
115
                        // and the peer disagree about what is in flight.
116
                        // This used to `unwrap` — a panic on any stray
117
                        // frame — and then, briefly, to log and carry
118
                        // on. Carrying on is the worse of the two: the
119
                        // tag stays eligible for reuse, so the *next*
120
                        // stray frame bearing it is delivered to
121
                        // whichever call has since been bound to it, and
122
                        // a detected desynchronisation turns into a
123
                        // response misbinding that nothing detects.
124
                        //
125
                        // The lane is the unit that is out of step, so
126
                        // the lane is what fails. Everything on it is
127
                        // resolved by the drain below.
128
                        tracing::error!(
129
                            tag,
130
                            "response for an unknown tag; failing the lane"
131
                        );
132
                        break Some(Error::new(format!(
133
                            "peer sent a response for tag {tag}, which is \
134
                             not in flight on this lane"
135
                        )));
136
                    }
137
424
                    Some(Waiter::Unary(_)) => map.remove(&tag),
138
                    Some(Waiter::Streaming { binding, tx }) => {
139
1332
                        let (binding, tx) = (*binding, tx.clone());
140
1332
                        if ends_here {
141
3
                            map.remove(&tag);
142
1329
                        }
143
1332
                        Some(Waiter::Streaming { binding, tx })
144
                    }
145
                    Some(Waiter::Abandoned) => {
146
622
                        if ends_here {
147
94
                            map.remove(&tag);
148
528
                        }
149
622
                        Some(Waiter::Abandoned)
150
                    }
151
                }
152
            };
153

            
154
2378
            match waiter {
155
                Some(Waiter::Unary(tx)) => {
156
424
                    if tx.send(Ok(frame)).is_err() {
157
81
                        tracing::debug!(tag, "caller gone before its response");
158
343
                    }
159
424
                    tag_pool.release_tag(tag).await;
160
                }
161
                Some(Waiter::Streaming { binding, tx }) => {
162
                    // r[impl jetstream.subscription.backpressure]
163
                    // r[impl jetstream.subscription.backpressure.reporting]
164
                    // Never *await* the subscriber here. This is the
165
                    // lane's only reader, so a full channel would stop
166
                    // every other subscription and every unary call on
167
                    // the lane until this one caught up — the transport
168
                    // receive window consumed by whichever subscriber
169
                    // stopped polling. The specification offers exactly
170
                    // two conforming responses to a subscriber that
171
                    // cannot keep up, and blocking its neighbours is
172
                    // neither: apply backpressure, or terminate the
173
                    // subscription and report it.
174
                    //
175
                    // Backpressure belongs at the producer, which is
176
                    // where the token goes; here the honest move is to
177
                    // end this one subscription and say why.
178
1332
                    match tx.try_send(Ok(frame)) {
179
1332
                        Ok(()) => {}
180
                        Err(mpsc::error::TrySendError::Closed(_)) => {
181
                            tracing::debug!(tag, "subscriber gone mid-stream");
182
                        }
183
                        Err(mpsc::error::TrySendError::Full(_)) => {
184
                            tracing::warn!(
185
                                tag,
186
                                "subscriber fell behind; terminating it \
187
                                 rather than stalling the lane"
188
                            );
189
                            // Abandoned rather than removed: the tag is
190
                            // still in flight at the producer, and is
191
                            // released when its terminator arrives.
192
                            //
193
                            // Unless this frame *was* the terminator.
194
                            // `ends_here` has already taken the waiter
195
                            // out, and nothing further will arrive under
196
                            // this tag — so putting it back would strand
197
                            // the entry and the tag for the life of the
198
                            // lane, which is what a burst that fills the
199
                            // queue exactly at `RDONE` would do.
200
                            if !ends_here {
201
                                let mut map = in_flight.lock().await;
202
                                // Only if this subscription still owns
203
                                // the slot: the caller may have dropped
204
                                // its stream while the lock was open.
205
                                if matches!(
206
                                    map.get(&tag),
207
                                    Some(Waiter::Streaming { binding: b, .. })
208
                                        if *b == binding
209
                                ) {
210
                                    map.insert(tag, Waiter::Abandoned);
211
                                }
212
                            }
213
                            // Delivering the reason may block, so it
214
                            // does not happen here. The sender moves to
215
                            // a task; the lane keeps reading.
216
                            let overflowed = Error::new(format!(
217
                                "subscription {tag} fell behind and was \
218
                                 terminated"
219
                            ));
220
                            tokio::spawn(async move {
221
                                let _ = tx.send(Err(overflowed)).await;
222
                            });
223
                            if ends_here {
224
                                tag_pool.release_tag(tag).await;
225
                            }
226
                            // r[impl jetstream.subscription.cancel]
227
                            // And tell the producer, so the work stops
228
                            // rather than only the delivery.
229
                            let _ = cancels.send(tag);
230
                            continue;
231
                        }
232
                    }
233
1332
                    if ends_here {
234
3
                        tag_pool.release_tag(tag).await;
235
1329
                    }
236
                }
237
                Some(Waiter::Abandoned) => {
238
622
                    if ends_here {
239
94
                        tag_pool.release_tag(tag).await;
240
528
                    }
241
                }
242
                None => unreachable!("checked under the lock"),
243
            }
244
        };
245

            
246
        // r[impl jetstream.subscription.surface.termination]
247
        // The lane is finished, and every waiter still on it has to be
248
        // told. A streaming waiter left in the map keeps its receiver
249
        // open — `RpcStream` holds the same map, so dropping the loop's
250
        // handle closes nothing — and its caller waits forever for an
251
        // item that cannot come, holding a tag that can never be
252
        // released. "The connection dropped" is one of the three
253
        // outcomes the surface has to distinguish; silence is not one of
254
        // them.
255
155
        let dropped = |tag: u16| {
256
155
            Error::new(match &reason {
257
                Some(e) => {
258
154
                    format!("lane failed before tag {tag} finished: {e}")
259
                }
260
1
                None => format!("lane closed before tag {tag} finished"),
261
            })
262
155
        };
263
        let orphans: Vec<(u16, Waiter<P>)> = {
264
108
            let mut map = in_flight.lock().await;
265
108
            std::mem::take(&mut *map).into_iter().collect()
266
        };
267
108
        for (tag, waiter) in orphans {
268
308
            match waiter {
269
                Waiter::Unary(tx) => {
270
154
                    let _ = tx.send(Err(dropped(tag)));
271
                }
272
                // The same rule as the delivery loop above, for the same
273
                // reason: never *await* a subscriber. These are drained
274
                // in sequence, so one subscriber whose queue is full
275
                // would hold up every waiter behind it — none of them
276
                // told the lane had closed, none of their tags released.
277
                // The waiter that cannot take its error now gets it from
278
                // a task instead; the drain moves on.
279
                Waiter::Streaming { tx, .. } => {
280
1
                    let closed = dropped(tag);
281
1
                    if let Err(mpsc::error::TrySendError::Full(item)) =
282
1
                        tx.try_send(Err(closed))
283
                    {
284
                        tokio::spawn(async move {
285
                            let _ = tx.send(item).await;
286
                        });
287
1
                    }
288
                }
289
153
                Waiter::Abandoned => {}
290
            }
291
308
            tag_pool.release_tag(tag).await;
292
        }
293

            
294
108
        match reason {
295
107
            Some(e) => Err(e),
296
1
            None => Ok(()),
297
        }
298
108
    }
299

            
300
285
    async fn mux(
301
285
        mut send_queue: tokio::sync::mpsc::Receiver<Frame<P::Request>>,
302
285
        mut tx_sink: TxSink<P>,
303
285
    ) -> Result<()> {
304
1735
        while let Some(frame) = send_queue.recv().await {
305
            use futures::SinkExt;
306
1624
            tx_sink.send(frame).await?;
307
        }
308
63
        Ok(())
309
237
    }
310

            
311
568
    pub async fn rpc(&self, _ctx: Context, request: P::Request) -> RpcCall<P> {
312
568
        let tag = self.tag_pool.acquire_tag().await;
313
568
        let (tx, rx) = oneshot::channel();
314
568
        self.in_flight.lock().await.insert(tag, Waiter::Unary(tx));
315

            
316
        // r[impl jetstream.subscription.dispatch.issue-order]
317
        // Queued here, not from a spawned task. A lane is one *ordered*
318
        // sequence, and handing each request to the scheduler makes the
319
        // order in which they reach it the scheduler's to decide.
320
        // Between independent calls that is merely surprising; between a
321
        // subscription and the cancellation naming it, the cancellation
322
        // arrives first and names a subscription that does not exist.
323
        let (tag, future) =
324
568
            match self.send_queue.send(Frame { tag, msg: request }).await {
325
568
                Ok(()) => (tag, RpcFuture::Waiting(rx)),
326
                Err(_) => {
327
                    self.in_flight.lock().await.remove(&tag);
328
                    self.tag_pool.release_tag(tag).await;
329
                    (
330
                        tag,
331
                        RpcFuture::Failed(Some(Error::new(
332
                            "the lane is closed",
333
                        ))),
334
                    )
335
                }
336
            };
337
568
        RpcCall { tag, future }
338
568
    }
339

            
340
    /// r[impl jetstream.subscription.cancel]
341
    /// Turns a dropped subscription's tag into a cancellation on the
342
    /// wire.
343
    ///
344
    /// This is a task rather than something `Drop` does itself because
345
    /// the cancellation needs a *fresh* tag — the specification forbids
346
    /// reusing the subscription's own — and acquiring one may wait,
347
    /// which a synchronous `Drop` cannot.
348
285
    async fn cancel_dropped(
349
285
        mut cancels: mpsc::UnboundedReceiver<u16>,
350
285
        in_flight: InFlight<P>,
351
285
        tag_pool: Arc<TagPool>,
352
285
        send_queue: mpsc::Sender<Frame<P::Request>>,
353
285
    ) {
354
833
        while let Some(oldtag) = cancels.recv().await {
355
            // Zero: the cancellation travels the subscription's own lane,
356
            // where `oldtag` is unambiguous.
357
578
            let Some(msg) = P::tcancel(oldtag, 0) else {
358
                continue;
359
            };
360
            // r[impl jetstream.subscription.cancel]
361
            // Control capacity, not the ordinary pool. Taking an ordinary
362
            // tag here deadlocks precisely when cancellation matters:
363
            // with the pool saturated by live subscriptions, this waits
364
            // for a tag that is only released by the terminator this
365
            // cancellation would produce.
366
578
            let tag = tag_pool.acquire_control_tag().await;
367
            // A unary waiter whose receiver is dropped: nobody is left to
368
            // read the acknowledgement, but the tag must stay held until
369
            // it arrives, and that is what the unary path already does.
370
578
            let (tx, _rx) = oneshot::channel();
371
578
            in_flight.lock().await.insert(tag, Waiter::Unary(tx));
372
578
            if send_queue.send(Frame { tag, msg }).await.is_err() {
373
30
                break;
374
548
            }
375
        }
376
126
    }
377

            
378
285
    pub fn new(
379
285
        max_concurrent_requests: u16,
380
285
        transport: Box<dyn ClientTransport<P>>,
381
285
    ) -> Self {
382
285
        let tag_pool = Arc::new(TagPool::new(max_concurrent_requests));
383
285
        let (send_queue, send_queue_rx) = tokio::sync::mpsc::channel(1024);
384
285
        let (tx, rx) = StreamExt::split(transport);
385
285
        let (tx, rx) = (Box::pin(tx), Box::pin(rx));
386
285
        let in_flight = Arc::new(Mutex::new(BTreeMap::new()));
387
285
        let pending = in_flight.clone();
388
285
        let tags = tag_pool.clone();
389
285
        let (cancels, cancels_rx) = mpsc::unbounded_channel();
390
285
        let lagged = cancels.clone();
391
285
        tokio::spawn(
392
285
            async move { Self::demux(rx, pending, tags, lagged).await },
393
        );
394
285
        tokio::spawn(async move { Self::mux(send_queue_rx, tx).await });
395

            
396
285
        let map = in_flight.clone();
397
285
        let tags = tag_pool.clone();
398
285
        let queue = send_queue.clone();
399
285
        tokio::spawn(Self::cancel_dropped(cancels_rx, map, tags, queue));
400

            
401
285
        Self {
402
285
            in_flight,
403
285
            send_queue,
404
285
            tag_pool,
405
285
            cancels,
406
285
            bindings: std::sync::atomic::AtomicU64::new(1),
407
285
        }
408
285
    }
409

            
410
    /// Issue a streaming call: one request, many responses under its tag.
411
    ///
412
    /// r[impl jetstream.subscription.overview]
413
    /// r[impl jetstream.subscription.definition]
414
    /// The tag stays in flight until the terminator, which is what makes
415
    /// it the subscription's identity for as long as it lives.
416
628
    pub async fn rpc_stream(
417
628
        &self,
418
628
        _ctx: Context,
419
628
        request: P::Request,
420
628
        capacity: usize,
421
628
    ) -> RpcStream<P> {
422
        // r[impl jetstream.subscription.cancel]
423
        // From the streaming region, so no number of live subscriptions
424
        // can make an ordinary call impossible.
425
628
        let tag = self.tag_pool.acquire_streaming_tag().await;
426
628
        let binding = self
427
628
            .bindings
428
628
            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
429
628
        let (tx, items) = mpsc::channel(capacity);
430
628
        self.in_flight
431
628
            .lock()
432
628
            .await
433
628
            .insert(tag, Waiter::Streaming { binding, tx });
434

            
435
        // r[impl jetstream.subscription.dispatch.issue-order]
436
        // Queued here rather than from a spawned task, for the ordering
437
        // reason spelled out in `rpc`.
438
        // r[impl jetstream.subscription.surface.termination]
439
        // A request that never reached the lane must not look like a
440
        // subscription that ended normally. Dropping the sender here
441
        // closes `items`, so the first poll would yield `None` — the
442
        // same thing a clean, empty subscription yields, and the caller
443
        // has no way to tell the difference. The unary path resolves
444
        // this as an error; so does this one.
445
628
        let failed = if self
446
628
            .send_queue
447
628
            .send(Frame { tag, msg: request })
448
628
            .await
449
            .is_err()
450
        {
451
            self.in_flight.lock().await.remove(&tag);
452
            self.tag_pool.release_tag(tag).await;
453
            Some(Error::new(
454
                "the lane is closed; the subscription was never issued",
455
            ))
456
        } else {
457
628
            None
458
        };
459
628
        RpcStream {
460
628
            tag,
461
628
            binding,
462
628
            items,
463
628
            in_flight: self.in_flight.clone(),
464
628
            cancels: self.cancels.clone(),
465
628
            finished: false,
466
628
            failed,
467
628
        }
468
628
    }
469
}
470

            
471
#[cfg(test)]
472
mod tests;