Skip to main content

enclave_runner_sgx/usercalls/
mod.rs

1/* Copyright (c) Fortanix, Inc.
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7use std::alloc::{GlobalAlloc, Layout, System};
8use std::cell::RefCell;
9use std::collections::{HashMap, VecDeque};
10use std::io::{self, ErrorKind as IoErrorKind, Result as IoResult};
11use std::pin::Pin;
12use std::ptr;
13use std::result::Result as StdResult;
14use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
15use std::sync::{Arc, LazyLock};
16use std::task::{Context, Poll, Waker};
17use std::thread::{self, JoinHandle};
18use std::time::{self, Duration};
19use std::{cmp, fmt, str};
20
21use anyhow::bail;
22use enclave_runner::platform::CommandConfiguration;
23use enclave_runner::stream_router::{AsyncListener, AsyncStream, StreamRouter};
24use fnv::FnvHashMap;
25use fortanix_sgx_abi::*;
26use futures::future::{poll_fn, Either, Future, FutureExt};
27use futures::lock::Mutex;
28use insecure_time::{Freq, Rdtscp};
29use ipc_queue::position::WritePosition;
30use ipc_queue::{DescriptorGuard, Identified, QueueEvent};
31use lazy_static::lazy_static;
32#[cfg(unix)]
33use libc::*;
34#[cfg(unix)]
35use nix::sys::signal;
36use sgxs::loader::Tcs as SgxsTcs;
37use tokio::io::ReadBuf;
38use tokio::runtime::Builder as RuntimeBuilder;
39use tokio::sync::broadcast::error::RecvError;
40use tokio::sync::{broadcast, mpsc as async_mpsc, oneshot, Semaphore};
41
42use self::abi::dispatch;
43use self::abi::ReturnValue;
44use self::abi::UsercallList;
45use self::interface::ToSgxResult;
46use self::interface::{Handler, OutputBuffer};
47use crate::loader::{EnclavePanic, ErasedTcs};
48use crate::tcs::{self, CoResult, ThreadResult};
49
50pub(crate) mod abi;
51mod interface;
52
53lazy_static! {
54    static ref DEBUGGER_TOGGLE_SYNC: Mutex<()> = Mutex::new(());
55}
56
57const NANOS_PER_SEC: u64 = 1_000_000_000;
58
59static TIME_INFO: LazyLock<Option<InsecureTimeInfo>> = LazyLock::new(|| {
60    if Rdtscp::is_supported() {
61        if let Ok(frequency) = Freq::get() {
62            return Some(InsecureTimeInfo {
63                version: 0,
64                frequency: frequency.as_u64(),
65            });
66        }
67    }
68    None
69});
70
71// This is not an event in the sense that it could be passed to `send()` or
72// `wait()` usercalls in enclave code. However, it's easier for the enclave
73// runner implementation to lump it in with events. Also note that this constant
74// is not public.
75const EV_ABORT: u64 = 0b0000_0000_0001_0000;
76
77const USERCALL_QUEUE_SIZE: usize = 16;
78const RETURN_QUEUE_SIZE: usize = 1024;
79const CANCEL_QUEUE_SIZE: usize = USERCALL_QUEUE_SIZE * 2;
80
81enum UsercallSendData {
82    Sync(ThreadResult<ErasedTcs>, RunningTcs, RefCell<[u8; 1024]>),
83    Async(Identified<Usercall>, Option<oneshot::Receiver<()>>),
84}
85
86// This is the same as UsercallSendData except that it can't be Sync(CoResult::Return(...), ...)
87enum UsercallHandleData {
88    Sync(tcs::Usercall<ErasedTcs>, RunningTcs, RefCell<[u8; 1024]>),
89    Async(
90        Identified<Usercall>,
91        Option<oneshot::Receiver<()>>,
92        Option<async_mpsc::UnboundedSender<UsercallEvent>>,
93    ),
94}
95
96type EnclaveResult = StdResult<(u64, u64), EnclaveAbort<Option<EnclavePanic>>>;
97
98struct AsyncStreamAdapter {
99    stream: Pin<Box<dyn AsyncStream>>,
100    read_queue: VecDeque<Waker>,
101    write_queue: VecDeque<Waker>,
102    flush_queue: VecDeque<Waker>,
103}
104
105fn notify_other_tasks(cx: &mut Context, queue: &mut VecDeque<Waker>) {
106    for task in queue.drain(..) {
107        if !task.will_wake(&cx.waker()) {
108            task.wake();
109        }
110    }
111}
112
113impl AsyncStreamAdapter {
114    fn poll_read_alloc(
115        mut self: Pin<&mut Self>,
116        cx: &mut Context,
117    ) -> Poll<tokio::io::Result<Vec<u8>>> {
118        match self.stream.as_mut().poll_read_alloc(cx) {
119            Poll::Pending => {
120                self.read_queue.push_back(cx.waker().clone());
121                Poll::Pending
122            }
123            Poll::Ready(Ok(ret)) => {
124                notify_other_tasks(cx, &mut self.read_queue);
125                Poll::Ready(Ok(ret))
126            }
127            Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
128        }
129    }
130
131    fn poll_read(
132        mut self: Pin<&mut Self>,
133        cx: &mut Context,
134        buf: &mut ReadBuf,
135    ) -> Poll<tokio::io::Result<()>> {
136        match self.stream.as_mut().poll_read(cx, buf) {
137            Poll::Pending => {
138                self.read_queue.push_back(cx.waker().clone());
139                Poll::Pending
140            }
141            Poll::Ready(Ok(())) => {
142                notify_other_tasks(cx, &mut self.read_queue);
143                Poll::Ready(Ok(()))
144            }
145            Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
146        }
147    }
148
149    fn poll_write(
150        mut self: Pin<&mut Self>,
151        cx: &mut Context,
152        buf: &[u8],
153    ) -> Poll<tokio::io::Result<usize>> {
154        match self.stream.as_mut().poll_write(cx, buf) {
155            Poll::Pending => {
156                self.write_queue.push_back(cx.waker().clone());
157                Poll::Pending
158            }
159            Poll::Ready(Ok(ret)) => {
160                notify_other_tasks(cx, &mut self.write_queue);
161                Poll::Ready(Ok(ret))
162            }
163            Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
164        }
165    }
166
167    fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<tokio::io::Result<()>> {
168        match self.stream.as_mut().poll_flush(cx) {
169            Poll::Pending => {
170                self.flush_queue.push_back(cx.waker().clone());
171                Poll::Pending
172            }
173            Poll::Ready(Ok(ret)) => {
174                notify_other_tasks(cx, &mut self.flush_queue);
175                Poll::Ready(Ok(ret))
176            }
177            Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
178        }
179    }
180}
181
182struct AsyncStreamContainer {
183    inner: Mutex<Pin<Box<AsyncStreamAdapter>>>,
184}
185
186impl AsyncStreamContainer {
187    fn new(s: Box<dyn AsyncStream>) -> Self {
188        AsyncStreamContainer {
189            inner: Mutex::new(Box::pin(AsyncStreamAdapter {
190                stream: s.into(),
191                read_queue: VecDeque::new(),
192                write_queue: VecDeque::new(),
193                flush_queue: VecDeque::new(),
194            })),
195        }
196    }
197
198    async fn async_read(&self, buf: &mut ReadBuf<'_>) -> IoResult<()> {
199        poll_fn(|cx| {
200            let inner_ref = &mut self.inner.lock();
201            let mut inner = Pin::new(inner_ref);
202            match inner.as_mut().poll(cx) {
203                Poll::Ready(mut adapter) => adapter.as_mut().poll_read(cx, buf),
204                Poll::Pending => Poll::Pending,
205            }
206        })
207        .await
208    }
209
210    async fn async_read_alloc(&self) -> IoResult<Vec<u8>> {
211        poll_fn(|cx| {
212            let inner_ref = &mut self.inner.lock();
213            let mut inner = Pin::new(inner_ref);
214            match inner.as_mut().poll(cx) {
215                Poll::Ready(mut adapter) => adapter.as_mut().poll_read_alloc(cx),
216                Poll::Pending => Poll::Pending,
217            }
218        })
219        .await
220    }
221
222    async fn async_write(&self, buf: &[u8]) -> IoResult<usize> {
223        poll_fn(|cx| {
224            let inner_ref = &mut self.inner.lock();
225            let mut inner = Pin::new(inner_ref);
226            match inner.as_mut().poll(cx) {
227                Poll::Ready(mut adapter) => adapter.as_mut().poll_write(cx, buf),
228                Poll::Pending => Poll::Pending,
229            }
230        })
231        .await
232    }
233
234    async fn async_flush(&self) -> IoResult<()> {
235        poll_fn(|cx| {
236            let inner_ref = &mut self.inner.lock();
237            let mut inner = Pin::new(inner_ref);
238            match inner.as_mut().poll(cx) {
239                Poll::Ready(mut adapter) => adapter.as_mut().poll_flush(cx),
240                Poll::Pending => Poll::Pending,
241            }
242        })
243        .await
244    }
245}
246
247struct AsyncListenerAdapter {
248    listener: Pin<Box<dyn AsyncListener>>,
249    accept_queue: VecDeque<Waker>,
250}
251
252impl AsyncListenerAdapter {
253    fn poll_accept(
254        mut self: Pin<&mut Self>,
255        cx: &mut Context,
256        local_addr: Option<&mut String>,
257        peer_addr: Option<&mut String>,
258    ) -> Poll<tokio::io::Result<Box<dyn AsyncStream>>> {
259        match self
260            .listener
261            .as_mut()
262            .poll_accept(cx, local_addr, peer_addr)
263        {
264            Poll::Pending => {
265                self.accept_queue.push_back(cx.waker().clone());
266                Poll::Pending
267            }
268            Poll::Ready(Ok(ret)) => {
269                notify_other_tasks(cx, &mut self.accept_queue);
270                Poll::Ready(Ok(ret))
271            }
272            Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
273        }
274    }
275}
276
277struct AsyncListenerContainer {
278    inner: Mutex<Pin<Box<AsyncListenerAdapter>>>,
279}
280
281impl AsyncListenerContainer {
282    fn new(l: Box<dyn AsyncListener>) -> Self {
283        AsyncListenerContainer {
284            inner: Mutex::new(Pin::new(Box::new(AsyncListenerAdapter {
285                listener: l.into(),
286                accept_queue: VecDeque::new(),
287            }))),
288        }
289    }
290
291    async fn async_accept(
292        &self,
293        local_addr: Option<&mut String>,
294        peer_addr: Option<&mut String>,
295    ) -> IoResult<Box<dyn AsyncStream>> {
296        let mut local_addr_owned: Option<String> = if local_addr.is_some() {
297            Some(String::new())
298        } else {
299            None
300        };
301        let mut peer_addr_owned: Option<String> = if peer_addr.is_some() {
302            Some(String::new())
303        } else {
304            None
305        };
306        let res = poll_fn(|cx| {
307            let inner_ref = &mut self.inner.lock();
308            let mut inner = Pin::new(inner_ref);
309            match inner.as_mut().poll(cx) {
310                Poll::Ready(mut adapter) => adapter.as_mut().poll_accept(
311                    cx,
312                    local_addr_owned.as_mut(),
313                    peer_addr_owned.as_mut(),
314                ),
315                Poll::Pending => Poll::Pending,
316            }
317        })
318        .await;
319
320        if let Some(local_addr) = local_addr {
321            *local_addr = local_addr_owned.unwrap();
322        }
323        if let Some(peer_addr) = peer_addr {
324            *peer_addr = peer_addr_owned.unwrap();
325        }
326        res
327    }
328}
329
330enum AsyncFileDesc {
331    Stream(AsyncStreamContainer),
332    Listener(AsyncListenerContainer),
333}
334
335impl AsyncFileDesc {
336    fn stream(s: Box<dyn AsyncStream>) -> AsyncFileDesc {
337        AsyncFileDesc::Stream(AsyncStreamContainer::new(s))
338    }
339
340    fn listener(l: Box<dyn AsyncListener>) -> AsyncFileDesc {
341        AsyncFileDesc::Listener(AsyncListenerContainer::new(l))
342    }
343
344    fn as_stream(&self) -> IoResult<&AsyncStreamContainer> {
345        if let AsyncFileDesc::Stream(ref s) = self {
346            Ok(s)
347        } else {
348            Err(IoErrorKind::InvalidInput.into())
349        }
350    }
351
352    fn as_listener(&self) -> IoResult<&AsyncListenerContainer> {
353        if let AsyncFileDesc::Listener(ref l) = self {
354            Ok(l)
355        } else {
356            Err(IoErrorKind::InvalidInput.into())
357        }
358    }
359}
360
361#[derive(Debug)]
362pub(crate) enum EnclaveAbort<T> {
363    Exit {
364        panic: T,
365    },
366    /// Secondary threads exiting due to an abort
367    Secondary,
368    IndefiniteWait,
369    InvalidUsercall(u64),
370    MainReturned,
371}
372
373impl<T> EnclaveAbort<T> {
374    fn map_panic<U, F: FnOnce(T) -> U>(self, f: F) -> EnclaveAbort<U> {
375        match self {
376            EnclaveAbort::Exit { panic } => EnclaveAbort::Exit { panic: f(panic) },
377            EnclaveAbort::Secondary => EnclaveAbort::Secondary,
378            EnclaveAbort::IndefiniteWait => EnclaveAbort::IndefiniteWait,
379            EnclaveAbort::InvalidUsercall(n) => EnclaveAbort::InvalidUsercall(n),
380            EnclaveAbort::MainReturned => EnclaveAbort::MainReturned,
381        }
382    }
383}
384
385#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)]
386struct TcsAddress(usize);
387
388impl ErasedTcs {
389    fn address(&self) -> TcsAddress {
390        TcsAddress(SgxsTcs::address(self) as _)
391    }
392}
393
394impl fmt::Pointer for TcsAddress {
395    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
396        (self.0 as *const u8).fmt(f)
397    }
398}
399
400struct StoppedTcs {
401    tcs: ErasedTcs,
402}
403
404struct IOHandlerInput<'tcs> {
405    tcs: Option<&'tcs mut RunningTcs>,
406    enclave: Arc<EnclaveState>,
407    work_sender: &'tcs crossbeam::channel::Sender<Work>,
408}
409
410struct PendingEvents {
411    // The Semaphores are basically counting how many times each event set has
412    // been sent. The count is decreased when an event set is consumed through
413    // `take` or `wait_for` methods by calling `SemaphorePermit::forget`.
414    counts: [Semaphore; Self::EV_MAX],
415    abort: Semaphore,
416}
417
418impl PendingEvents {
419    // Will error if it doesn't fit in a `u64`
420    const EV_MAX_U64: u64 =
421        (EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK | EV_CANCELQ_NOT_FULL) + 1;
422    const EV_MAX: usize = Self::EV_MAX_U64 as _;
423    // Will error if it doesn't fit in a `usize`
424    const _ERROR_IF_USIZE_TOO_SMALL: u64 = u64::MAX + (Self::EV_MAX_U64 - (Self::EV_MAX as u64));
425    const _EV_MAX_U16: u16 = Self::EV_MAX_U64 as u16;
426    // Will error if it doesn't fit in a `u16`. We should consider a different approach if we reach this limit.
427    const _ERROR_IF_TOO_BIG: u64 = u64::MAX + (Self::EV_MAX_U64 - (Self::_EV_MAX_U16 as u64));
428
429    fn new() -> Self {
430        PendingEvents {
431            counts: [
432                Semaphore::new(0),
433                Semaphore::new(0),
434                Semaphore::new(0),
435                Semaphore::new(0),
436                Semaphore::new(0),
437                Semaphore::new(0),
438                Semaphore::new(0),
439                Semaphore::new(0),
440                Semaphore::new(0),
441                Semaphore::new(0),
442                Semaphore::new(0),
443                Semaphore::new(0),
444                Semaphore::new(0),
445                Semaphore::new(0),
446                Semaphore::new(0),
447                Semaphore::new(0),
448            ],
449            abort: Semaphore::new(0),
450        }
451    }
452
453    fn take(&self, event_mask: u64) -> Option<u64> {
454        assert!(event_mask < Self::EV_MAX_U64);
455
456        if let Ok(_) = self.abort.try_acquire() {
457            return Some(EV_ABORT);
458        }
459
460        for i in (1..Self::EV_MAX).rev() {
461            let ev = i as u64;
462            if (ev & event_mask) != 0 {
463                if let Ok(permit) = self.counts[i].try_acquire() {
464                    permit.forget();
465                    return Some(ev);
466                }
467            }
468        }
469        None
470    }
471
472    async fn wait_for(&self, event_mask: u64) -> u64 {
473        assert!(event_mask < Self::EV_MAX_U64);
474
475        if let Ok(_) = self.abort.try_acquire() {
476            return EV_ABORT;
477        }
478
479        let it = std::iter::once((EV_ABORT, &self.abort))
480            .chain(
481                self.counts
482                    .iter()
483                    .enumerate()
484                    .map(|(ev, sem)| (ev as u64, sem))
485                    .filter(|&(ev, _)| ev & event_mask != 0),
486            )
487            .map(|(ev, sem)| sem.acquire().map(move |permit| (ev, permit)).boxed());
488
489        let ((ev, permit), _, _) = futures::future::select_all(it).await;
490
491        // Abort should take precedence if it happens concurrently with another
492        // event. The abort semaphore may not have been selected.
493        if let Ok(_) = self.abort.try_acquire() {
494            return EV_ABORT;
495        }
496        if ev != EV_ABORT {
497            if let Ok(permit) = permit {
498                permit.forget();
499            }
500        }
501        ev
502    }
503
504    fn push(&self, event: u64) {
505        debug_assert!(event != 0 && event < Self::EV_MAX_U64);
506        let index = event as usize;
507        // add_permits() panics if the permit limit is exceeded.
508        // NOTE: [the documentation] incorrectly specifies the maximum to be
509        // `usize::MAX >> 3`, while the actual maximum is `usize::MAX >> 4`.
510        // It's possible to have multiple threads pushing the same event
511        // concurrently, hence the smaller bound.
512        //
513        // [the documentation]: https://docs.rs/tokio/0.2.22/tokio/sync/struct.Semaphore.html#method.add_permits
514        const MAX_PERMITS: usize = usize::MAX >> 5;
515        if self.counts[index].available_permits() < MAX_PERMITS {
516            self.counts[index].add_permits(1);
517        }
518    }
519
520    fn abort(&self) {
521        if self.abort.available_permits() == 0 {
522            self.abort.add_permits(usize::MAX >> 5);
523        }
524    }
525}
526
527struct RunningTcs {
528    tcs_address: TcsAddress,
529    mode: EnclaveEntry,
530}
531
532enum EnclaveKind {
533    Command(Command),
534    Library(Library),
535}
536
537struct PanicReason {
538    primary_panic_reason: Option<EnclaveAbort<EnclavePanic>>,
539    other_reasons: Vec<EnclaveAbort<EnclavePanic>>,
540}
541
542struct Command {
543    panic_reason: Mutex<PanicReason>,
544}
545
546struct Library {}
547
548impl EnclaveKind {
549    fn as_command(&self) -> Option<&Command> {
550        match self {
551            EnclaveKind::Command(c) => Some(c),
552            _ => None,
553        }
554    }
555
556    fn _as_library(&self) -> Option<&Library> {
557        match self {
558            EnclaveKind::Library(l) => Some(l),
559            _ => None,
560        }
561    }
562}
563
564struct FifoGuards {
565    usercall_queue: DescriptorGuard<Usercall>,
566    return_queue: DescriptorGuard<Return>,
567    cancel_queue: DescriptorGuard<Cancel>,
568    async_queues_called: bool,
569}
570
571pub(crate) struct EnclaveState {
572    kind: EnclaveKind,
573    event_queues: FnvHashMap<TcsAddress, PendingEvents>,
574    fds: Mutex<FnvHashMap<Fd, Arc<AsyncFileDesc>>>,
575    last_fd: AtomicUsize,
576    exiting: AtomicBool,
577    stream_router: Box<dyn StreamRouter>,
578    threads_queue: crossbeam::queue::SegQueue<StoppedTcs>,
579    forward_panics: bool,
580    force_time_usercalls: bool,
581    // Once set to Some, the guards should not be dropped for the lifetime of the enclave.
582    fifo_guards: Mutex<Option<FifoGuards>>,
583    return_queue_tx: Mutex<Option<ipc_queue::AsyncSender<Return, QueueSynchronizer>>>,
584}
585
586struct Work {
587    tcs: RunningTcs,
588    entry: CoEntry,
589}
590
591enum CoEntry {
592    Initial(ErasedTcs, u64, u64, u64, u64, u64),
593    Resume(tcs::Usercall<ErasedTcs>, (u64, u64)),
594}
595
596impl Work {
597    fn do_work(self, io_send_queue: &mut tokio::sync::mpsc::UnboundedSender<UsercallSendData>) {
598        let buf = RefCell::new([0u8; 1024]);
599        let usercall_send_data = match self.entry {
600            CoEntry::Initial(erased_tcs, p1, p2, p3, p4, p5) => {
601                let coresult = tcs::coenter(erased_tcs, p1, p2, p3, p4, p5, Some(&buf));
602                UsercallSendData::Sync(coresult, self.tcs, buf)
603            }
604            CoEntry::Resume(usercall, coresult) => {
605                let coresult = usercall.coreturn(coresult, Some(&buf));
606                UsercallSendData::Sync(coresult, self.tcs, buf)
607            }
608        };
609        // if there is an error do nothing, as it means that the main thread has exited
610        let _ = io_send_queue.send(usercall_send_data);
611    }
612}
613
614enum UsercallEvent {
615    Started(u64, oneshot::Sender<()>),
616    Finished(u64),
617    Cancelled(u64, WritePosition),
618}
619
620trait IgnoreCancel {
621    fn ignore_cancel(&self) -> bool;
622}
623
624impl IgnoreCancel for Identified<Usercall> {
625    fn ignore_cancel(&self) -> bool {
626        self.data.0 != UsercallList::read as u64
627            && self.data.0 != UsercallList::read_alloc as u64
628            && self.data.0 != UsercallList::write as u64
629            && self.data.0 != UsercallList::accept_stream as u64
630            && self.data.0 != UsercallList::connect_stream as u64
631            && self.data.0 != UsercallList::wait as u64
632    }
633}
634
635impl EnclaveState {
636    fn event_queue_add_tcs(
637        event_queues: &mut FnvHashMap<TcsAddress, PendingEvents>,
638        tcs: ErasedTcs,
639    ) -> StoppedTcs {
640        if event_queues
641            .insert(tcs.address(), PendingEvents::new())
642            .is_some()
643        {
644            panic!("duplicate TCS address: {:p}", tcs.address())
645        }
646        StoppedTcs { tcs }
647    }
648
649    fn new(
650        kind: EnclaveKind,
651        mut event_queues: FnvHashMap<TcsAddress, PendingEvents>,
652        stream_router: Box<dyn StreamRouter>,
653        threads_vector: Vec<ErasedTcs>,
654        forward_panics: bool,
655        force_time_usercalls: bool,
656    ) -> Arc<Self> {
657        const ABI_BASIC_FDS: [u64; 3] = [FD_STDIN, FD_STDOUT, FD_STDERR];
658
659        let mut fds = FnvHashMap::default();
660
661        for (fd, stream) in stream_router.basic_streams().into_iter().enumerate() {
662            fds.insert(
663                ABI_BASIC_FDS.get(fd).cloned().unwrap_or(fd as _),
664                Arc::new(AsyncFileDesc::stream(stream)),
665            );
666        }
667
668        let last_fd = AtomicUsize::new(fds.keys().cloned().max().unwrap() as _);
669
670        let threads_queue = crossbeam::queue::SegQueue::new();
671
672        for thread in threads_vector {
673            threads_queue.push(Self::event_queue_add_tcs(&mut event_queues, thread));
674        }
675
676        Arc::new(EnclaveState {
677            kind,
678            event_queues,
679            fds: Mutex::new(fds),
680            last_fd,
681            exiting: AtomicBool::new(false),
682            stream_router,
683            threads_queue,
684            forward_panics,
685            force_time_usercalls,
686            fifo_guards: Mutex::new(None),
687            return_queue_tx: Mutex::new(None),
688        })
689    }
690
691    async fn handle_usercall(
692        enclave: Arc<EnclaveState>,
693        work_sender: crossbeam::channel::Sender<Work>,
694        tx_return_channel: tokio::sync::mpsc::UnboundedSender<(EnclaveResult, ReturnSource)>,
695        mut handle_data: UsercallHandleData,
696    ) {
697        let notifier_rx = match handle_data {
698            UsercallHandleData::Async(_, ref mut notifier_rx, _) => notifier_rx.take(),
699            _ => None,
700        };
701        let (parameters, mode, tcs) = match handle_data {
702            UsercallHandleData::Sync(ref usercall, ref mut tcs, _) => {
703                (usercall.parameters(), tcs.mode.into(), Some(tcs))
704            }
705            UsercallHandleData::Async(ref usercall, _, _) => {
706                (usercall.data.into(), ReturnSource::AsyncUsercall, None)
707            }
708        };
709        let mut input = IOHandlerInput {
710            enclave: enclave.clone(),
711            tcs,
712            work_sender: &work_sender,
713        };
714        let handler = Handler(&mut input);
715        let result = {
716            let (p1, p2, p3, p4, p5) = parameters;
717            match notifier_rx {
718                None => dispatch(handler, p1, p2, p3, p4, p5).await.1,
719                Some(notifier_rx) => {
720                    let a = dispatch(handler, p1, p2, p3, p4, p5).boxed_local();
721                    let b = notifier_rx;
722                    match futures::future::select(a, b).await {
723                        Either::Left((ret, _)) => ret.1,
724                        Either::Right((Ok(()), _)) => {
725                            let result: IoResult<usize> = Err(IoErrorKind::Interrupted.into());
726                            ReturnValue::into_registers(Ok(result.to_sgx_result()))
727                        }
728                        Either::Right((Err(_), _)) => {
729                            panic!("notifier channel closed unexpectedly")
730                        }
731                    }
732                }
733            }
734        };
735        let ret = match result {
736            Ok(ret) => {
737                match handle_data {
738                    UsercallHandleData::Sync(usercall, tcs, _) => {
739                        work_sender
740                            .send(Work {
741                                tcs,
742                                entry: CoEntry::Resume(usercall, ret),
743                            })
744                            .expect("Work sender couldn't send data to receiver");
745                    }
746                    UsercallHandleData::Async(usercall, _, usercall_event_tx) => {
747                        if let Some(usercall_event_tx) = usercall_event_tx {
748                            usercall_event_tx
749                                .send(UsercallEvent::Finished(usercall.id))
750                                .ok()
751                                .expect("failed to send usercall event");
752                        }
753                        let return_queue_tx = enclave
754                            .return_queue_tx
755                            .lock()
756                            .await
757                            .clone()
758                            .expect("return_queue_tx not initialized");
759                        let ret = Identified {
760                            id: usercall.id,
761                            data: Return(ret.0, ret.1),
762                        };
763                        return_queue_tx.send(ret).await.unwrap();
764                    }
765                }
766                return;
767            }
768            Err(EnclaveAbort::Exit { panic: true }) => {
769                let panic = match handle_data {
770                    UsercallHandleData::Sync(usercall, _, debug_buf) => {
771                        let debug_buf = debug_buf.into_inner();
772                        #[cfg(unix)]
773                        {
774                            eprintln!("Attaching debugger");
775                            trap_attached_debugger(usercall.tcs_address() as _, debug_buf.as_ptr())
776                                .await;
777                        }
778                        EnclavePanic::from(debug_buf)
779                    }
780                    UsercallHandleData::Async(_, _, _) => {
781                        // TODO: https://github.com/fortanix/rust-sgx/issues/235#issuecomment-641811437
782                        EnclavePanic::DebugStr("async exit with a panic".to_owned())
783                    }
784                };
785                Err(EnclaveAbort::Exit { panic: Some(panic) })
786            }
787            Err(EnclaveAbort::Exit { panic: false }) => Err(EnclaveAbort::Exit { panic: None }),
788            Err(EnclaveAbort::IndefiniteWait) => Err(EnclaveAbort::IndefiniteWait),
789            Err(EnclaveAbort::InvalidUsercall(n)) => Err(EnclaveAbort::InvalidUsercall(n)),
790            Err(EnclaveAbort::MainReturned) => Err(EnclaveAbort::MainReturned),
791            Err(EnclaveAbort::Secondary) => Err(EnclaveAbort::Secondary),
792        };
793        let _ = tx_return_channel.send((ret, mode));
794    }
795
796    fn syscall_loop(
797        enclave: Arc<EnclaveState>,
798        mut io_queue_receive: tokio::sync::mpsc::UnboundedReceiver<UsercallSendData>,
799        io_queue_send: tokio::sync::mpsc::UnboundedSender<UsercallSendData>,
800        work_sender: crossbeam::channel::Sender<Work>,
801    ) -> EnclaveResult {
802        let (tx_return_channel, mut rx_return_channel) = tokio::sync::mpsc::unbounded_channel();
803        let enclave_clone = enclave.clone();
804        let mut rt = RuntimeBuilder::new_current_thread()
805            .enable_all()
806            .build()
807            .expect("failed to create tokio Runtime");
808        let local_set = tokio::task::LocalSet::new();
809
810        let return_future = async move {
811            while let Some((my_result, mode)) = rx_return_channel.recv().await {
812                if enclave_clone.forward_panics {
813                    if let Err(EnclaveAbort::Exit { panic: Some(panic) }) = &my_result {
814                        panic!("{}", panic);
815                    }
816                }
817
818                let res = match (my_result, mode) {
819                    (Err(EnclaveAbort::Secondary), _)
820                    | (Ok(_), ReturnSource::ExecutableNonMain) => continue,
821
822                    (e, ReturnSource::Library)
823                    | (e, ReturnSource::ExecutableMain)
824                    | (e @ Err(EnclaveAbort::Exit { panic: None }), _) => e,
825
826                    (Ok(_), ReturnSource::AsyncUsercall)
827                    | (Err(EnclaveAbort::MainReturned), ReturnSource::AsyncUsercall) => {
828                        unreachable!()
829                    }
830
831                    (Err(e @ EnclaveAbort::Exit { panic: Some(_) }), _)
832                    | (Err(e @ EnclaveAbort::InvalidUsercall(_)), _) => {
833                        let e = e.map_panic(|opt| opt.unwrap());
834                        let cmd = enclave_clone.kind.as_command().unwrap();
835                        let mut cmddata = cmd.panic_reason.lock().await;
836
837                        if cmddata.primary_panic_reason.is_none() {
838                            cmddata.primary_panic_reason = Some(e)
839                        } else {
840                            cmddata.other_reasons.push(e)
841                        }
842                        Err(EnclaveAbort::Secondary)
843                    }
844
845                    (Err(e), _) => {
846                        let e = e.map_panic(|opt| opt.unwrap());
847                        let cmd = enclave_clone.kind.as_command().unwrap();
848                        let mut cmddata = cmd.panic_reason.lock().await;
849                        cmddata.other_reasons.push(e);
850                        continue;
851                    }
852                };
853                return res;
854            }
855            unreachable!();
856        };
857        let enclave_clone = enclave.clone();
858        let io_future = async move {
859            let (uqs, rqs, cqs, sync_usercall_tx) = QueueSynchronizer::new(enclave_clone.clone());
860
861            let (usercall_queue_tx, usercall_queue_rx) =
862                ipc_queue::bounded_async(USERCALL_QUEUE_SIZE, uqs);
863            let (return_queue_tx, return_queue_rx) =
864                ipc_queue::bounded_async(RETURN_QUEUE_SIZE, rqs);
865            let (cancel_queue_tx, cancel_queue_rx) =
866                ipc_queue::bounded_async(CANCEL_QUEUE_SIZE, cqs);
867
868            let fifo_guards = FifoGuards {
869                usercall_queue: usercall_queue_tx.into_descriptor_guard(),
870                return_queue: return_queue_rx.into_descriptor_guard(),
871                cancel_queue: cancel_queue_tx.into_descriptor_guard(),
872                async_queues_called: false,
873            };
874
875            *enclave_clone.fifo_guards.lock().await = Some(fifo_guards);
876            *enclave_clone.return_queue_tx.lock().await = Some(return_queue_tx);
877
878            let usercall_queue_monitor = usercall_queue_rx.position_monitor();
879
880            let (usercall_event_tx, mut usercall_event_rx) = async_mpsc::unbounded_channel();
881            let usercall_event_tx_clone = usercall_event_tx.clone();
882            tokio::task::spawn_local(async move {
883                while let Ok(usercall) = usercall_queue_rx.recv().await {
884                    let notifier_rx = if usercall.ignore_cancel() {
885                        None
886                    } else {
887                        let (notifier_tx, notifier_rx) = oneshot::channel();
888                        usercall_event_tx_clone
889                            .send(UsercallEvent::Started(usercall.id, notifier_tx))
890                            .ok()
891                            .expect("failed to send usercall event");
892                        Some(notifier_rx)
893                    };
894                    let _ = io_queue_send.send(UsercallSendData::Async(usercall, notifier_rx));
895                }
896            });
897
898            let usercall_event_tx_clone = usercall_event_tx.clone();
899            let usercall_queue_monitor_clone = usercall_queue_monitor.clone();
900            tokio::task::spawn_local(async move {
901                while let Ok(c) = cancel_queue_rx.recv().await {
902                    let write_position = usercall_queue_monitor_clone.write_position();
903                    let _ = usercall_event_tx_clone
904                        .send(UsercallEvent::Cancelled(c.id, write_position));
905                }
906            });
907
908            tokio::task::spawn_local(async move {
909                let mut notifiers = HashMap::new();
910                let mut cancels: HashMap<u64, WritePosition> = HashMap::new();
911                loop {
912                    match usercall_event_rx
913                        .recv()
914                        .await
915                        .expect("usercall_event channel closed unexpectedly")
916                    {
917                        UsercallEvent::Started(id, notifier) => match cancels.remove(&id) {
918                            Some(_) => {
919                                let _ = notifier.send(());
920                            }
921                            _ => {
922                                notifiers.insert(id, notifier);
923                            }
924                        },
925                        UsercallEvent::Finished(id) => {
926                            notifiers.remove(&id);
927                        }
928                        UsercallEvent::Cancelled(id, wp) => match notifiers.remove(&id) {
929                            Some(notifier) => {
930                                let _ = notifier.send(());
931                            }
932                            None => {
933                                cancels.insert(id, wp);
934                            }
935                        },
936                    }
937                    // cleanup old cancels
938                    let read_position = usercall_queue_monitor.read_position();
939                    cancels.retain(|_id, wp| {
940                        if let Some(past) = read_position.is_past(wp) {
941                            !past
942                        } else {
943                            false
944                        }
945                    });
946                }
947            });
948
949            while let Some(work) = io_queue_receive.recv().await {
950                let enclave_clone = enclave_clone.clone();
951                let tx_return_channel = tx_return_channel.clone();
952                match work {
953                    UsercallSendData::Async(usercall, notifier_rx) => {
954                        let usercall_event_tx = if usercall.ignore_cancel() {
955                            None
956                        } else {
957                            Some(usercall_event_tx.clone())
958                        };
959                        let uchd =
960                            UsercallHandleData::Async(usercall, notifier_rx, usercall_event_tx);
961                        let fut = Self::handle_usercall(
962                            enclave_clone,
963                            work_sender.clone(),
964                            tx_return_channel,
965                            uchd,
966                        );
967                        tokio::task::spawn_local(fut);
968                    }
969                    UsercallSendData::Sync(CoResult::Yield(usercall), state, buf) => {
970                        let _ = sync_usercall_tx.send(());
971                        let uchd = UsercallHandleData::Sync(usercall, state, buf);
972                        let fut = Self::handle_usercall(
973                            enclave_clone,
974                            work_sender.clone(),
975                            tx_return_channel,
976                            uchd,
977                        );
978                        tokio::task::spawn_local(fut);
979                    }
980                    UsercallSendData::Sync(CoResult::Return((tcs, v1, v2)), state, _buf) => {
981                        let fut = async move {
982                            let ret = match state.mode {
983                                EnclaveEntry::Library => {
984                                    enclave_clone.threads_queue.push(StoppedTcs { tcs });
985                                    Ok((v1, v2))
986                                }
987                                EnclaveEntry::ExecutableMain => Err(EnclaveAbort::MainReturned),
988                                EnclaveEntry::ExecutableNonMain => {
989                                    assert_eq!(
990                                        (v1, v2),
991                                        (0, 0),
992                                        "Expected enclave thread entrypoint to return zero"
993                                    );
994                                    // If the enclave is in the exit-state, threads are no
995                                    // longer able to be launched
996                                    if !enclave_clone.exiting.load(Ordering::SeqCst) {
997                                        enclave_clone.threads_queue.push(StoppedTcs { tcs });
998                                    }
999                                    Ok((0, 0))
1000                                }
1001                            };
1002                            let _ = tx_return_channel.send((ret, state.mode.into()));
1003                        };
1004                        tokio::task::spawn_local(fut);
1005                    }
1006                };
1007            }
1008            unreachable!();
1009        };
1010
1011        // Note that:
1012        // - io_future will never return, its job is to spawn new futures that handle I/O.
1013        // - return_future returns in certain cases (see above) and in such cases we want to
1014        //   terminate the syscall loop.
1015        let select_fut =
1016            futures::future::select(return_future.boxed_local(), io_future.boxed_local()).map(
1017                |either| match either {
1018                    Either::Left((x, _)) => x,
1019                    _ => unreachable!(),
1020                },
1021            );
1022
1023        local_set
1024            .block_on(&mut rt, select_fut.unit_error())
1025            .unwrap()
1026    }
1027
1028    fn run(
1029        enclave: Arc<EnclaveState>,
1030        num_of_worker_threads: usize,
1031        start_work: Work,
1032    ) -> EnclaveResult {
1033        fn create_worker_threads(
1034            num_of_worker_threads: usize,
1035            work_receiver: crossbeam::channel::Receiver<Work>,
1036            io_queue_send: tokio::sync::mpsc::UnboundedSender<UsercallSendData>,
1037        ) -> Vec<JoinHandle<()>> {
1038            let mut thread_handles = vec![];
1039            for _ in 0..num_of_worker_threads {
1040                let work_receiver = work_receiver.clone();
1041                let mut io_queue_send = io_queue_send.clone();
1042
1043                thread_handles.push(thread::spawn(move || {
1044                    while let Ok(work) = work_receiver.recv() {
1045                        work.do_work(&mut io_queue_send);
1046                    }
1047                }));
1048            }
1049            thread_handles
1050        }
1051
1052        let (io_queue_send, io_queue_receive) = tokio::sync::mpsc::unbounded_channel();
1053
1054        let (work_sender, work_receiver) = crossbeam::channel::unbounded();
1055        work_sender
1056            .send(start_work)
1057            .expect("Work sender couldn't send data to receiver");
1058
1059        let join_handlers =
1060            create_worker_threads(num_of_worker_threads, work_receiver, io_queue_send.clone());
1061        // main syscall polling loop
1062        let main_result = EnclaveState::syscall_loop(
1063            enclave.clone(),
1064            io_queue_receive,
1065            io_queue_send,
1066            work_sender,
1067        );
1068
1069        for handler in join_handlers {
1070            let _ = handler.join();
1071        }
1072
1073        main_result
1074    }
1075
1076    pub(crate) fn main_entry(
1077        main: ErasedTcs,
1078        threads: Vec<ErasedTcs>,
1079        stream_router: Box<dyn StreamRouter>,
1080        forward_panics: bool,
1081        force_time_usercalls: bool,
1082        cmd_configuration: CommandConfiguration,
1083    ) -> StdResult<(), anyhow::Error> {
1084        let num_of_worker_threads = cmd_configuration.num_worker_threads;
1085        assert!(num_of_worker_threads > 0, "worker_threads cannot be zero");
1086        let mut event_queues =
1087            FnvHashMap::with_capacity_and_hasher(threads.len() + 1, Default::default());
1088        let main = Self::event_queue_add_tcs(&mut event_queues, main);
1089
1090        let args: Vec<_> = cmd_configuration
1091            .cmd_args
1092            .into_iter()
1093            .map(|a| ByteBuffer {
1094                len: a.len(),
1095                data: Box::into_raw(a.into_boxed_slice()) as *const u8,
1096            })
1097            .collect();
1098        let argc = args.len();
1099        let argv = Box::into_raw(args.into_boxed_slice()) as *const u8;
1100
1101        let main_work = Work {
1102            tcs: RunningTcs {
1103                tcs_address: main.tcs.address(),
1104                mode: EnclaveEntry::ExecutableMain,
1105            },
1106            entry: CoEntry::Initial(main.tcs, argv as _, argc as _, 0, 0, 0),
1107        };
1108
1109        let kind = EnclaveKind::Command(Command {
1110            panic_reason: Mutex::new(PanicReason {
1111                primary_panic_reason: None,
1112                other_reasons: vec![],
1113            }),
1114        });
1115        let enclave = EnclaveState::new(
1116            kind,
1117            event_queues,
1118            stream_router,
1119            threads,
1120            forward_panics,
1121            force_time_usercalls,
1122        );
1123
1124        let main_result = EnclaveState::run(enclave.clone(), num_of_worker_threads, main_work);
1125
1126        let main_panicking = match main_result {
1127            Err(EnclaveAbort::MainReturned)
1128            | Err(EnclaveAbort::InvalidUsercall(_))
1129            | Err(EnclaveAbort::Exit { .. }) => true,
1130            Err(EnclaveAbort::IndefiniteWait) | Err(EnclaveAbort::Secondary) | Ok(_) => false,
1131        };
1132
1133        let rt = tokio::runtime::Builder::new_current_thread()
1134            .enable_all()
1135            .build()
1136            .expect("failed to create tokio Runtime");
1137
1138        rt.block_on(async move {
1139            enclave.abort_all_threads();
1140            //clear the threads_queue
1141            while enclave.threads_queue.pop().is_some() {}
1142
1143            let cmd = enclave.kind.as_command().unwrap();
1144            let mut cmddata = cmd.panic_reason.lock().await;
1145            let main_result = match (main_panicking, cmddata.primary_panic_reason.take()) {
1146                (false, Some(reason)) => Err(reason.map_panic(Some)),
1147                // TODO: interpret other_reasons
1148                _ => main_result,
1149            };
1150            match main_result {
1151                Err(EnclaveAbort::Exit { panic: None }) => Ok(()),
1152                Err(EnclaveAbort::Exit { panic: Some(panic) }) => Err(panic.into()),
1153                Err(EnclaveAbort::IndefiniteWait) => {
1154                    bail!("All enclave threads are waiting indefinitely without possibility of wakeup")
1155                }
1156                Err(EnclaveAbort::InvalidUsercall(n)) => {
1157                    bail!("The enclave performed an invalid usercall 0x{:x}", n)
1158                }
1159                Err(EnclaveAbort::MainReturned) => bail!(
1160                    "The enclave returned from the main entrypoint in violation of the specification."
1161                ),
1162                // Should always be able to return the real exit reason
1163                Err(EnclaveAbort::Secondary) => unreachable!(),
1164                Ok(_) => Ok(()),
1165            }
1166        }.boxed_local())
1167    }
1168
1169    pub(crate) fn library(
1170        threads: Vec<ErasedTcs>,
1171        stream_router: Box<dyn StreamRouter>,
1172        forward_panics: bool,
1173        force_time_usercalls: bool,
1174    ) -> Arc<Self> {
1175        let event_queues = FnvHashMap::with_capacity_and_hasher(threads.len(), Default::default());
1176
1177        let kind = EnclaveKind::Library(Library {});
1178
1179        let enclave = EnclaveState::new(
1180            kind,
1181            event_queues,
1182            stream_router,
1183            threads,
1184            forward_panics,
1185            force_time_usercalls,
1186        );
1187        return enclave;
1188    }
1189
1190    pub(crate) fn library_entry(
1191        enclave: &Arc<Self>,
1192        p1: u64,
1193        p2: u64,
1194        p3: u64,
1195        p4: u64,
1196        p5: u64,
1197    ) -> StdResult<(u64, u64), anyhow::Error> {
1198        let thread = enclave.threads_queue.pop().expect("threads queue empty");
1199        let work = Work {
1200            tcs: RunningTcs {
1201                tcs_address: thread.tcs.address(),
1202                mode: EnclaveEntry::Library,
1203            },
1204            entry: CoEntry::Initial(thread.tcs, p1, p2, p3, p4, p5),
1205        };
1206        // As currently we are not supporting spawning threads let the number of threads be 2
1207        // one for usercall handling the other for actually running
1208        let num_of_worker_threads = 1;
1209
1210        let library_result = EnclaveState::run(enclave.clone(), num_of_worker_threads, work);
1211
1212        match library_result {
1213            Err(EnclaveAbort::Exit { panic: None }) => bail!("This thread aborted execution"),
1214            Err(EnclaveAbort::Exit { panic: Some(panic) }) => Err(panic.into()),
1215            Err(EnclaveAbort::IndefiniteWait) => {
1216                bail!("This thread is waiting indefinitely without possibility of wakeup")
1217            }
1218            Err(EnclaveAbort::InvalidUsercall(n)) => {
1219                bail!("The enclave performed an invalid usercall 0x{:x}", n)
1220            }
1221            Err(EnclaveAbort::Secondary) => {
1222                bail!("This thread exited because another thread aborted")
1223            }
1224            Err(EnclaveAbort::MainReturned) => unreachable!(),
1225            Ok(result) => Ok(result),
1226        }
1227    }
1228
1229    fn abort_all_threads(&self) {
1230        self.exiting.store(true, Ordering::SeqCst);
1231        // wake other threads
1232        for pending_events in self.event_queues.values() {
1233            pending_events.abort();
1234        }
1235    }
1236}
1237
1238#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1239enum EnclaveEntry {
1240    ExecutableMain,
1241    ExecutableNonMain,
1242    Library,
1243}
1244
1245#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1246enum ReturnSource {
1247    ExecutableMain,
1248    ExecutableNonMain,
1249    Library,
1250    AsyncUsercall,
1251}
1252
1253impl From<EnclaveEntry> for ReturnSource {
1254    fn from(e: EnclaveEntry) -> Self {
1255        match e {
1256            EnclaveEntry::ExecutableMain => ReturnSource::ExecutableMain,
1257            EnclaveEntry::ExecutableNonMain => ReturnSource::ExecutableNonMain,
1258            EnclaveEntry::Library => ReturnSource::Library,
1259        }
1260    }
1261}
1262
1263#[repr(C)]
1264#[allow(unused)]
1265enum Greg {
1266    R8 = 0,
1267    R9,
1268    R10,
1269    R11,
1270    R12,
1271    R13,
1272    R14,
1273    R15,
1274    RDI,
1275    RSI,
1276    RBP,
1277    RBX,
1278    RDX,
1279    RAX,
1280    RCX,
1281    RSP,
1282    RIP,
1283    EFL,
1284    CSGSFS, /* Actually short cs, gs, fs, __pad0. */
1285    ERR,
1286    TRAPNO,
1287    OLDMASK,
1288    CR2,
1289}
1290
1291#[cfg(unix)]
1292/* Here we are passing control to debugger `fixup` style by raising Sigtrap.
1293 * If there is no debugger attached, this function, would skip the `int3` instructon
1294 * and resume execution.
1295 */
1296extern "C" fn handle_trap(_signo: c_int, _info: *mut siginfo_t, context: *mut c_void) {
1297    unsafe {
1298        let context = &mut *(context as *mut ucontext_t);
1299        let rip = &mut context.uc_mcontext.gregs[Greg::RIP as usize];
1300        let inst: *const u8 = *rip as _;
1301        if *inst == 0xcc {
1302            *rip += 1;
1303        }
1304    }
1305    return;
1306}
1307
1308#[cfg(unix)]
1309/* Raising Sigtrap to allow debugger to take control.
1310 * Here, we also store tcs in rbx, so that the debugger could read it, to
1311 * set sgx state and correctly map the enclave symbols.
1312 */
1313async fn trap_attached_debugger(tcs: usize, debug_buf: *const u8) {
1314    let _g = DEBUGGER_TOGGLE_SYNC.lock().await;
1315    let hdl = self::signal::SigHandler::SigAction(handle_trap);
1316    let sig_action = signal::SigAction::new(hdl, signal::SaFlags::empty(), signal::SigSet::empty());
1317    // Synchronized
1318    unsafe {
1319        let old = signal::sigaction(signal::SIGTRAP, &sig_action).unwrap();
1320        std::arch::asm!("
1321            xchg %rbx, {0}
1322            int3
1323            xchg {0}, %rbx
1324            ",
1325            in(reg) tcs, // rbx is used internally by LLVM and cannot be used as an operand for inline asm (#84658)
1326            in("r10") debug_buf,
1327            options(nomem, nostack, att_syntax));
1328        signal::sigaction(signal::SIGTRAP, &old).unwrap();
1329    }
1330}
1331
1332impl<'tcs> IOHandlerInput<'tcs> {
1333    async fn lookup_fd(&self, fd: Fd) -> IoResult<Arc<AsyncFileDesc>> {
1334        match self.enclave.fds.lock().await.get(&fd) {
1335            Some(stream) => Ok(stream.clone()),
1336            None => Err(IoErrorKind::BrokenPipe.into()), // FIXME: Rust normally maps Unix EBADF to `Other`
1337        }
1338    }
1339
1340    async fn alloc_fd(&self, stream: AsyncFileDesc) -> Fd {
1341        let fd = (self
1342            .enclave
1343            .last_fd
1344            .fetch_add(1, Ordering::Relaxed)
1345            .checked_add(1)
1346            .expect("FD overflow")) as Fd;
1347        let prev = self.enclave.fds.lock().await.insert(fd, Arc::new(stream));
1348        debug_assert!(prev.is_none());
1349        fd
1350    }
1351
1352    #[inline(always)]
1353    fn is_exiting(&self) -> bool {
1354        self.enclave.exiting.load(Ordering::SeqCst)
1355    }
1356
1357    #[inline(always)]
1358    async fn read(&self, fd: Fd, buf: &mut ReadBuf<'_>) -> IoResult<()> {
1359        let file_desc = self.lookup_fd(fd).await?;
1360        file_desc.as_stream()?.async_read(buf).await
1361    }
1362
1363    #[inline(always)]
1364    async fn read_alloc(&self, fd: Fd, buf: &mut OutputBuffer<'tcs, Vec<u8>>) -> IoResult<()> {
1365        let file_desc = self.lookup_fd(fd).await?;
1366        **buf = Some(file_desc.as_stream()?.async_read_alloc().await?);
1367        Ok(())
1368    }
1369
1370    #[inline(always)]
1371    async fn write(&self, fd: Fd, buf: &[u8]) -> IoResult<usize> {
1372        let file_desc = self.lookup_fd(fd).await?;
1373        return file_desc.as_stream()?.async_write(buf).await;
1374    }
1375
1376    #[inline(always)]
1377    async fn flush(&self, fd: Fd) -> IoResult<()> {
1378        let file_desc = self.lookup_fd(fd).await?;
1379        file_desc.as_stream()?.async_flush().await
1380    }
1381
1382    #[inline(always)]
1383    async fn close(&self, fd: Fd) {
1384        self.enclave.fds.lock().await.remove(&fd);
1385    }
1386
1387    #[inline(always)]
1388    async fn bind_stream(
1389        &self,
1390        addr: &[u8],
1391        local_addr: Option<&mut OutputBuffer<'tcs, String>>,
1392    ) -> IoResult<Fd> {
1393        let addr = str::from_utf8(addr).map_err(|_| IoErrorKind::ConnectionRefused)?;
1394
1395        let socket = self
1396            .enclave
1397            .stream_router
1398            .bind_stream(addr, local_addr.map(OutputBuffer::init_default))
1399            .await?;
1400
1401        Ok(self.alloc_fd(AsyncFileDesc::listener(socket)).await)
1402    }
1403
1404    #[inline(always)]
1405    async fn accept_stream(
1406        &self,
1407        fd: Fd,
1408        local_addr: Option<&mut OutputBuffer<'tcs, String>>,
1409        peer_addr: Option<&mut OutputBuffer<'tcs, String>>,
1410    ) -> IoResult<Fd> {
1411        let file_desc = self.lookup_fd(fd).await?;
1412
1413        let stream = file_desc
1414            .as_listener()?
1415            .async_accept(
1416                local_addr.map(OutputBuffer::init_default),
1417                peer_addr.map(OutputBuffer::init_default),
1418            )
1419            .await?;
1420
1421        Ok(self.alloc_fd(AsyncFileDesc::stream(stream)).await)
1422    }
1423
1424    #[inline(always)]
1425    async fn connect_stream(
1426        &self,
1427        addr: &[u8],
1428        local_addr: Option<&mut OutputBuffer<'tcs, String>>,
1429        peer_addr: Option<&mut OutputBuffer<'tcs, String>>,
1430    ) -> IoResult<Fd> {
1431        let addr = str::from_utf8(addr).map_err(|_| IoErrorKind::ConnectionRefused)?;
1432
1433        let socket = self
1434            .enclave
1435            .stream_router
1436            .connect_stream(
1437                addr,
1438                local_addr.map(OutputBuffer::init_default),
1439                peer_addr.map(OutputBuffer::init_default),
1440            )
1441            .await?;
1442
1443        Ok(self.alloc_fd(AsyncFileDesc::stream(socket)).await)
1444    }
1445
1446    #[inline(always)]
1447    fn launch_thread(&self) -> IoResult<()> {
1448        // check if enclave is of type command
1449        self.enclave
1450            .kind
1451            .as_command()
1452            .ok_or(IoErrorKind::InvalidInput)?;
1453        let new_tcs = match self.enclave.threads_queue.pop() {
1454            Some(tcs) => tcs,
1455            None => {
1456                return Err(IoErrorKind::WouldBlock.into());
1457            }
1458        };
1459
1460        let ret = self.work_sender.send(Work {
1461            tcs: RunningTcs {
1462                tcs_address: new_tcs.tcs.address(),
1463                mode: EnclaveEntry::ExecutableNonMain,
1464            },
1465            entry: CoEntry::Initial(new_tcs.tcs, 0, 0, 0, 0, 0),
1466        });
1467        match ret {
1468            Ok(()) => Ok(()),
1469            Err(e) => {
1470                let entry = e.0.entry;
1471                match entry {
1472                    CoEntry::Initial(tcs, _, _, _, _, _) => {
1473                        self.enclave.threads_queue.push(StoppedTcs { tcs });
1474                    }
1475                    _ => unreachable!(),
1476                };
1477                Err(std::io::Error::new(
1478                    IoErrorKind::NotConnected,
1479                    "Work Sender: send error",
1480                ))
1481            }
1482        }
1483    }
1484
1485    #[inline(always)]
1486    fn exit(&mut self, panic: bool) -> EnclaveAbort<bool> {
1487        self.enclave.abort_all_threads();
1488        EnclaveAbort::Exit { panic }
1489    }
1490
1491    fn check_event_set(set: u64) -> IoResult<()> {
1492        const EV_ALL: u64 =
1493            EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK | EV_CANCELQ_NOT_FULL;
1494        if (set & !EV_ALL) != 0 {
1495            return Err(IoErrorKind::InvalidInput.into());
1496        }
1497
1498        assert!((EV_ALL | EV_ABORT) <= u8::MAX.into());
1499        assert!((EV_ALL & EV_ABORT) == 0);
1500        Ok(())
1501    }
1502
1503    #[inline(always)]
1504    async fn wait(&mut self, event_mask: u64, timeout: u64) -> IoResult<u64> {
1505        Self::check_event_set(event_mask)?;
1506
1507        let timeout = match timeout {
1508            WAIT_NO | WAIT_INDEFINITE => timeout,
1509            _ => {
1510                // tokio::time::timeout does not handle large timeout values
1511                // well which may or may not be a bug in tokio, but it seems to
1512                // work ok with timeout values smaller than 2 ^ 55 nanoseconds.
1513                // NOTE: 2 ^ 55 nanoseconds is roughly 417 days.
1514                // TODO: revisit when https://github.com/tokio-rs/tokio/issues/2667 is resolved.
1515                cmp::min(1 << 55, timeout)
1516            }
1517        };
1518
1519        // TODO: https://github.com/fortanix/rust-sgx/issues/290
1520        let tcs = self
1521            .tcs
1522            .as_mut()
1523            .ok_or(io::Error::from(io::ErrorKind::Other))?;
1524
1525        let pending_events = self
1526            .enclave
1527            .event_queues
1528            .get(&tcs.tcs_address)
1529            .expect("invalid tcs address");
1530
1531        let ret = match timeout {
1532            WAIT_NO => pending_events.take(event_mask),
1533            WAIT_INDEFINITE => Some(pending_events.wait_for(event_mask).await),
1534            n => tokio::time::timeout(Duration::from_nanos(n), pending_events.wait_for(event_mask))
1535                .await
1536                .ok(),
1537        };
1538
1539        if let Some(ev) = ret {
1540            if (ev & EV_ABORT) != 0 {
1541                // dispatch will make sure this is not returned to enclave
1542                return Err(IoErrorKind::Other.into());
1543            }
1544            return Ok(ev.into());
1545        }
1546        Err(if timeout == WAIT_NO {
1547            IoErrorKind::WouldBlock
1548        } else {
1549            IoErrorKind::TimedOut
1550        }
1551        .into())
1552    }
1553
1554    #[inline(always)]
1555    fn send(&self, event_set: u64, target: Option<Tcs>) -> IoResult<()> {
1556        Self::check_event_set(event_set)?;
1557
1558        if event_set == 0 {
1559            return Err(IoErrorKind::InvalidInput.into());
1560        }
1561
1562        if let Some(tcs) = target {
1563            let tcs = TcsAddress(tcs.as_ptr() as _);
1564            let pending_events = self
1565                .enclave
1566                .event_queues
1567                .get(&tcs)
1568                .ok_or(IoErrorKind::InvalidInput)?;
1569            pending_events.push(event_set);
1570        } else {
1571            for pending_events in self.enclave.event_queues.values() {
1572                pending_events.push(event_set);
1573            }
1574        }
1575
1576        Ok(())
1577    }
1578
1579    #[inline(always)]
1580    fn insecure_time(&mut self) -> (u64, *const InsecureTimeInfo) {
1581        let time = time::SystemTime::now()
1582            .duration_since(time::UNIX_EPOCH)
1583            .unwrap();
1584        let t = (time.subsec_nanos() as u64) + time.as_secs() * NANOS_PER_SEC;
1585        let insecure_time_ref: Option<&'static InsecureTimeInfo> = (&*TIME_INFO).as_ref();
1586        let info =
1587            if let (Some(info), false) = (insecure_time_ref, self.enclave.force_time_usercalls) {
1588                info
1589            } else {
1590                ptr::null()
1591            };
1592        (t, info)
1593    }
1594
1595    #[inline(always)]
1596    fn alloc(&self, size: usize, alignment: usize) -> IoResult<*mut u8> {
1597        unsafe {
1598            let layout =
1599                Layout::from_size_align(size, alignment).map_err(|_| IoErrorKind::InvalidInput)?;
1600            if layout.size() == 0 {
1601                return Err(IoErrorKind::InvalidInput.into());
1602            }
1603            let ptr = System.alloc(layout);
1604            if ptr.is_null() {
1605                Err(IoErrorKind::Other.into())
1606            } else {
1607                Ok(ptr)
1608            }
1609        }
1610    }
1611
1612    #[inline(always)]
1613    fn free(&self, ptr: *mut u8, size: usize, alignment: usize) -> IoResult<()> {
1614        unsafe {
1615            let layout =
1616                Layout::from_size_align(size, alignment).map_err(|_| IoErrorKind::InvalidInput)?;
1617            if size == 0 {
1618                return Ok(());
1619            }
1620            Ok(System.dealloc(ptr, layout))
1621        }
1622    }
1623
1624    #[inline(always)]
1625    async fn async_queues(
1626        &mut self,
1627        usercall_queue: &mut FifoDescriptor<Usercall>,
1628        return_queue: &mut FifoDescriptor<Return>,
1629        cancel_queue: Option<&mut FifoDescriptor<Cancel>>,
1630    ) -> StdResult<(), EnclaveAbort<bool>> {
1631        let mut fifo_guards = self.enclave.fifo_guards.lock().await;
1632        match &mut *fifo_guards {
1633            Some(ref mut fifo_guards) if !fifo_guards.async_queues_called => {
1634                *usercall_queue = fifo_guards.usercall_queue.fifo_descriptor();
1635                *return_queue = fifo_guards.return_queue.fifo_descriptor();
1636                if let Some(cancel_queue) = cancel_queue {
1637                    *cancel_queue = fifo_guards.cancel_queue.fifo_descriptor();
1638                }
1639                fifo_guards.async_queues_called = true;
1640                Ok(())
1641            }
1642            Some(_) => {
1643                drop(fifo_guards);
1644                Err(self.exit(true))
1645            }
1646            // Enclave would not be able to call `async_queues()` before the
1647            // fifo_guards is set to Some in `fn syscall_loop`.
1648            None => unreachable!(),
1649        }
1650    }
1651}
1652
1653#[derive(Clone, Copy)]
1654enum Queue {
1655    Usercall,
1656    Return,
1657    Cancel,
1658}
1659
1660struct QueueSynchronizer {
1661    queue: Queue,
1662    enclave: Arc<EnclaveState>,
1663    // the Mutex is uncontested and is used for providing interior mutability.
1664    subscription: Mutex<broadcast::Receiver<()>>,
1665    // this is only used to create a new Receiver in the Clone implementation
1666    subscription_maker: broadcast::Sender<()>,
1667}
1668
1669impl QueueSynchronizer {
1670    fn new(enclave: Arc<EnclaveState>) -> (Self, Self, Self, broadcast::Sender<()>) {
1671        // This broadcast channel is used to notify enclave-runner of any
1672        // synchronous usercalls made by the enclave for the purpose of
1673        // synchronizing access to usercall and return queues.
1674        // The size of this channel should not matter since recv() can
1675        // return RecvError::Lagged.
1676        let (tx, rx1) = broadcast::channel(1);
1677        let rx2 = tx.subscribe();
1678        let rx3 = tx.subscribe();
1679        let usercall_queue_synchronizer = QueueSynchronizer {
1680            queue: Queue::Usercall,
1681            enclave: enclave.clone(),
1682            subscription: Mutex::new(rx1),
1683            subscription_maker: tx.clone(),
1684        };
1685        let return_queue_synchronizer = QueueSynchronizer {
1686            queue: Queue::Return,
1687            enclave: enclave.clone(),
1688            subscription: Mutex::new(rx2),
1689            subscription_maker: tx.clone(),
1690        };
1691        let cancel_queue_synchronizer = QueueSynchronizer {
1692            queue: Queue::Cancel,
1693            enclave,
1694            subscription: Mutex::new(rx3),
1695            subscription_maker: tx.clone(),
1696        };
1697        (
1698            usercall_queue_synchronizer,
1699            return_queue_synchronizer,
1700            cancel_queue_synchronizer,
1701            tx,
1702        )
1703    }
1704}
1705
1706impl Clone for QueueSynchronizer {
1707    fn clone(&self) -> Self {
1708        Self {
1709            queue: self.queue,
1710            enclave: self.enclave.clone(),
1711            subscription: Mutex::new(self.subscription_maker.subscribe()),
1712            subscription_maker: self.subscription_maker.clone(),
1713        }
1714    }
1715}
1716
1717impl ipc_queue::AsyncSynchronizer for QueueSynchronizer {
1718    fn wait(
1719        &self,
1720        event: QueueEvent,
1721    ) -> Pin<Box<dyn Future<Output = StdResult<(), ipc_queue::SynchronizationError>> + '_>> {
1722        match (self.queue, event) {
1723            (Queue::Usercall, QueueEvent::NotFull) => {
1724                panic!("enclave runner should not send on the usercall queue")
1725            }
1726            (Queue::Cancel, QueueEvent::NotFull) => {
1727                panic!("enclave runner should not send on the cancel queue")
1728            }
1729            (Queue::Return, QueueEvent::NotEmpty) => {
1730                panic!("enclave runner should not receive on the return queue")
1731            }
1732            _ => {}
1733        }
1734        // When userspace needs to wait on a queue, it will park the current thread (or do whatever
1735        // else is appropriate for the synchronization model currently in use by userspace).
1736        // Any synchronous usercall will wake the blocked thread (or otherwise signal that either queue is ready).
1737        async move {
1738            let mut subscription = self.subscription.lock().await;
1739            match subscription.recv().await {
1740                Ok(()) | Err(RecvError::Lagged(_)) => Ok(()),
1741                Err(RecvError::Closed) => Err(ipc_queue::SynchronizationError::ChannelClosed),
1742            }
1743        }
1744        .boxed_local()
1745    }
1746
1747    fn notify(&self, event: QueueEvent) {
1748        let ev = match (self.queue, event) {
1749            (Queue::Usercall, QueueEvent::NotEmpty) => {
1750                panic!("enclave runner should not send on the usercall queue")
1751            }
1752            (Queue::Cancel, QueueEvent::NotEmpty) => {
1753                panic!("enclave runner should not send on the cancel queue")
1754            }
1755            (Queue::Return, QueueEvent::NotFull) => {
1756                panic!("enclave runner should not receive on the return queue")
1757            }
1758            (Queue::Usercall, QueueEvent::NotFull) => EV_USERCALLQ_NOT_FULL,
1759            (Queue::Return, QueueEvent::NotEmpty) => EV_RETURNQ_NOT_EMPTY,
1760            (Queue::Cancel, QueueEvent::NotFull) => EV_CANCELQ_NOT_FULL,
1761        };
1762        // When the enclave needs to wait on a queue, it executes the wait() usercall synchronously,
1763        // specifying EV_USERCALLQ_NOT_FULL, EV_RETURNQ_NOT_EMPTY or EV_CANCELQ_NOT_FULL in the event_mask.
1764        // Userspace will wake any or all threads waiting on the appropriate event when it is triggered.
1765        for pending_events in self.enclave.event_queues.values() {
1766            pending_events.push(ev as _);
1767        }
1768    }
1769}