1use std::alloc::{GlobalAlloc, Layout, System};
8use std::cell::RefCell;
9use std::collections::{HashMap, VecDeque};
10use std::io::{self, ErrorKind as IoErrorKind, Result as IoResult};
11use std::pin::Pin;
12use std::ptr;
13use std::result::Result as StdResult;
14use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
15use std::sync::{Arc, LazyLock};
16use std::task::{Context, Poll, Waker};
17use std::thread::{self, JoinHandle};
18use std::time::{self, Duration};
19use std::{cmp, fmt, str};
20
21use anyhow::bail;
22use enclave_runner::platform::CommandConfiguration;
23use enclave_runner::stream_router::{AsyncListener, AsyncStream, StreamRouter};
24use fnv::FnvHashMap;
25use fortanix_sgx_abi::*;
26use futures::future::{poll_fn, Either, Future, FutureExt};
27use futures::lock::Mutex;
28use insecure_time::{Freq, Rdtscp};
29use ipc_queue::position::WritePosition;
30use ipc_queue::{DescriptorGuard, Identified, QueueEvent};
31use lazy_static::lazy_static;
32#[cfg(unix)]
33use libc::*;
34#[cfg(unix)]
35use nix::sys::signal;
36use sgxs::loader::Tcs as SgxsTcs;
37use tokio::io::ReadBuf;
38use tokio::runtime::Builder as RuntimeBuilder;
39use tokio::sync::broadcast::error::RecvError;
40use tokio::sync::{broadcast, mpsc as async_mpsc, oneshot, Semaphore};
41
42use self::abi::dispatch;
43use self::abi::ReturnValue;
44use self::abi::UsercallList;
45use self::interface::ToSgxResult;
46use self::interface::{Handler, OutputBuffer};
47use crate::loader::{EnclavePanic, ErasedTcs};
48use crate::tcs::{self, CoResult, ThreadResult};
49
50pub(crate) mod abi;
51mod interface;
52
53lazy_static! {
54 static ref DEBUGGER_TOGGLE_SYNC: Mutex<()> = Mutex::new(());
55}
56
57const NANOS_PER_SEC: u64 = 1_000_000_000;
58
59static TIME_INFO: LazyLock<Option<InsecureTimeInfo>> = LazyLock::new(|| {
60 if Rdtscp::is_supported() {
61 if let Ok(frequency) = Freq::get() {
62 return Some(InsecureTimeInfo {
63 version: 0,
64 frequency: frequency.as_u64(),
65 });
66 }
67 }
68 None
69});
70
71const EV_ABORT: u64 = 0b0000_0000_0001_0000;
76
77const USERCALL_QUEUE_SIZE: usize = 16;
78const RETURN_QUEUE_SIZE: usize = 1024;
79const CANCEL_QUEUE_SIZE: usize = USERCALL_QUEUE_SIZE * 2;
80
81enum UsercallSendData {
82 Sync(ThreadResult<ErasedTcs>, RunningTcs, RefCell<[u8; 1024]>),
83 Async(Identified<Usercall>, Option<oneshot::Receiver<()>>),
84}
85
86enum UsercallHandleData {
88 Sync(tcs::Usercall<ErasedTcs>, RunningTcs, RefCell<[u8; 1024]>),
89 Async(
90 Identified<Usercall>,
91 Option<oneshot::Receiver<()>>,
92 Option<async_mpsc::UnboundedSender<UsercallEvent>>,
93 ),
94}
95
96type EnclaveResult = StdResult<(u64, u64), EnclaveAbort<Option<EnclavePanic>>>;
97
98struct AsyncStreamAdapter {
99 stream: Pin<Box<dyn AsyncStream>>,
100 read_queue: VecDeque<Waker>,
101 write_queue: VecDeque<Waker>,
102 flush_queue: VecDeque<Waker>,
103}
104
105fn notify_other_tasks(cx: &mut Context, queue: &mut VecDeque<Waker>) {
106 for task in queue.drain(..) {
107 if !task.will_wake(&cx.waker()) {
108 task.wake();
109 }
110 }
111}
112
113impl AsyncStreamAdapter {
114 fn poll_read_alloc(
115 mut self: Pin<&mut Self>,
116 cx: &mut Context,
117 ) -> Poll<tokio::io::Result<Vec<u8>>> {
118 match self.stream.as_mut().poll_read_alloc(cx) {
119 Poll::Pending => {
120 self.read_queue.push_back(cx.waker().clone());
121 Poll::Pending
122 }
123 Poll::Ready(Ok(ret)) => {
124 notify_other_tasks(cx, &mut self.read_queue);
125 Poll::Ready(Ok(ret))
126 }
127 Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
128 }
129 }
130
131 fn poll_read(
132 mut self: Pin<&mut Self>,
133 cx: &mut Context,
134 buf: &mut ReadBuf,
135 ) -> Poll<tokio::io::Result<()>> {
136 match self.stream.as_mut().poll_read(cx, buf) {
137 Poll::Pending => {
138 self.read_queue.push_back(cx.waker().clone());
139 Poll::Pending
140 }
141 Poll::Ready(Ok(())) => {
142 notify_other_tasks(cx, &mut self.read_queue);
143 Poll::Ready(Ok(()))
144 }
145 Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
146 }
147 }
148
149 fn poll_write(
150 mut self: Pin<&mut Self>,
151 cx: &mut Context,
152 buf: &[u8],
153 ) -> Poll<tokio::io::Result<usize>> {
154 match self.stream.as_mut().poll_write(cx, buf) {
155 Poll::Pending => {
156 self.write_queue.push_back(cx.waker().clone());
157 Poll::Pending
158 }
159 Poll::Ready(Ok(ret)) => {
160 notify_other_tasks(cx, &mut self.write_queue);
161 Poll::Ready(Ok(ret))
162 }
163 Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
164 }
165 }
166
167 fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<tokio::io::Result<()>> {
168 match self.stream.as_mut().poll_flush(cx) {
169 Poll::Pending => {
170 self.flush_queue.push_back(cx.waker().clone());
171 Poll::Pending
172 }
173 Poll::Ready(Ok(ret)) => {
174 notify_other_tasks(cx, &mut self.flush_queue);
175 Poll::Ready(Ok(ret))
176 }
177 Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
178 }
179 }
180}
181
182struct AsyncStreamContainer {
183 inner: Mutex<Pin<Box<AsyncStreamAdapter>>>,
184}
185
186impl AsyncStreamContainer {
187 fn new(s: Box<dyn AsyncStream>) -> Self {
188 AsyncStreamContainer {
189 inner: Mutex::new(Box::pin(AsyncStreamAdapter {
190 stream: s.into(),
191 read_queue: VecDeque::new(),
192 write_queue: VecDeque::new(),
193 flush_queue: VecDeque::new(),
194 })),
195 }
196 }
197
198 async fn async_read(&self, buf: &mut ReadBuf<'_>) -> IoResult<()> {
199 poll_fn(|cx| {
200 let inner_ref = &mut self.inner.lock();
201 let mut inner = Pin::new(inner_ref);
202 match inner.as_mut().poll(cx) {
203 Poll::Ready(mut adapter) => adapter.as_mut().poll_read(cx, buf),
204 Poll::Pending => Poll::Pending,
205 }
206 })
207 .await
208 }
209
210 async fn async_read_alloc(&self) -> IoResult<Vec<u8>> {
211 poll_fn(|cx| {
212 let inner_ref = &mut self.inner.lock();
213 let mut inner = Pin::new(inner_ref);
214 match inner.as_mut().poll(cx) {
215 Poll::Ready(mut adapter) => adapter.as_mut().poll_read_alloc(cx),
216 Poll::Pending => Poll::Pending,
217 }
218 })
219 .await
220 }
221
222 async fn async_write(&self, buf: &[u8]) -> IoResult<usize> {
223 poll_fn(|cx| {
224 let inner_ref = &mut self.inner.lock();
225 let mut inner = Pin::new(inner_ref);
226 match inner.as_mut().poll(cx) {
227 Poll::Ready(mut adapter) => adapter.as_mut().poll_write(cx, buf),
228 Poll::Pending => Poll::Pending,
229 }
230 })
231 .await
232 }
233
234 async fn async_flush(&self) -> IoResult<()> {
235 poll_fn(|cx| {
236 let inner_ref = &mut self.inner.lock();
237 let mut inner = Pin::new(inner_ref);
238 match inner.as_mut().poll(cx) {
239 Poll::Ready(mut adapter) => adapter.as_mut().poll_flush(cx),
240 Poll::Pending => Poll::Pending,
241 }
242 })
243 .await
244 }
245}
246
247struct AsyncListenerAdapter {
248 listener: Pin<Box<dyn AsyncListener>>,
249 accept_queue: VecDeque<Waker>,
250}
251
252impl AsyncListenerAdapter {
253 fn poll_accept(
254 mut self: Pin<&mut Self>,
255 cx: &mut Context,
256 local_addr: Option<&mut String>,
257 peer_addr: Option<&mut String>,
258 ) -> Poll<tokio::io::Result<Box<dyn AsyncStream>>> {
259 match self
260 .listener
261 .as_mut()
262 .poll_accept(cx, local_addr, peer_addr)
263 {
264 Poll::Pending => {
265 self.accept_queue.push_back(cx.waker().clone());
266 Poll::Pending
267 }
268 Poll::Ready(Ok(ret)) => {
269 notify_other_tasks(cx, &mut self.accept_queue);
270 Poll::Ready(Ok(ret))
271 }
272 Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
273 }
274 }
275}
276
277struct AsyncListenerContainer {
278 inner: Mutex<Pin<Box<AsyncListenerAdapter>>>,
279}
280
281impl AsyncListenerContainer {
282 fn new(l: Box<dyn AsyncListener>) -> Self {
283 AsyncListenerContainer {
284 inner: Mutex::new(Pin::new(Box::new(AsyncListenerAdapter {
285 listener: l.into(),
286 accept_queue: VecDeque::new(),
287 }))),
288 }
289 }
290
291 async fn async_accept(
292 &self,
293 local_addr: Option<&mut String>,
294 peer_addr: Option<&mut String>,
295 ) -> IoResult<Box<dyn AsyncStream>> {
296 let mut local_addr_owned: Option<String> = if local_addr.is_some() {
297 Some(String::new())
298 } else {
299 None
300 };
301 let mut peer_addr_owned: Option<String> = if peer_addr.is_some() {
302 Some(String::new())
303 } else {
304 None
305 };
306 let res = poll_fn(|cx| {
307 let inner_ref = &mut self.inner.lock();
308 let mut inner = Pin::new(inner_ref);
309 match inner.as_mut().poll(cx) {
310 Poll::Ready(mut adapter) => adapter.as_mut().poll_accept(
311 cx,
312 local_addr_owned.as_mut(),
313 peer_addr_owned.as_mut(),
314 ),
315 Poll::Pending => Poll::Pending,
316 }
317 })
318 .await;
319
320 if let Some(local_addr) = local_addr {
321 *local_addr = local_addr_owned.unwrap();
322 }
323 if let Some(peer_addr) = peer_addr {
324 *peer_addr = peer_addr_owned.unwrap();
325 }
326 res
327 }
328}
329
330enum AsyncFileDesc {
331 Stream(AsyncStreamContainer),
332 Listener(AsyncListenerContainer),
333}
334
335impl AsyncFileDesc {
336 fn stream(s: Box<dyn AsyncStream>) -> AsyncFileDesc {
337 AsyncFileDesc::Stream(AsyncStreamContainer::new(s))
338 }
339
340 fn listener(l: Box<dyn AsyncListener>) -> AsyncFileDesc {
341 AsyncFileDesc::Listener(AsyncListenerContainer::new(l))
342 }
343
344 fn as_stream(&self) -> IoResult<&AsyncStreamContainer> {
345 if let AsyncFileDesc::Stream(ref s) = self {
346 Ok(s)
347 } else {
348 Err(IoErrorKind::InvalidInput.into())
349 }
350 }
351
352 fn as_listener(&self) -> IoResult<&AsyncListenerContainer> {
353 if let AsyncFileDesc::Listener(ref l) = self {
354 Ok(l)
355 } else {
356 Err(IoErrorKind::InvalidInput.into())
357 }
358 }
359}
360
361#[derive(Debug)]
362pub(crate) enum EnclaveAbort<T> {
363 Exit {
364 panic: T,
365 },
366 Secondary,
368 IndefiniteWait,
369 InvalidUsercall(u64),
370 MainReturned,
371}
372
373impl<T> EnclaveAbort<T> {
374 fn map_panic<U, F: FnOnce(T) -> U>(self, f: F) -> EnclaveAbort<U> {
375 match self {
376 EnclaveAbort::Exit { panic } => EnclaveAbort::Exit { panic: f(panic) },
377 EnclaveAbort::Secondary => EnclaveAbort::Secondary,
378 EnclaveAbort::IndefiniteWait => EnclaveAbort::IndefiniteWait,
379 EnclaveAbort::InvalidUsercall(n) => EnclaveAbort::InvalidUsercall(n),
380 EnclaveAbort::MainReturned => EnclaveAbort::MainReturned,
381 }
382 }
383}
384
385#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)]
386struct TcsAddress(usize);
387
388impl ErasedTcs {
389 fn address(&self) -> TcsAddress {
390 TcsAddress(SgxsTcs::address(self) as _)
391 }
392}
393
394impl fmt::Pointer for TcsAddress {
395 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
396 (self.0 as *const u8).fmt(f)
397 }
398}
399
400struct StoppedTcs {
401 tcs: ErasedTcs,
402}
403
404struct IOHandlerInput<'tcs> {
405 tcs: Option<&'tcs mut RunningTcs>,
406 enclave: Arc<EnclaveState>,
407 work_sender: &'tcs crossbeam::channel::Sender<Work>,
408}
409
410struct PendingEvents {
411 counts: [Semaphore; Self::EV_MAX],
415 abort: Semaphore,
416}
417
418impl PendingEvents {
419 const EV_MAX_U64: u64 =
421 (EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK | EV_CANCELQ_NOT_FULL) + 1;
422 const EV_MAX: usize = Self::EV_MAX_U64 as _;
423 const _ERROR_IF_USIZE_TOO_SMALL: u64 = u64::MAX + (Self::EV_MAX_U64 - (Self::EV_MAX as u64));
425 const _EV_MAX_U16: u16 = Self::EV_MAX_U64 as u16;
426 const _ERROR_IF_TOO_BIG: u64 = u64::MAX + (Self::EV_MAX_U64 - (Self::_EV_MAX_U16 as u64));
428
429 fn new() -> Self {
430 PendingEvents {
431 counts: [
432 Semaphore::new(0),
433 Semaphore::new(0),
434 Semaphore::new(0),
435 Semaphore::new(0),
436 Semaphore::new(0),
437 Semaphore::new(0),
438 Semaphore::new(0),
439 Semaphore::new(0),
440 Semaphore::new(0),
441 Semaphore::new(0),
442 Semaphore::new(0),
443 Semaphore::new(0),
444 Semaphore::new(0),
445 Semaphore::new(0),
446 Semaphore::new(0),
447 Semaphore::new(0),
448 ],
449 abort: Semaphore::new(0),
450 }
451 }
452
453 fn take(&self, event_mask: u64) -> Option<u64> {
454 assert!(event_mask < Self::EV_MAX_U64);
455
456 if let Ok(_) = self.abort.try_acquire() {
457 return Some(EV_ABORT);
458 }
459
460 for i in (1..Self::EV_MAX).rev() {
461 let ev = i as u64;
462 if (ev & event_mask) != 0 {
463 if let Ok(permit) = self.counts[i].try_acquire() {
464 permit.forget();
465 return Some(ev);
466 }
467 }
468 }
469 None
470 }
471
472 async fn wait_for(&self, event_mask: u64) -> u64 {
473 assert!(event_mask < Self::EV_MAX_U64);
474
475 if let Ok(_) = self.abort.try_acquire() {
476 return EV_ABORT;
477 }
478
479 let it = std::iter::once((EV_ABORT, &self.abort))
480 .chain(
481 self.counts
482 .iter()
483 .enumerate()
484 .map(|(ev, sem)| (ev as u64, sem))
485 .filter(|&(ev, _)| ev & event_mask != 0),
486 )
487 .map(|(ev, sem)| sem.acquire().map(move |permit| (ev, permit)).boxed());
488
489 let ((ev, permit), _, _) = futures::future::select_all(it).await;
490
491 if let Ok(_) = self.abort.try_acquire() {
494 return EV_ABORT;
495 }
496 if ev != EV_ABORT {
497 if let Ok(permit) = permit {
498 permit.forget();
499 }
500 }
501 ev
502 }
503
504 fn push(&self, event: u64) {
505 debug_assert!(event != 0 && event < Self::EV_MAX_U64);
506 let index = event as usize;
507 const MAX_PERMITS: usize = usize::MAX >> 5;
515 if self.counts[index].available_permits() < MAX_PERMITS {
516 self.counts[index].add_permits(1);
517 }
518 }
519
520 fn abort(&self) {
521 if self.abort.available_permits() == 0 {
522 self.abort.add_permits(usize::MAX >> 5);
523 }
524 }
525}
526
527struct RunningTcs {
528 tcs_address: TcsAddress,
529 mode: EnclaveEntry,
530}
531
532enum EnclaveKind {
533 Command(Command),
534 Library(Library),
535}
536
537struct PanicReason {
538 primary_panic_reason: Option<EnclaveAbort<EnclavePanic>>,
539 other_reasons: Vec<EnclaveAbort<EnclavePanic>>,
540}
541
542struct Command {
543 panic_reason: Mutex<PanicReason>,
544}
545
546struct Library {}
547
548impl EnclaveKind {
549 fn as_command(&self) -> Option<&Command> {
550 match self {
551 EnclaveKind::Command(c) => Some(c),
552 _ => None,
553 }
554 }
555
556 fn _as_library(&self) -> Option<&Library> {
557 match self {
558 EnclaveKind::Library(l) => Some(l),
559 _ => None,
560 }
561 }
562}
563
564struct FifoGuards {
565 usercall_queue: DescriptorGuard<Usercall>,
566 return_queue: DescriptorGuard<Return>,
567 cancel_queue: DescriptorGuard<Cancel>,
568 async_queues_called: bool,
569}
570
571pub(crate) struct EnclaveState {
572 kind: EnclaveKind,
573 event_queues: FnvHashMap<TcsAddress, PendingEvents>,
574 fds: Mutex<FnvHashMap<Fd, Arc<AsyncFileDesc>>>,
575 last_fd: AtomicUsize,
576 exiting: AtomicBool,
577 stream_router: Box<dyn StreamRouter>,
578 threads_queue: crossbeam::queue::SegQueue<StoppedTcs>,
579 forward_panics: bool,
580 force_time_usercalls: bool,
581 fifo_guards: Mutex<Option<FifoGuards>>,
583 return_queue_tx: Mutex<Option<ipc_queue::AsyncSender<Return, QueueSynchronizer>>>,
584}
585
586struct Work {
587 tcs: RunningTcs,
588 entry: CoEntry,
589}
590
591enum CoEntry {
592 Initial(ErasedTcs, u64, u64, u64, u64, u64),
593 Resume(tcs::Usercall<ErasedTcs>, (u64, u64)),
594}
595
596impl Work {
597 fn do_work(self, io_send_queue: &mut tokio::sync::mpsc::UnboundedSender<UsercallSendData>) {
598 let buf = RefCell::new([0u8; 1024]);
599 let usercall_send_data = match self.entry {
600 CoEntry::Initial(erased_tcs, p1, p2, p3, p4, p5) => {
601 let coresult = tcs::coenter(erased_tcs, p1, p2, p3, p4, p5, Some(&buf));
602 UsercallSendData::Sync(coresult, self.tcs, buf)
603 }
604 CoEntry::Resume(usercall, coresult) => {
605 let coresult = usercall.coreturn(coresult, Some(&buf));
606 UsercallSendData::Sync(coresult, self.tcs, buf)
607 }
608 };
609 let _ = io_send_queue.send(usercall_send_data);
611 }
612}
613
614enum UsercallEvent {
615 Started(u64, oneshot::Sender<()>),
616 Finished(u64),
617 Cancelled(u64, WritePosition),
618}
619
620trait IgnoreCancel {
621 fn ignore_cancel(&self) -> bool;
622}
623
624impl IgnoreCancel for Identified<Usercall> {
625 fn ignore_cancel(&self) -> bool {
626 self.data.0 != UsercallList::read as u64
627 && self.data.0 != UsercallList::read_alloc as u64
628 && self.data.0 != UsercallList::write as u64
629 && self.data.0 != UsercallList::accept_stream as u64
630 && self.data.0 != UsercallList::connect_stream as u64
631 && self.data.0 != UsercallList::wait as u64
632 }
633}
634
635impl EnclaveState {
636 fn event_queue_add_tcs(
637 event_queues: &mut FnvHashMap<TcsAddress, PendingEvents>,
638 tcs: ErasedTcs,
639 ) -> StoppedTcs {
640 if event_queues
641 .insert(tcs.address(), PendingEvents::new())
642 .is_some()
643 {
644 panic!("duplicate TCS address: {:p}", tcs.address())
645 }
646 StoppedTcs { tcs }
647 }
648
649 fn new(
650 kind: EnclaveKind,
651 mut event_queues: FnvHashMap<TcsAddress, PendingEvents>,
652 stream_router: Box<dyn StreamRouter>,
653 threads_vector: Vec<ErasedTcs>,
654 forward_panics: bool,
655 force_time_usercalls: bool,
656 ) -> Arc<Self> {
657 const ABI_BASIC_FDS: [u64; 3] = [FD_STDIN, FD_STDOUT, FD_STDERR];
658
659 let mut fds = FnvHashMap::default();
660
661 for (fd, stream) in stream_router.basic_streams().into_iter().enumerate() {
662 fds.insert(
663 ABI_BASIC_FDS.get(fd).cloned().unwrap_or(fd as _),
664 Arc::new(AsyncFileDesc::stream(stream)),
665 );
666 }
667
668 let last_fd = AtomicUsize::new(fds.keys().cloned().max().unwrap() as _);
669
670 let threads_queue = crossbeam::queue::SegQueue::new();
671
672 for thread in threads_vector {
673 threads_queue.push(Self::event_queue_add_tcs(&mut event_queues, thread));
674 }
675
676 Arc::new(EnclaveState {
677 kind,
678 event_queues,
679 fds: Mutex::new(fds),
680 last_fd,
681 exiting: AtomicBool::new(false),
682 stream_router,
683 threads_queue,
684 forward_panics,
685 force_time_usercalls,
686 fifo_guards: Mutex::new(None),
687 return_queue_tx: Mutex::new(None),
688 })
689 }
690
691 async fn handle_usercall(
692 enclave: Arc<EnclaveState>,
693 work_sender: crossbeam::channel::Sender<Work>,
694 tx_return_channel: tokio::sync::mpsc::UnboundedSender<(EnclaveResult, ReturnSource)>,
695 mut handle_data: UsercallHandleData,
696 ) {
697 let notifier_rx = match handle_data {
698 UsercallHandleData::Async(_, ref mut notifier_rx, _) => notifier_rx.take(),
699 _ => None,
700 };
701 let (parameters, mode, tcs) = match handle_data {
702 UsercallHandleData::Sync(ref usercall, ref mut tcs, _) => {
703 (usercall.parameters(), tcs.mode.into(), Some(tcs))
704 }
705 UsercallHandleData::Async(ref usercall, _, _) => {
706 (usercall.data.into(), ReturnSource::AsyncUsercall, None)
707 }
708 };
709 let mut input = IOHandlerInput {
710 enclave: enclave.clone(),
711 tcs,
712 work_sender: &work_sender,
713 };
714 let handler = Handler(&mut input);
715 let result = {
716 let (p1, p2, p3, p4, p5) = parameters;
717 match notifier_rx {
718 None => dispatch(handler, p1, p2, p3, p4, p5).await.1,
719 Some(notifier_rx) => {
720 let a = dispatch(handler, p1, p2, p3, p4, p5).boxed_local();
721 let b = notifier_rx;
722 match futures::future::select(a, b).await {
723 Either::Left((ret, _)) => ret.1,
724 Either::Right((Ok(()), _)) => {
725 let result: IoResult<usize> = Err(IoErrorKind::Interrupted.into());
726 ReturnValue::into_registers(Ok(result.to_sgx_result()))
727 }
728 Either::Right((Err(_), _)) => {
729 panic!("notifier channel closed unexpectedly")
730 }
731 }
732 }
733 }
734 };
735 let ret = match result {
736 Ok(ret) => {
737 match handle_data {
738 UsercallHandleData::Sync(usercall, tcs, _) => {
739 work_sender
740 .send(Work {
741 tcs,
742 entry: CoEntry::Resume(usercall, ret),
743 })
744 .expect("Work sender couldn't send data to receiver");
745 }
746 UsercallHandleData::Async(usercall, _, usercall_event_tx) => {
747 if let Some(usercall_event_tx) = usercall_event_tx {
748 usercall_event_tx
749 .send(UsercallEvent::Finished(usercall.id))
750 .ok()
751 .expect("failed to send usercall event");
752 }
753 let return_queue_tx = enclave
754 .return_queue_tx
755 .lock()
756 .await
757 .clone()
758 .expect("return_queue_tx not initialized");
759 let ret = Identified {
760 id: usercall.id,
761 data: Return(ret.0, ret.1),
762 };
763 return_queue_tx.send(ret).await.unwrap();
764 }
765 }
766 return;
767 }
768 Err(EnclaveAbort::Exit { panic: true }) => {
769 let panic = match handle_data {
770 UsercallHandleData::Sync(usercall, _, debug_buf) => {
771 let debug_buf = debug_buf.into_inner();
772 #[cfg(unix)]
773 {
774 eprintln!("Attaching debugger");
775 trap_attached_debugger(usercall.tcs_address() as _, debug_buf.as_ptr())
776 .await;
777 }
778 EnclavePanic::from(debug_buf)
779 }
780 UsercallHandleData::Async(_, _, _) => {
781 EnclavePanic::DebugStr("async exit with a panic".to_owned())
783 }
784 };
785 Err(EnclaveAbort::Exit { panic: Some(panic) })
786 }
787 Err(EnclaveAbort::Exit { panic: false }) => Err(EnclaveAbort::Exit { panic: None }),
788 Err(EnclaveAbort::IndefiniteWait) => Err(EnclaveAbort::IndefiniteWait),
789 Err(EnclaveAbort::InvalidUsercall(n)) => Err(EnclaveAbort::InvalidUsercall(n)),
790 Err(EnclaveAbort::MainReturned) => Err(EnclaveAbort::MainReturned),
791 Err(EnclaveAbort::Secondary) => Err(EnclaveAbort::Secondary),
792 };
793 let _ = tx_return_channel.send((ret, mode));
794 }
795
796 fn syscall_loop(
797 enclave: Arc<EnclaveState>,
798 mut io_queue_receive: tokio::sync::mpsc::UnboundedReceiver<UsercallSendData>,
799 io_queue_send: tokio::sync::mpsc::UnboundedSender<UsercallSendData>,
800 work_sender: crossbeam::channel::Sender<Work>,
801 ) -> EnclaveResult {
802 let (tx_return_channel, mut rx_return_channel) = tokio::sync::mpsc::unbounded_channel();
803 let enclave_clone = enclave.clone();
804 let mut rt = RuntimeBuilder::new_current_thread()
805 .enable_all()
806 .build()
807 .expect("failed to create tokio Runtime");
808 let local_set = tokio::task::LocalSet::new();
809
810 let return_future = async move {
811 while let Some((my_result, mode)) = rx_return_channel.recv().await {
812 if enclave_clone.forward_panics {
813 if let Err(EnclaveAbort::Exit { panic: Some(panic) }) = &my_result {
814 panic!("{}", panic);
815 }
816 }
817
818 let res = match (my_result, mode) {
819 (Err(EnclaveAbort::Secondary), _)
820 | (Ok(_), ReturnSource::ExecutableNonMain) => continue,
821
822 (e, ReturnSource::Library)
823 | (e, ReturnSource::ExecutableMain)
824 | (e @ Err(EnclaveAbort::Exit { panic: None }), _) => e,
825
826 (Ok(_), ReturnSource::AsyncUsercall)
827 | (Err(EnclaveAbort::MainReturned), ReturnSource::AsyncUsercall) => {
828 unreachable!()
829 }
830
831 (Err(e @ EnclaveAbort::Exit { panic: Some(_) }), _)
832 | (Err(e @ EnclaveAbort::InvalidUsercall(_)), _) => {
833 let e = e.map_panic(|opt| opt.unwrap());
834 let cmd = enclave_clone.kind.as_command().unwrap();
835 let mut cmddata = cmd.panic_reason.lock().await;
836
837 if cmddata.primary_panic_reason.is_none() {
838 cmddata.primary_panic_reason = Some(e)
839 } else {
840 cmddata.other_reasons.push(e)
841 }
842 Err(EnclaveAbort::Secondary)
843 }
844
845 (Err(e), _) => {
846 let e = e.map_panic(|opt| opt.unwrap());
847 let cmd = enclave_clone.kind.as_command().unwrap();
848 let mut cmddata = cmd.panic_reason.lock().await;
849 cmddata.other_reasons.push(e);
850 continue;
851 }
852 };
853 return res;
854 }
855 unreachable!();
856 };
857 let enclave_clone = enclave.clone();
858 let io_future = async move {
859 let (uqs, rqs, cqs, sync_usercall_tx) = QueueSynchronizer::new(enclave_clone.clone());
860
861 let (usercall_queue_tx, usercall_queue_rx) =
862 ipc_queue::bounded_async(USERCALL_QUEUE_SIZE, uqs);
863 let (return_queue_tx, return_queue_rx) =
864 ipc_queue::bounded_async(RETURN_QUEUE_SIZE, rqs);
865 let (cancel_queue_tx, cancel_queue_rx) =
866 ipc_queue::bounded_async(CANCEL_QUEUE_SIZE, cqs);
867
868 let fifo_guards = FifoGuards {
869 usercall_queue: usercall_queue_tx.into_descriptor_guard(),
870 return_queue: return_queue_rx.into_descriptor_guard(),
871 cancel_queue: cancel_queue_tx.into_descriptor_guard(),
872 async_queues_called: false,
873 };
874
875 *enclave_clone.fifo_guards.lock().await = Some(fifo_guards);
876 *enclave_clone.return_queue_tx.lock().await = Some(return_queue_tx);
877
878 let usercall_queue_monitor = usercall_queue_rx.position_monitor();
879
880 let (usercall_event_tx, mut usercall_event_rx) = async_mpsc::unbounded_channel();
881 let usercall_event_tx_clone = usercall_event_tx.clone();
882 tokio::task::spawn_local(async move {
883 while let Ok(usercall) = usercall_queue_rx.recv().await {
884 let notifier_rx = if usercall.ignore_cancel() {
885 None
886 } else {
887 let (notifier_tx, notifier_rx) = oneshot::channel();
888 usercall_event_tx_clone
889 .send(UsercallEvent::Started(usercall.id, notifier_tx))
890 .ok()
891 .expect("failed to send usercall event");
892 Some(notifier_rx)
893 };
894 let _ = io_queue_send.send(UsercallSendData::Async(usercall, notifier_rx));
895 }
896 });
897
898 let usercall_event_tx_clone = usercall_event_tx.clone();
899 let usercall_queue_monitor_clone = usercall_queue_monitor.clone();
900 tokio::task::spawn_local(async move {
901 while let Ok(c) = cancel_queue_rx.recv().await {
902 let write_position = usercall_queue_monitor_clone.write_position();
903 let _ = usercall_event_tx_clone
904 .send(UsercallEvent::Cancelled(c.id, write_position));
905 }
906 });
907
908 tokio::task::spawn_local(async move {
909 let mut notifiers = HashMap::new();
910 let mut cancels: HashMap<u64, WritePosition> = HashMap::new();
911 loop {
912 match usercall_event_rx
913 .recv()
914 .await
915 .expect("usercall_event channel closed unexpectedly")
916 {
917 UsercallEvent::Started(id, notifier) => match cancels.remove(&id) {
918 Some(_) => {
919 let _ = notifier.send(());
920 }
921 _ => {
922 notifiers.insert(id, notifier);
923 }
924 },
925 UsercallEvent::Finished(id) => {
926 notifiers.remove(&id);
927 }
928 UsercallEvent::Cancelled(id, wp) => match notifiers.remove(&id) {
929 Some(notifier) => {
930 let _ = notifier.send(());
931 }
932 None => {
933 cancels.insert(id, wp);
934 }
935 },
936 }
937 let read_position = usercall_queue_monitor.read_position();
939 cancels.retain(|_id, wp| {
940 if let Some(past) = read_position.is_past(wp) {
941 !past
942 } else {
943 false
944 }
945 });
946 }
947 });
948
949 while let Some(work) = io_queue_receive.recv().await {
950 let enclave_clone = enclave_clone.clone();
951 let tx_return_channel = tx_return_channel.clone();
952 match work {
953 UsercallSendData::Async(usercall, notifier_rx) => {
954 let usercall_event_tx = if usercall.ignore_cancel() {
955 None
956 } else {
957 Some(usercall_event_tx.clone())
958 };
959 let uchd =
960 UsercallHandleData::Async(usercall, notifier_rx, usercall_event_tx);
961 let fut = Self::handle_usercall(
962 enclave_clone,
963 work_sender.clone(),
964 tx_return_channel,
965 uchd,
966 );
967 tokio::task::spawn_local(fut);
968 }
969 UsercallSendData::Sync(CoResult::Yield(usercall), state, buf) => {
970 let _ = sync_usercall_tx.send(());
971 let uchd = UsercallHandleData::Sync(usercall, state, buf);
972 let fut = Self::handle_usercall(
973 enclave_clone,
974 work_sender.clone(),
975 tx_return_channel,
976 uchd,
977 );
978 tokio::task::spawn_local(fut);
979 }
980 UsercallSendData::Sync(CoResult::Return((tcs, v1, v2)), state, _buf) => {
981 let fut = async move {
982 let ret = match state.mode {
983 EnclaveEntry::Library => {
984 enclave_clone.threads_queue.push(StoppedTcs { tcs });
985 Ok((v1, v2))
986 }
987 EnclaveEntry::ExecutableMain => Err(EnclaveAbort::MainReturned),
988 EnclaveEntry::ExecutableNonMain => {
989 assert_eq!(
990 (v1, v2),
991 (0, 0),
992 "Expected enclave thread entrypoint to return zero"
993 );
994 if !enclave_clone.exiting.load(Ordering::SeqCst) {
997 enclave_clone.threads_queue.push(StoppedTcs { tcs });
998 }
999 Ok((0, 0))
1000 }
1001 };
1002 let _ = tx_return_channel.send((ret, state.mode.into()));
1003 };
1004 tokio::task::spawn_local(fut);
1005 }
1006 };
1007 }
1008 unreachable!();
1009 };
1010
1011 let select_fut =
1016 futures::future::select(return_future.boxed_local(), io_future.boxed_local()).map(
1017 |either| match either {
1018 Either::Left((x, _)) => x,
1019 _ => unreachable!(),
1020 },
1021 );
1022
1023 local_set
1024 .block_on(&mut rt, select_fut.unit_error())
1025 .unwrap()
1026 }
1027
1028 fn run(
1029 enclave: Arc<EnclaveState>,
1030 num_of_worker_threads: usize,
1031 start_work: Work,
1032 ) -> EnclaveResult {
1033 fn create_worker_threads(
1034 num_of_worker_threads: usize,
1035 work_receiver: crossbeam::channel::Receiver<Work>,
1036 io_queue_send: tokio::sync::mpsc::UnboundedSender<UsercallSendData>,
1037 ) -> Vec<JoinHandle<()>> {
1038 let mut thread_handles = vec![];
1039 for _ in 0..num_of_worker_threads {
1040 let work_receiver = work_receiver.clone();
1041 let mut io_queue_send = io_queue_send.clone();
1042
1043 thread_handles.push(thread::spawn(move || {
1044 while let Ok(work) = work_receiver.recv() {
1045 work.do_work(&mut io_queue_send);
1046 }
1047 }));
1048 }
1049 thread_handles
1050 }
1051
1052 let (io_queue_send, io_queue_receive) = tokio::sync::mpsc::unbounded_channel();
1053
1054 let (work_sender, work_receiver) = crossbeam::channel::unbounded();
1055 work_sender
1056 .send(start_work)
1057 .expect("Work sender couldn't send data to receiver");
1058
1059 let join_handlers =
1060 create_worker_threads(num_of_worker_threads, work_receiver, io_queue_send.clone());
1061 let main_result = EnclaveState::syscall_loop(
1063 enclave.clone(),
1064 io_queue_receive,
1065 io_queue_send,
1066 work_sender,
1067 );
1068
1069 for handler in join_handlers {
1070 let _ = handler.join();
1071 }
1072
1073 main_result
1074 }
1075
1076 pub(crate) fn main_entry(
1077 main: ErasedTcs,
1078 threads: Vec<ErasedTcs>,
1079 stream_router: Box<dyn StreamRouter>,
1080 forward_panics: bool,
1081 force_time_usercalls: bool,
1082 cmd_configuration: CommandConfiguration,
1083 ) -> StdResult<(), anyhow::Error> {
1084 let num_of_worker_threads = cmd_configuration.num_worker_threads;
1085 assert!(num_of_worker_threads > 0, "worker_threads cannot be zero");
1086 let mut event_queues =
1087 FnvHashMap::with_capacity_and_hasher(threads.len() + 1, Default::default());
1088 let main = Self::event_queue_add_tcs(&mut event_queues, main);
1089
1090 let args: Vec<_> = cmd_configuration
1091 .cmd_args
1092 .into_iter()
1093 .map(|a| ByteBuffer {
1094 len: a.len(),
1095 data: Box::into_raw(a.into_boxed_slice()) as *const u8,
1096 })
1097 .collect();
1098 let argc = args.len();
1099 let argv = Box::into_raw(args.into_boxed_slice()) as *const u8;
1100
1101 let main_work = Work {
1102 tcs: RunningTcs {
1103 tcs_address: main.tcs.address(),
1104 mode: EnclaveEntry::ExecutableMain,
1105 },
1106 entry: CoEntry::Initial(main.tcs, argv as _, argc as _, 0, 0, 0),
1107 };
1108
1109 let kind = EnclaveKind::Command(Command {
1110 panic_reason: Mutex::new(PanicReason {
1111 primary_panic_reason: None,
1112 other_reasons: vec![],
1113 }),
1114 });
1115 let enclave = EnclaveState::new(
1116 kind,
1117 event_queues,
1118 stream_router,
1119 threads,
1120 forward_panics,
1121 force_time_usercalls,
1122 );
1123
1124 let main_result = EnclaveState::run(enclave.clone(), num_of_worker_threads, main_work);
1125
1126 let main_panicking = match main_result {
1127 Err(EnclaveAbort::MainReturned)
1128 | Err(EnclaveAbort::InvalidUsercall(_))
1129 | Err(EnclaveAbort::Exit { .. }) => true,
1130 Err(EnclaveAbort::IndefiniteWait) | Err(EnclaveAbort::Secondary) | Ok(_) => false,
1131 };
1132
1133 let rt = tokio::runtime::Builder::new_current_thread()
1134 .enable_all()
1135 .build()
1136 .expect("failed to create tokio Runtime");
1137
1138 rt.block_on(async move {
1139 enclave.abort_all_threads();
1140 while enclave.threads_queue.pop().is_some() {}
1142
1143 let cmd = enclave.kind.as_command().unwrap();
1144 let mut cmddata = cmd.panic_reason.lock().await;
1145 let main_result = match (main_panicking, cmddata.primary_panic_reason.take()) {
1146 (false, Some(reason)) => Err(reason.map_panic(Some)),
1147 _ => main_result,
1149 };
1150 match main_result {
1151 Err(EnclaveAbort::Exit { panic: None }) => Ok(()),
1152 Err(EnclaveAbort::Exit { panic: Some(panic) }) => Err(panic.into()),
1153 Err(EnclaveAbort::IndefiniteWait) => {
1154 bail!("All enclave threads are waiting indefinitely without possibility of wakeup")
1155 }
1156 Err(EnclaveAbort::InvalidUsercall(n)) => {
1157 bail!("The enclave performed an invalid usercall 0x{:x}", n)
1158 }
1159 Err(EnclaveAbort::MainReturned) => bail!(
1160 "The enclave returned from the main entrypoint in violation of the specification."
1161 ),
1162 Err(EnclaveAbort::Secondary) => unreachable!(),
1164 Ok(_) => Ok(()),
1165 }
1166 }.boxed_local())
1167 }
1168
1169 pub(crate) fn library(
1170 threads: Vec<ErasedTcs>,
1171 stream_router: Box<dyn StreamRouter>,
1172 forward_panics: bool,
1173 force_time_usercalls: bool,
1174 ) -> Arc<Self> {
1175 let event_queues = FnvHashMap::with_capacity_and_hasher(threads.len(), Default::default());
1176
1177 let kind = EnclaveKind::Library(Library {});
1178
1179 let enclave = EnclaveState::new(
1180 kind,
1181 event_queues,
1182 stream_router,
1183 threads,
1184 forward_panics,
1185 force_time_usercalls,
1186 );
1187 return enclave;
1188 }
1189
1190 pub(crate) fn library_entry(
1191 enclave: &Arc<Self>,
1192 p1: u64,
1193 p2: u64,
1194 p3: u64,
1195 p4: u64,
1196 p5: u64,
1197 ) -> StdResult<(u64, u64), anyhow::Error> {
1198 let thread = enclave.threads_queue.pop().expect("threads queue empty");
1199 let work = Work {
1200 tcs: RunningTcs {
1201 tcs_address: thread.tcs.address(),
1202 mode: EnclaveEntry::Library,
1203 },
1204 entry: CoEntry::Initial(thread.tcs, p1, p2, p3, p4, p5),
1205 };
1206 let num_of_worker_threads = 1;
1209
1210 let library_result = EnclaveState::run(enclave.clone(), num_of_worker_threads, work);
1211
1212 match library_result {
1213 Err(EnclaveAbort::Exit { panic: None }) => bail!("This thread aborted execution"),
1214 Err(EnclaveAbort::Exit { panic: Some(panic) }) => Err(panic.into()),
1215 Err(EnclaveAbort::IndefiniteWait) => {
1216 bail!("This thread is waiting indefinitely without possibility of wakeup")
1217 }
1218 Err(EnclaveAbort::InvalidUsercall(n)) => {
1219 bail!("The enclave performed an invalid usercall 0x{:x}", n)
1220 }
1221 Err(EnclaveAbort::Secondary) => {
1222 bail!("This thread exited because another thread aborted")
1223 }
1224 Err(EnclaveAbort::MainReturned) => unreachable!(),
1225 Ok(result) => Ok(result),
1226 }
1227 }
1228
1229 fn abort_all_threads(&self) {
1230 self.exiting.store(true, Ordering::SeqCst);
1231 for pending_events in self.event_queues.values() {
1233 pending_events.abort();
1234 }
1235 }
1236}
1237
1238#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1239enum EnclaveEntry {
1240 ExecutableMain,
1241 ExecutableNonMain,
1242 Library,
1243}
1244
1245#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1246enum ReturnSource {
1247 ExecutableMain,
1248 ExecutableNonMain,
1249 Library,
1250 AsyncUsercall,
1251}
1252
1253impl From<EnclaveEntry> for ReturnSource {
1254 fn from(e: EnclaveEntry) -> Self {
1255 match e {
1256 EnclaveEntry::ExecutableMain => ReturnSource::ExecutableMain,
1257 EnclaveEntry::ExecutableNonMain => ReturnSource::ExecutableNonMain,
1258 EnclaveEntry::Library => ReturnSource::Library,
1259 }
1260 }
1261}
1262
1263#[repr(C)]
1264#[allow(unused)]
1265enum Greg {
1266 R8 = 0,
1267 R9,
1268 R10,
1269 R11,
1270 R12,
1271 R13,
1272 R14,
1273 R15,
1274 RDI,
1275 RSI,
1276 RBP,
1277 RBX,
1278 RDX,
1279 RAX,
1280 RCX,
1281 RSP,
1282 RIP,
1283 EFL,
1284 CSGSFS, ERR,
1286 TRAPNO,
1287 OLDMASK,
1288 CR2,
1289}
1290
1291#[cfg(unix)]
1292extern "C" fn handle_trap(_signo: c_int, _info: *mut siginfo_t, context: *mut c_void) {
1297 unsafe {
1298 let context = &mut *(context as *mut ucontext_t);
1299 let rip = &mut context.uc_mcontext.gregs[Greg::RIP as usize];
1300 let inst: *const u8 = *rip as _;
1301 if *inst == 0xcc {
1302 *rip += 1;
1303 }
1304 }
1305 return;
1306}
1307
1308#[cfg(unix)]
1309async fn trap_attached_debugger(tcs: usize, debug_buf: *const u8) {
1314 let _g = DEBUGGER_TOGGLE_SYNC.lock().await;
1315 let hdl = self::signal::SigHandler::SigAction(handle_trap);
1316 let sig_action = signal::SigAction::new(hdl, signal::SaFlags::empty(), signal::SigSet::empty());
1317 unsafe {
1319 let old = signal::sigaction(signal::SIGTRAP, &sig_action).unwrap();
1320 std::arch::asm!("
1321 xchg %rbx, {0}
1322 int3
1323 xchg {0}, %rbx
1324 ",
1325 in(reg) tcs, in("r10") debug_buf,
1327 options(nomem, nostack, att_syntax));
1328 signal::sigaction(signal::SIGTRAP, &old).unwrap();
1329 }
1330}
1331
1332impl<'tcs> IOHandlerInput<'tcs> {
1333 async fn lookup_fd(&self, fd: Fd) -> IoResult<Arc<AsyncFileDesc>> {
1334 match self.enclave.fds.lock().await.get(&fd) {
1335 Some(stream) => Ok(stream.clone()),
1336 None => Err(IoErrorKind::BrokenPipe.into()), }
1338 }
1339
1340 async fn alloc_fd(&self, stream: AsyncFileDesc) -> Fd {
1341 let fd = (self
1342 .enclave
1343 .last_fd
1344 .fetch_add(1, Ordering::Relaxed)
1345 .checked_add(1)
1346 .expect("FD overflow")) as Fd;
1347 let prev = self.enclave.fds.lock().await.insert(fd, Arc::new(stream));
1348 debug_assert!(prev.is_none());
1349 fd
1350 }
1351
1352 #[inline(always)]
1353 fn is_exiting(&self) -> bool {
1354 self.enclave.exiting.load(Ordering::SeqCst)
1355 }
1356
1357 #[inline(always)]
1358 async fn read(&self, fd: Fd, buf: &mut ReadBuf<'_>) -> IoResult<()> {
1359 let file_desc = self.lookup_fd(fd).await?;
1360 file_desc.as_stream()?.async_read(buf).await
1361 }
1362
1363 #[inline(always)]
1364 async fn read_alloc(&self, fd: Fd, buf: &mut OutputBuffer<'tcs, Vec<u8>>) -> IoResult<()> {
1365 let file_desc = self.lookup_fd(fd).await?;
1366 **buf = Some(file_desc.as_stream()?.async_read_alloc().await?);
1367 Ok(())
1368 }
1369
1370 #[inline(always)]
1371 async fn write(&self, fd: Fd, buf: &[u8]) -> IoResult<usize> {
1372 let file_desc = self.lookup_fd(fd).await?;
1373 return file_desc.as_stream()?.async_write(buf).await;
1374 }
1375
1376 #[inline(always)]
1377 async fn flush(&self, fd: Fd) -> IoResult<()> {
1378 let file_desc = self.lookup_fd(fd).await?;
1379 file_desc.as_stream()?.async_flush().await
1380 }
1381
1382 #[inline(always)]
1383 async fn close(&self, fd: Fd) {
1384 self.enclave.fds.lock().await.remove(&fd);
1385 }
1386
1387 #[inline(always)]
1388 async fn bind_stream(
1389 &self,
1390 addr: &[u8],
1391 local_addr: Option<&mut OutputBuffer<'tcs, String>>,
1392 ) -> IoResult<Fd> {
1393 let addr = str::from_utf8(addr).map_err(|_| IoErrorKind::ConnectionRefused)?;
1394
1395 let socket = self
1396 .enclave
1397 .stream_router
1398 .bind_stream(addr, local_addr.map(OutputBuffer::init_default))
1399 .await?;
1400
1401 Ok(self.alloc_fd(AsyncFileDesc::listener(socket)).await)
1402 }
1403
1404 #[inline(always)]
1405 async fn accept_stream(
1406 &self,
1407 fd: Fd,
1408 local_addr: Option<&mut OutputBuffer<'tcs, String>>,
1409 peer_addr: Option<&mut OutputBuffer<'tcs, String>>,
1410 ) -> IoResult<Fd> {
1411 let file_desc = self.lookup_fd(fd).await?;
1412
1413 let stream = file_desc
1414 .as_listener()?
1415 .async_accept(
1416 local_addr.map(OutputBuffer::init_default),
1417 peer_addr.map(OutputBuffer::init_default),
1418 )
1419 .await?;
1420
1421 Ok(self.alloc_fd(AsyncFileDesc::stream(stream)).await)
1422 }
1423
1424 #[inline(always)]
1425 async fn connect_stream(
1426 &self,
1427 addr: &[u8],
1428 local_addr: Option<&mut OutputBuffer<'tcs, String>>,
1429 peer_addr: Option<&mut OutputBuffer<'tcs, String>>,
1430 ) -> IoResult<Fd> {
1431 let addr = str::from_utf8(addr).map_err(|_| IoErrorKind::ConnectionRefused)?;
1432
1433 let socket = self
1434 .enclave
1435 .stream_router
1436 .connect_stream(
1437 addr,
1438 local_addr.map(OutputBuffer::init_default),
1439 peer_addr.map(OutputBuffer::init_default),
1440 )
1441 .await?;
1442
1443 Ok(self.alloc_fd(AsyncFileDesc::stream(socket)).await)
1444 }
1445
1446 #[inline(always)]
1447 fn launch_thread(&self) -> IoResult<()> {
1448 self.enclave
1450 .kind
1451 .as_command()
1452 .ok_or(IoErrorKind::InvalidInput)?;
1453 let new_tcs = match self.enclave.threads_queue.pop() {
1454 Some(tcs) => tcs,
1455 None => {
1456 return Err(IoErrorKind::WouldBlock.into());
1457 }
1458 };
1459
1460 let ret = self.work_sender.send(Work {
1461 tcs: RunningTcs {
1462 tcs_address: new_tcs.tcs.address(),
1463 mode: EnclaveEntry::ExecutableNonMain,
1464 },
1465 entry: CoEntry::Initial(new_tcs.tcs, 0, 0, 0, 0, 0),
1466 });
1467 match ret {
1468 Ok(()) => Ok(()),
1469 Err(e) => {
1470 let entry = e.0.entry;
1471 match entry {
1472 CoEntry::Initial(tcs, _, _, _, _, _) => {
1473 self.enclave.threads_queue.push(StoppedTcs { tcs });
1474 }
1475 _ => unreachable!(),
1476 };
1477 Err(std::io::Error::new(
1478 IoErrorKind::NotConnected,
1479 "Work Sender: send error",
1480 ))
1481 }
1482 }
1483 }
1484
1485 #[inline(always)]
1486 fn exit(&mut self, panic: bool) -> EnclaveAbort<bool> {
1487 self.enclave.abort_all_threads();
1488 EnclaveAbort::Exit { panic }
1489 }
1490
1491 fn check_event_set(set: u64) -> IoResult<()> {
1492 const EV_ALL: u64 =
1493 EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK | EV_CANCELQ_NOT_FULL;
1494 if (set & !EV_ALL) != 0 {
1495 return Err(IoErrorKind::InvalidInput.into());
1496 }
1497
1498 assert!((EV_ALL | EV_ABORT) <= u8::MAX.into());
1499 assert!((EV_ALL & EV_ABORT) == 0);
1500 Ok(())
1501 }
1502
1503 #[inline(always)]
1504 async fn wait(&mut self, event_mask: u64, timeout: u64) -> IoResult<u64> {
1505 Self::check_event_set(event_mask)?;
1506
1507 let timeout = match timeout {
1508 WAIT_NO | WAIT_INDEFINITE => timeout,
1509 _ => {
1510 cmp::min(1 << 55, timeout)
1516 }
1517 };
1518
1519 let tcs = self
1521 .tcs
1522 .as_mut()
1523 .ok_or(io::Error::from(io::ErrorKind::Other))?;
1524
1525 let pending_events = self
1526 .enclave
1527 .event_queues
1528 .get(&tcs.tcs_address)
1529 .expect("invalid tcs address");
1530
1531 let ret = match timeout {
1532 WAIT_NO => pending_events.take(event_mask),
1533 WAIT_INDEFINITE => Some(pending_events.wait_for(event_mask).await),
1534 n => tokio::time::timeout(Duration::from_nanos(n), pending_events.wait_for(event_mask))
1535 .await
1536 .ok(),
1537 };
1538
1539 if let Some(ev) = ret {
1540 if (ev & EV_ABORT) != 0 {
1541 return Err(IoErrorKind::Other.into());
1543 }
1544 return Ok(ev.into());
1545 }
1546 Err(if timeout == WAIT_NO {
1547 IoErrorKind::WouldBlock
1548 } else {
1549 IoErrorKind::TimedOut
1550 }
1551 .into())
1552 }
1553
1554 #[inline(always)]
1555 fn send(&self, event_set: u64, target: Option<Tcs>) -> IoResult<()> {
1556 Self::check_event_set(event_set)?;
1557
1558 if event_set == 0 {
1559 return Err(IoErrorKind::InvalidInput.into());
1560 }
1561
1562 if let Some(tcs) = target {
1563 let tcs = TcsAddress(tcs.as_ptr() as _);
1564 let pending_events = self
1565 .enclave
1566 .event_queues
1567 .get(&tcs)
1568 .ok_or(IoErrorKind::InvalidInput)?;
1569 pending_events.push(event_set);
1570 } else {
1571 for pending_events in self.enclave.event_queues.values() {
1572 pending_events.push(event_set);
1573 }
1574 }
1575
1576 Ok(())
1577 }
1578
1579 #[inline(always)]
1580 fn insecure_time(&mut self) -> (u64, *const InsecureTimeInfo) {
1581 let time = time::SystemTime::now()
1582 .duration_since(time::UNIX_EPOCH)
1583 .unwrap();
1584 let t = (time.subsec_nanos() as u64) + time.as_secs() * NANOS_PER_SEC;
1585 let insecure_time_ref: Option<&'static InsecureTimeInfo> = (&*TIME_INFO).as_ref();
1586 let info =
1587 if let (Some(info), false) = (insecure_time_ref, self.enclave.force_time_usercalls) {
1588 info
1589 } else {
1590 ptr::null()
1591 };
1592 (t, info)
1593 }
1594
1595 #[inline(always)]
1596 fn alloc(&self, size: usize, alignment: usize) -> IoResult<*mut u8> {
1597 unsafe {
1598 let layout =
1599 Layout::from_size_align(size, alignment).map_err(|_| IoErrorKind::InvalidInput)?;
1600 if layout.size() == 0 {
1601 return Err(IoErrorKind::InvalidInput.into());
1602 }
1603 let ptr = System.alloc(layout);
1604 if ptr.is_null() {
1605 Err(IoErrorKind::Other.into())
1606 } else {
1607 Ok(ptr)
1608 }
1609 }
1610 }
1611
1612 #[inline(always)]
1613 fn free(&self, ptr: *mut u8, size: usize, alignment: usize) -> IoResult<()> {
1614 unsafe {
1615 let layout =
1616 Layout::from_size_align(size, alignment).map_err(|_| IoErrorKind::InvalidInput)?;
1617 if size == 0 {
1618 return Ok(());
1619 }
1620 Ok(System.dealloc(ptr, layout))
1621 }
1622 }
1623
1624 #[inline(always)]
1625 async fn async_queues(
1626 &mut self,
1627 usercall_queue: &mut FifoDescriptor<Usercall>,
1628 return_queue: &mut FifoDescriptor<Return>,
1629 cancel_queue: Option<&mut FifoDescriptor<Cancel>>,
1630 ) -> StdResult<(), EnclaveAbort<bool>> {
1631 let mut fifo_guards = self.enclave.fifo_guards.lock().await;
1632 match &mut *fifo_guards {
1633 Some(ref mut fifo_guards) if !fifo_guards.async_queues_called => {
1634 *usercall_queue = fifo_guards.usercall_queue.fifo_descriptor();
1635 *return_queue = fifo_guards.return_queue.fifo_descriptor();
1636 if let Some(cancel_queue) = cancel_queue {
1637 *cancel_queue = fifo_guards.cancel_queue.fifo_descriptor();
1638 }
1639 fifo_guards.async_queues_called = true;
1640 Ok(())
1641 }
1642 Some(_) => {
1643 drop(fifo_guards);
1644 Err(self.exit(true))
1645 }
1646 None => unreachable!(),
1649 }
1650 }
1651}
1652
1653#[derive(Clone, Copy)]
1654enum Queue {
1655 Usercall,
1656 Return,
1657 Cancel,
1658}
1659
1660struct QueueSynchronizer {
1661 queue: Queue,
1662 enclave: Arc<EnclaveState>,
1663 subscription: Mutex<broadcast::Receiver<()>>,
1665 subscription_maker: broadcast::Sender<()>,
1667}
1668
1669impl QueueSynchronizer {
1670 fn new(enclave: Arc<EnclaveState>) -> (Self, Self, Self, broadcast::Sender<()>) {
1671 let (tx, rx1) = broadcast::channel(1);
1677 let rx2 = tx.subscribe();
1678 let rx3 = tx.subscribe();
1679 let usercall_queue_synchronizer = QueueSynchronizer {
1680 queue: Queue::Usercall,
1681 enclave: enclave.clone(),
1682 subscription: Mutex::new(rx1),
1683 subscription_maker: tx.clone(),
1684 };
1685 let return_queue_synchronizer = QueueSynchronizer {
1686 queue: Queue::Return,
1687 enclave: enclave.clone(),
1688 subscription: Mutex::new(rx2),
1689 subscription_maker: tx.clone(),
1690 };
1691 let cancel_queue_synchronizer = QueueSynchronizer {
1692 queue: Queue::Cancel,
1693 enclave,
1694 subscription: Mutex::new(rx3),
1695 subscription_maker: tx.clone(),
1696 };
1697 (
1698 usercall_queue_synchronizer,
1699 return_queue_synchronizer,
1700 cancel_queue_synchronizer,
1701 tx,
1702 )
1703 }
1704}
1705
1706impl Clone for QueueSynchronizer {
1707 fn clone(&self) -> Self {
1708 Self {
1709 queue: self.queue,
1710 enclave: self.enclave.clone(),
1711 subscription: Mutex::new(self.subscription_maker.subscribe()),
1712 subscription_maker: self.subscription_maker.clone(),
1713 }
1714 }
1715}
1716
1717impl ipc_queue::AsyncSynchronizer for QueueSynchronizer {
1718 fn wait(
1719 &self,
1720 event: QueueEvent,
1721 ) -> Pin<Box<dyn Future<Output = StdResult<(), ipc_queue::SynchronizationError>> + '_>> {
1722 match (self.queue, event) {
1723 (Queue::Usercall, QueueEvent::NotFull) => {
1724 panic!("enclave runner should not send on the usercall queue")
1725 }
1726 (Queue::Cancel, QueueEvent::NotFull) => {
1727 panic!("enclave runner should not send on the cancel queue")
1728 }
1729 (Queue::Return, QueueEvent::NotEmpty) => {
1730 panic!("enclave runner should not receive on the return queue")
1731 }
1732 _ => {}
1733 }
1734 async move {
1738 let mut subscription = self.subscription.lock().await;
1739 match subscription.recv().await {
1740 Ok(()) | Err(RecvError::Lagged(_)) => Ok(()),
1741 Err(RecvError::Closed) => Err(ipc_queue::SynchronizationError::ChannelClosed),
1742 }
1743 }
1744 .boxed_local()
1745 }
1746
1747 fn notify(&self, event: QueueEvent) {
1748 let ev = match (self.queue, event) {
1749 (Queue::Usercall, QueueEvent::NotEmpty) => {
1750 panic!("enclave runner should not send on the usercall queue")
1751 }
1752 (Queue::Cancel, QueueEvent::NotEmpty) => {
1753 panic!("enclave runner should not send on the cancel queue")
1754 }
1755 (Queue::Return, QueueEvent::NotFull) => {
1756 panic!("enclave runner should not receive on the return queue")
1757 }
1758 (Queue::Usercall, QueueEvent::NotFull) => EV_USERCALLQ_NOT_FULL,
1759 (Queue::Return, QueueEvent::NotEmpty) => EV_RETURNQ_NOT_EMPTY,
1760 (Queue::Cancel, QueueEvent::NotFull) => EV_CANCELQ_NOT_FULL,
1761 };
1762 for pending_events in self.enclave.event_queues.values() {
1766 pending_events.push(ev as _);
1767 }
1768 }
1769}