From c8d62f7b85b921e7ee1f1c492806e3c9e55e45f7 Mon Sep 17 00:00:00 2001 From: Ray Kraesig Date: Mon, 16 Oct 2023 22:26:00 +0000 Subject: [PATCH] Bug 1858739 - update tokio to 1.29.1 r=glandium,supply-chain-reviewers Differential Revision: https://phabricator.services.mozilla.com/D190838 --- Cargo.lock | 13 +- Cargo.toml | 5 + build/rust/backtrace/Cargo.toml | 8 + build/rust/backtrace/lib.rs | 0 supply-chain/imports.lock | 6 + third_party/rust/tokio/.cargo-checksum.json | 2 +- third_party/rust/tokio/CHANGELOG.md | 906 +++++++++++- third_party/rust/tokio/Cargo.toml | 196 ++- third_party/rust/tokio/LICENSE | 2 +- third_party/rust/tokio/README.md | 54 +- third_party/rust/tokio/build.rs | 192 +++ .../rust/tokio/docs/reactor-refactor.md | 8 +- third_party/rust/tokio/external-types.toml | 11 + third_party/rust/tokio/src/doc/mod.rs | 1 - third_party/rust/tokio/src/doc/os.rs | 41 +- third_party/rust/tokio/src/doc/winapi.rs | 66 - third_party/rust/tokio/src/fs/file.rs | 103 +- third_party/rust/tokio/src/fs/file/tests.rs | 29 +- third_party/rust/tokio/src/fs/mocks.rs | 2 +- third_party/rust/tokio/src/fs/mod.rs | 25 +- third_party/rust/tokio/src/fs/open_options.rs | 19 +- .../src/fs/open_options/mock_open_options.rs | 1 + third_party/rust/tokio/src/fs/read_dir.rs | 110 +- third_party/rust/tokio/src/fs/symlink_dir.rs | 2 +- third_party/rust/tokio/src/fs/symlink_file.rs | 2 +- third_party/rust/tokio/src/fs/try_exists.rs | 34 + third_party/rust/tokio/src/future/block_on.rs | 11 +- third_party/rust/tokio/src/future/poll_fn.rs | 28 +- third_party/rust/tokio/src/fuzz.rs | 1 + third_party/rust/tokio/src/io/async_fd.rs | 645 ++++++++- third_party/rust/tokio/src/io/blocking.rs | 5 +- third_party/rust/tokio/src/io/bsd/poll_aio.rs | 6 +- .../rust/tokio/src/io/driver/platform.rs | 44 - .../tokio/src/io/{driver => }/interest.rs | 30 +- third_party/rust/tokio/src/io/mod.rs | 50 +- third_party/rust/tokio/src/io/poll_evented.rs | 86 +- third_party/rust/tokio/src/io/read_buf.rs | 31 + .../rust/tokio/src/io/{driver => }/ready.rs | 57 +- third_party/rust/tokio/src/io/split.rs | 6 +- third_party/rust/tokio/src/io/stderr.rs | 41 +- third_party/rust/tokio/src/io/stdin.rs | 41 +- third_party/rust/tokio/src/io/stdio_common.rs | 11 +- third_party/rust/tokio/src/io/stdout.rs | 41 +- .../tokio/src/io/util/async_buf_read_ext.rs | 2 +- .../rust/tokio/src/io/util/async_seek_ext.rs | 2 +- .../rust/tokio/src/io/util/async_write_ext.rs | 10 +- third_party/rust/tokio/src/io/util/copy.rs | 72 +- .../tokio/src/io/util/copy_bidirectional.rs | 57 +- .../rust/tokio/src/io/util/copy_buf.rs | 8 +- third_party/rust/tokio/src/io/util/empty.rs | 4 +- third_party/rust/tokio/src/io/util/mem.rs | 8 +- third_party/rust/tokio/src/io/util/read.rs | 2 +- .../rust/tokio/src/io/util/read_to_end.rs | 53 +- .../rust/tokio/src/io/util/read_until.rs | 1 + .../tokio/src/io/util/vec_with_initialized.rs | 36 +- third_party/rust/tokio/src/lib.rs | 247 +++- third_party/rust/tokio/src/loom/mocked.rs | 12 + .../rust/tokio/src/loom/std/atomic_ptr.rs | 34 - .../rust/tokio/src/loom/std/atomic_u16.rs | 4 +- .../rust/tokio/src/loom/std/atomic_u32.rs | 12 +- .../rust/tokio/src/loom/std/atomic_u64.rs | 71 +- .../tokio/src/loom/std/atomic_u64_as_mutex.rs | 76 + .../tokio/src/loom/std/atomic_u64_native.rs | 4 + .../loom/std/atomic_u64_static_const_new.rs | 12 + .../loom/std/atomic_u64_static_once_cell.rs | 57 + .../rust/tokio/src/loom/std/atomic_u8.rs | 34 - .../rust/tokio/src/loom/std/atomic_usize.rs | 4 +- .../rust/tokio/src/loom/std/barrier.rs | 217 +++ third_party/rust/tokio/src/loom/std/mod.rs | 37 +- third_party/rust/tokio/src/loom/std/mutex.rs | 6 + .../rust/tokio/src/loom/std/parking_lot.rs | 2 +- third_party/rust/tokio/src/macros/addr_of.rs | 22 + third_party/rust/tokio/src/macros/cfg.rs | 168 ++- third_party/rust/tokio/src/macros/join.rs | 73 +- third_party/rust/tokio/src/macros/loom.rs | 6 +- third_party/rust/tokio/src/macros/mod.rs | 7 +- .../rust/tokio/src/macros/scoped_tls.rs | 77 -- third_party/rust/tokio/src/macros/select.rs | 38 +- third_party/rust/tokio/src/macros/support.rs | 6 +- .../rust/tokio/src/macros/thread_local.rs | 30 +- third_party/rust/tokio/src/macros/try_join.rs | 75 +- third_party/rust/tokio/src/net/addr.rs | 19 +- third_party/rust/tokio/src/net/mod.rs | 14 +- .../rust/tokio/src/net/tcp/listener.rs | 199 ++- third_party/rust/tokio/src/net/tcp/mod.rs | 4 +- third_party/rust/tokio/src/net/tcp/socket.rs | 209 ++- third_party/rust/tokio/src/net/tcp/split.rs | 38 +- .../rust/tokio/src/net/tcp/split_owned.rs | 42 +- third_party/rust/tokio/src/net/tcp/stream.rs | 383 ++++-- third_party/rust/tokio/src/net/udp.rs | 508 ++++++- .../tokio/src/net/unix/datagram/socket.rs | 204 ++- .../rust/tokio/src/net/unix/listener.rs | 65 +- third_party/rust/tokio/src/net/unix/mod.rs | 17 +- third_party/rust/tokio/src/net/unix/pipe.rs | 1222 +++++++++++++++++ third_party/rust/tokio/src/net/unix/split.rs | 42 +- .../rust/tokio/src/net/unix/split_owned.rs | 46 +- third_party/rust/tokio/src/net/unix/stream.rs | 109 +- third_party/rust/tokio/src/net/unix/ucred.rs | 99 +- .../rust/tokio/src/net/windows/named_pipe.rs | 686 +++++++-- third_party/rust/tokio/src/park/either.rs | 74 - third_party/rust/tokio/src/park/mod.rs | 117 -- third_party/rust/tokio/src/park/thread.rs | 346 ----- third_party/rust/tokio/src/process/mod.rs | 289 +++- .../rust/tokio/src/process/unix/mod.rs | 120 +- .../rust/tokio/src/process/unix/orphan.rs | 28 +- third_party/rust/tokio/src/process/windows.rs | 126 +- .../rust/tokio/src/runtime/basic_scheduler.rs | 574 -------- .../rust/tokio/src/runtime/blocking/mod.rs | 32 +- .../rust/tokio/src/runtime/blocking/pool.rs | 284 +++- .../tokio/src/runtime/blocking/schedule.rs | 49 +- .../tokio/src/runtime/blocking/shutdown.rs | 4 +- .../rust/tokio/src/runtime/blocking/task.rs | 2 +- third_party/rust/tokio/src/runtime/builder.rs | 654 ++++++++- third_party/rust/tokio/src/runtime/config.rs | 37 + third_party/rust/tokio/src/runtime/context.rs | 262 ++-- .../tokio/src/runtime/context/blocking.rs | 121 ++ .../rust/tokio/src/runtime/context/current.rs | 99 ++ .../rust/tokio/src/runtime/context/runtime.rs | 99 ++ .../tokio/src/runtime/context/runtime_mt.rs | 36 + .../rust/tokio/src/runtime/context/scoped.rs | 56 + .../rust/tokio/src/{ => runtime}/coop.rs | 99 +- third_party/rust/tokio/src/runtime/driver.rs | 339 +++-- third_party/rust/tokio/src/runtime/dump.rs | 76 + third_party/rust/tokio/src/runtime/enter.rs | 205 --- third_party/rust/tokio/src/runtime/handle.rs | 444 ++++-- .../rust/tokio/src/runtime/io/metrics.rs | 24 + .../src/{io/driver => runtime/io}/mod.rs | 368 ++--- .../{io/driver => runtime/io}/registration.rs | 66 +- .../{io/driver => runtime/io}/scheduled_io.rs | 75 +- .../rust/tokio/src/runtime/metrics/batch.rs | 91 +- .../tokio/src/runtime/metrics/histogram.rs | 502 +++++++ .../rust/tokio/src/runtime/metrics/io.rs | 24 + .../rust/tokio/src/runtime/metrics/mock.rs | 18 +- .../rust/tokio/src/runtime/metrics/mod.rs | 12 +- .../rust/tokio/src/runtime/metrics/runtime.rs | 490 ++++++- .../tokio/src/runtime/metrics/scheduler.rs | 7 + .../rust/tokio/src/runtime/metrics/worker.rs | 23 +- third_party/rust/tokio/src/runtime/mod.rs | 464 +------ third_party/rust/tokio/src/runtime/park.rs | 343 +++-- .../unix/driver.rs => runtime/process.rs} | 30 +- third_party/rust/tokio/src/runtime/runtime.rs | 445 ++++++ .../src/runtime/scheduler/current_thread.rs | 750 ++++++++++ .../rust/tokio/src/runtime/scheduler/defer.rs | 43 + .../tokio/src/runtime/scheduler/inject.rs | 72 + .../src/runtime/scheduler/inject/metrics.rs | 7 + .../tokio/src/runtime/scheduler/inject/pop.rs | 55 + .../scheduler/inject/rt_multi_thread.rs | 98 ++ .../src/runtime/scheduler/inject/shared.rs | 119 ++ .../src/runtime/scheduler/inject/synced.rs | 32 + .../rust/tokio/src/runtime/scheduler/lock.rs | 6 + .../rust/tokio/src/runtime/scheduler/mod.rs | 249 ++++ .../scheduler/multi_thread/counters.rs | 62 + .../runtime/scheduler/multi_thread/handle.rs | 68 + .../scheduler/multi_thread/handle/metrics.rs | 41 + .../scheduler/multi_thread/handle/taskdump.rs | 26 + .../multi_thread}/idle.rs | 52 +- .../src/runtime/scheduler/multi_thread/mod.rs | 103 ++ .../scheduler/multi_thread/overflow.rs | 26 + .../runtime/scheduler/multi_thread/park.rs | 232 ++++ .../{ => scheduler/multi_thread}/queue.rs | 211 ++- .../runtime/scheduler/multi_thread/stats.rs | 140 ++ .../runtime/scheduler/multi_thread/trace.rs | 61 + .../scheduler/multi_thread/trace_mock.rs | 11 + .../runtime/scheduler/multi_thread/worker.rs | 1216 ++++++++++++++++ .../scheduler/multi_thread/worker/metrics.rs | 11 + .../scheduler/multi_thread/worker/taskdump.rs | 79 ++ .../multi_thread/worker/taskdump_mock.rs | 7 + .../rust/tokio/src/runtime/signal/mod.rs | 142 ++ third_party/rust/tokio/src/runtime/spawner.rs | 83 -- .../rust/tokio/src/runtime/task/abort.rs | 87 ++ .../rust/tokio/src/runtime/task/core.rs | 247 +++- .../rust/tokio/src/runtime/task/error.rs | 33 +- .../rust/tokio/src/runtime/task/harness.rs | 310 +++-- third_party/rust/tokio/src/runtime/task/id.rs | 87 ++ .../rust/tokio/src/runtime/task/inject.rs | 220 --- .../rust/tokio/src/runtime/task/join.rs | 185 ++- .../rust/tokio/src/runtime/task/list.rs | 38 +- .../rust/tokio/src/runtime/task/mod.rs | 132 +- .../rust/tokio/src/runtime/task/raw.rs | 216 ++- .../rust/tokio/src/runtime/task/state.rs | 26 +- .../rust/tokio/src/runtime/task/trace/mod.rs | 330 +++++ .../tokio/src/runtime/task/trace/symbol.rs | 92 ++ .../rust/tokio/src/runtime/task/trace/tree.rs | 126 ++ .../rust/tokio/src/runtime/task/waker.rs | 84 +- .../rust/tokio/src/runtime/tests/inject.rs | 54 + .../tokio/src/runtime/tests/loom_blocking.rs | 21 + ...er.rs => loom_current_thread_scheduler.rs} | 0 .../tokio/src/runtime/tests/loom_join_set.rs | 18 +- .../rust/tokio/src/runtime/tests/loom_pool.rs | 44 +- .../tokio/src/runtime/tests/loom_queue.rs | 71 +- .../tokio/src/runtime/tests/loom_yield.rs | 37 + .../rust/tokio/src/runtime/tests/mod.rs | 38 +- .../rust/tokio/src/runtime/tests/queue.rs | 121 +- .../rust/tokio/src/runtime/tests/task.rs | 52 +- .../src/runtime/tests/task_combinations.rs | 153 ++- .../rust/tokio/src/runtime/thread_id.rs | 31 + .../rust/tokio/src/runtime/thread_pool/mod.rs | 136 -- .../tokio/src/runtime/thread_pool/worker.rs | 848 ------------ .../{time/driver => runtime/time}/entry.rs | 191 +-- .../rust/tokio/src/runtime/time/handle.rs | 62 + .../src/{time/driver => runtime/time}/mod.rs | 312 ++--- .../rust/tokio/src/runtime/time/source.rs | 39 + .../driver => runtime/time}/tests/mod.rs | 180 +-- .../driver => runtime/time}/wheel/level.rs | 4 +- .../driver => runtime/time}/wheel/mod.rs | 18 +- third_party/rust/tokio/src/signal/mod.rs | 2 +- third_party/rust/tokio/src/signal/registry.rs | 28 +- .../rust/tokio/src/signal/reusable_box.rs | 1 - third_party/rust/tokio/src/signal/unix.rs | 115 +- .../rust/tokio/src/signal/unix/driver.rs | 207 --- third_party/rust/tokio/src/signal/windows.rs | 361 ++++- .../rust/tokio/src/signal/windows/stub.rs | 14 +- .../rust/tokio/src/signal/windows/sys.rs | 122 +- third_party/rust/tokio/src/sync/barrier.rs | 6 +- .../rust/tokio/src/sync/batch_semaphore.rs | 33 +- third_party/rust/tokio/src/sync/broadcast.rs | 477 +++++-- third_party/rust/tokio/src/sync/mod.rs | 10 +- third_party/rust/tokio/src/sync/mpsc/block.rs | 143 +- .../rust/tokio/src/sync/mpsc/bounded.rs | 138 +- third_party/rust/tokio/src/sync/mpsc/chan.rs | 66 +- third_party/rust/tokio/src/sync/mpsc/error.rs | 38 +- third_party/rust/tokio/src/sync/mpsc/list.rs | 2 +- third_party/rust/tokio/src/sync/mpsc/mod.rs | 15 +- .../rust/tokio/src/sync/mpsc/unbounded.rs | 94 +- third_party/rust/tokio/src/sync/mutex.rs | 538 +++++++- third_party/rust/tokio/src/sync/notify.rs | 672 +++++++-- third_party/rust/tokio/src/sync/once_cell.rs | 8 +- third_party/rust/tokio/src/sync/oneshot.rs | 45 +- third_party/rust/tokio/src/sync/rwlock.rs | 272 ++-- .../tokio/src/sync/rwlock/owned_read_guard.rs | 58 +- .../src/sync/rwlock/owned_write_guard.rs | 251 +++- .../sync/rwlock/owned_write_guard_mapped.rs | 62 +- .../rust/tokio/src/sync/rwlock/read_guard.rs | 56 +- .../rust/tokio/src/sync/rwlock/write_guard.rs | 261 +++- .../src/sync/rwlock/write_guard_mapped.rs | 66 +- third_party/rust/tokio/src/sync/semaphore.rs | 58 +- .../rust/tokio/src/sync/tests/atomic_waker.rs | 6 +- .../rust/tokio/src/sync/tests/loom_notify.rs | 191 +++ .../rust/tokio/src/sync/tests/loom_watch.rs | 54 + .../rust/tokio/src/sync/tests/notify.rs | 43 +- .../tokio/src/sync/tests/semaphore_batch.rs | 43 +- third_party/rust/tokio/src/sync/watch.rs | 541 +++++++- third_party/rust/tokio/src/task/blocking.rs | 24 +- third_party/rust/tokio/src/task/builder.rs | 118 +- .../rust/tokio/src/task/consume_budget.rs | 46 + third_party/rust/tokio/src/task/join_set.rs | 420 +++++- third_party/rust/tokio/src/task/local.rs | 647 +++++++-- third_party/rust/tokio/src/task/mod.rs | 28 +- third_party/rust/tokio/src/task/spawn.rs | 65 +- third_party/rust/tokio/src/task/task_local.rs | 257 +++- .../rust/tokio/src/task/unconstrained.rs | 2 +- third_party/rust/tokio/src/task/yield_now.rs | 8 +- third_party/rust/tokio/src/time/clock.rs | 140 +- .../rust/tokio/src/time/driver/handle.rs | 94 -- .../rust/tokio/src/time/driver/wheel/stack.rs | 112 -- third_party/rust/tokio/src/time/error.rs | 5 +- third_party/rust/tokio/src/time/instant.rs | 2 +- third_party/rust/tokio/src/time/interval.rs | 15 +- third_party/rust/tokio/src/time/mod.rs | 13 +- .../rust/tokio/src/time/{driver => }/sleep.rs | 64 +- third_party/rust/tokio/src/time/tests/mod.rs | 22 - .../rust/tokio/src/time/tests/test_sleep.rs | 443 ------ third_party/rust/tokio/src/time/timeout.rs | 40 +- third_party/rust/tokio/src/util/error.rs | 7 +- .../rust/tokio/src/util/idle_notified_set.rs | 40 +- .../rust/tokio/src/util/linked_list.rs | 217 ++- third_party/rust/tokio/src/util/markers.rs | 8 + third_party/rust/tokio/src/util/memchr.rs | 74 + third_party/rust/tokio/src/util/mod.rs | 42 +- third_party/rust/tokio/src/util/once_cell.rs | 70 + third_party/rust/tokio/src/util/pad.rs | 52 - third_party/rust/tokio/src/util/rand.rs | 85 +- third_party/rust/tokio/src/util/rand/rt.rs | 61 + .../rust/tokio/src/util/rand/rt_unstable.rs | 20 + third_party/rust/tokio/src/util/rc_cell.rs | 57 + third_party/rust/tokio/src/util/slab.rs | 7 +- third_party/rust/tokio/src/util/trace.rs | 5 +- .../rust/tokio/src/util/vec_deque_cell.rs | 53 - third_party/rust/tokio/tests/_require_full.rs | 8 +- .../rust/tokio/tests/async_send_sync.rs | 221 +-- third_party/rust/tokio/tests/buffered.rs | 6 +- third_party/rust/tokio/tests/dump.rs | 99 ++ third_party/rust/tokio/tests/fs.rs | 2 +- .../rust/tokio/tests/fs_canonicalize_dir.rs | 22 + third_party/rust/tokio/tests/fs_copy.rs | 2 +- third_party/rust/tokio/tests/fs_dir.rs | 41 +- third_party/rust/tokio/tests/fs_file.rs | 106 +- third_party/rust/tokio/tests/fs_link.rs | 59 +- .../rust/tokio/tests/fs_open_options.rs | 80 ++ .../tokio/tests/fs_open_options_windows.rs | 51 + .../rust/tokio/tests/fs_remove_dir_all.rs | 31 + .../rust/tokio/tests/fs_remove_file.rs | 24 + third_party/rust/tokio/tests/fs_rename.rs | 28 + .../tokio/tests/fs_symlink_dir_windows.rs | 31 + .../tokio/tests/fs_symlink_file_windows.rs | 24 + third_party/rust/tokio/tests/fs_try_exists.rs | 44 + third_party/rust/tokio/tests/io_async_fd.rs | 86 ++ third_party/rust/tokio/tests/io_buf_writer.rs | 36 +- .../rust/tokio/tests/io_copy_bidirectional.rs | 38 +- third_party/rust/tokio/tests/io_driver.rs | 5 +- .../rust/tokio/tests/io_driver_drop.rs | 2 +- third_party/rust/tokio/tests/io_fill_buf.rs | 2 +- third_party/rust/tokio/tests/io_mem_stream.rs | 2 +- third_party/rust/tokio/tests/io_panic.rs | 176 +++ third_party/rust/tokio/tests/io_read.rs | 26 +- third_party/rust/tokio/tests/io_read_buf.rs | 58 + .../rust/tokio/tests/io_read_to_end.rs | 46 + third_party/rust/tokio/tests/io_split.rs | 2 +- third_party/rust/tokio/tests/io_take.rs | 27 +- .../rust/tokio/tests/join_handle_panic.rs | 2 +- third_party/rust/tokio/tests/macros_join.rs | 93 +- third_party/rust/tokio/tests/macros_pin.rs | 4 +- .../rust/tokio/tests/macros_rename_test.rs | 35 + third_party/rust/tokio/tests/macros_select.rs | 138 +- third_party/rust/tokio/tests/macros_test.rs | 18 +- .../rust/tokio/tests/macros_try_join.rs | 93 +- .../rust/tokio/tests/net_bind_resource.rs | 3 +- .../rust/tokio/tests/net_lookup_host.rs | 2 +- .../{named_pipe.rs => net_named_pipe.rs} | 126 +- third_party/rust/tokio/tests/net_panic.rs | 188 +++ third_party/rust/tokio/tests/net_unix_pipe.rs | 429 ++++++ third_party/rust/tokio/tests/no_rt.rs | 2 +- .../rust/tokio/tests/process_kill_on_drop.rs | 2 +- .../rust/tokio/tests/process_raw_handle.rs | 2 +- third_party/rust/tokio/tests/process_smoke.rs | 2 +- third_party/rust/tokio/tests/rt_basic.rs | 152 ++ third_party/rust/tokio/tests/rt_common.rs | 334 ++++- third_party/rust/tokio/tests/rt_handle.rs | 67 + .../rust/tokio/tests/rt_handle_block_on.rs | 24 +- third_party/rust/tokio/tests/rt_metrics.rs | 367 ++++- third_party/rust/tokio/tests/rt_panic.rs | 77 ++ third_party/rust/tokio/tests/rt_threaded.rs | 239 +++- .../rust/tokio/tests/rt_time_start_paused.rs | 14 + third_party/rust/tokio/tests/signal_no_rt.rs | 1 + third_party/rust/tokio/tests/signal_panic.rs | 29 + .../tokio/tests/support/leaked_buffers.rs | 26 + third_party/rust/tokio/tests/support/panic.rs | 34 + third_party/rust/tokio/tests/sync_barrier.rs | 2 +- .../rust/tokio/tests/sync_broadcast.rs | 191 ++- third_party/rust/tokio/tests/sync_errors.rs | 2 +- third_party/rust/tokio/tests/sync_mpsc.rs | 60 +- .../rust/tokio/tests/sync_mpsc_weak.rs | 513 +++++++ third_party/rust/tokio/tests/sync_mutex.rs | 18 +- .../rust/tokio/tests/sync_mutex_owned.rs | 8 +- third_party/rust/tokio/tests/sync_notify.rs | 73 +- .../rust/tokio/tests/sync_once_cell.rs | 355 ++--- third_party/rust/tokio/tests/sync_oneshot.rs | 20 +- third_party/rust/tokio/tests/sync_panic.rs | 197 +++ third_party/rust/tokio/tests/sync_rwlock.rs | 70 +- .../rust/tokio/tests/sync_semaphore.rs | 62 +- .../rust/tokio/tests/sync_semaphore_owned.rs | 29 +- third_party/rust/tokio/tests/sync_watch.rs | 32 +- third_party/rust/tokio/tests/task_abort.rs | 12 +- third_party/rust/tokio/tests/task_blocking.rs | 91 +- third_party/rust/tokio/tests/task_builder.rs | 20 +- third_party/rust/tokio/tests/task_id.rs | 303 ++++ third_party/rust/tokio/tests/task_join_set.rs | 151 +- third_party/rust/tokio/tests/task_local.rs | 98 +- .../rust/tokio/tests/task_local_set.rs | 206 ++- third_party/rust/tokio/tests/task_panic.rs | 123 ++ .../rust/tokio/tests/task_yield_now.rs | 16 + third_party/rust/tokio/tests/tcp_accept.rs | 2 +- third_party/rust/tokio/tests/tcp_connect.rs | 3 +- third_party/rust/tokio/tests/tcp_echo.rs | 2 +- .../rust/tokio/tests/tcp_into_split.rs | 2 +- third_party/rust/tokio/tests/tcp_into_std.rs | 2 +- third_party/rust/tokio/tests/tcp_peek.rs | 6 +- third_party/rust/tokio/tests/tcp_shutdown.rs | 2 +- third_party/rust/tokio/tests/tcp_socket.rs | 3 +- third_party/rust/tokio/tests/tcp_split.rs | 4 +- third_party/rust/tokio/tests/tcp_stream.rs | 51 +- third_party/rust/tokio/tests/time_interval.rs | 118 +- third_party/rust/tokio/tests/time_panic.rs | 93 ++ third_party/rust/tokio/tests/time_pause.rs | 7 +- third_party/rust/tokio/tests/time_rt.rs | 3 +- third_party/rust/tokio/tests/time_sleep.rs | 34 +- third_party/rust/tokio/tests/time_timeout.rs | 1 + third_party/rust/tokio/tests/udp.rs | 163 ++- third_party/rust/tokio/tests/uds_datagram.rs | 50 +- third_party/rust/tokio/tests/uds_stream.rs | 12 +- third_party/rust/tokio/tests/unwindsafe.rs | 7 +- 381 files changed, 30864 insertions(+), 9364 deletions(-) create mode 100644 build/rust/backtrace/Cargo.toml create mode 100644 build/rust/backtrace/lib.rs create mode 100644 third_party/rust/tokio/build.rs create mode 100644 third_party/rust/tokio/external-types.toml delete mode 100644 third_party/rust/tokio/src/doc/winapi.rs create mode 100644 third_party/rust/tokio/src/fs/try_exists.rs create mode 100644 third_party/rust/tokio/src/fuzz.rs delete mode 100644 third_party/rust/tokio/src/io/driver/platform.rs rename third_party/rust/tokio/src/io/{driver => }/interest.rs (76%) rename third_party/rust/tokio/src/io/{driver => }/ready.rs (77%) delete mode 100644 third_party/rust/tokio/src/loom/std/atomic_ptr.rs create mode 100644 third_party/rust/tokio/src/loom/std/atomic_u64_as_mutex.rs create mode 100644 third_party/rust/tokio/src/loom/std/atomic_u64_native.rs create mode 100644 third_party/rust/tokio/src/loom/std/atomic_u64_static_const_new.rs create mode 100644 third_party/rust/tokio/src/loom/std/atomic_u64_static_once_cell.rs delete mode 100644 third_party/rust/tokio/src/loom/std/atomic_u8.rs create mode 100644 third_party/rust/tokio/src/loom/std/barrier.rs create mode 100644 third_party/rust/tokio/src/macros/addr_of.rs delete mode 100644 third_party/rust/tokio/src/macros/scoped_tls.rs create mode 100644 third_party/rust/tokio/src/net/unix/pipe.rs delete mode 100644 third_party/rust/tokio/src/park/either.rs delete mode 100644 third_party/rust/tokio/src/park/mod.rs delete mode 100644 third_party/rust/tokio/src/park/thread.rs delete mode 100644 third_party/rust/tokio/src/runtime/basic_scheduler.rs create mode 100644 third_party/rust/tokio/src/runtime/config.rs create mode 100644 third_party/rust/tokio/src/runtime/context/blocking.rs create mode 100644 third_party/rust/tokio/src/runtime/context/current.rs create mode 100644 third_party/rust/tokio/src/runtime/context/runtime.rs create mode 100644 third_party/rust/tokio/src/runtime/context/runtime_mt.rs create mode 100644 third_party/rust/tokio/src/runtime/context/scoped.rs rename third_party/rust/tokio/src/{ => runtime}/coop.rs (78%) create mode 100644 third_party/rust/tokio/src/runtime/dump.rs delete mode 100644 third_party/rust/tokio/src/runtime/enter.rs create mode 100644 third_party/rust/tokio/src/runtime/io/metrics.rs rename third_party/rust/tokio/src/{io/driver => runtime/io}/mod.rs (51%) rename third_party/rust/tokio/src/{io/driver => runtime/io}/registration.rs (82%) rename third_party/rust/tokio/src/{io/driver => runtime/io}/scheduled_io.rs (88%) create mode 100644 third_party/rust/tokio/src/runtime/metrics/histogram.rs create mode 100644 third_party/rust/tokio/src/runtime/metrics/io.rs rename third_party/rust/tokio/src/{process/unix/driver.rs => runtime/process.rs} (55%) create mode 100644 third_party/rust/tokio/src/runtime/runtime.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/current_thread.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/defer.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/inject.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/inject/metrics.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/inject/pop.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/inject/shared.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/inject/synced.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/lock.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/mod.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/counters.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/taskdump.rs rename third_party/rust/tokio/src/runtime/{thread_pool => scheduler/multi_thread}/idle.rs (84%) create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/mod.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/overflow.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/park.rs rename third_party/rust/tokio/src/runtime/{ => scheduler/multi_thread}/queue.rs (69%) create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/stats.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace_mock.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/metrics.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs create mode 100644 third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump_mock.rs create mode 100644 third_party/rust/tokio/src/runtime/signal/mod.rs delete mode 100644 third_party/rust/tokio/src/runtime/spawner.rs create mode 100644 third_party/rust/tokio/src/runtime/task/abort.rs create mode 100644 third_party/rust/tokio/src/runtime/task/id.rs delete mode 100644 third_party/rust/tokio/src/runtime/task/inject.rs create mode 100644 third_party/rust/tokio/src/runtime/task/trace/mod.rs create mode 100644 third_party/rust/tokio/src/runtime/task/trace/symbol.rs create mode 100644 third_party/rust/tokio/src/runtime/task/trace/tree.rs create mode 100644 third_party/rust/tokio/src/runtime/tests/inject.rs rename third_party/rust/tokio/src/runtime/tests/{loom_basic_scheduler.rs => loom_current_thread_scheduler.rs} (100%) create mode 100644 third_party/rust/tokio/src/runtime/tests/loom_yield.rs create mode 100644 third_party/rust/tokio/src/runtime/thread_id.rs delete mode 100644 third_party/rust/tokio/src/runtime/thread_pool/mod.rs delete mode 100644 third_party/rust/tokio/src/runtime/thread_pool/worker.rs rename third_party/rust/tokio/src/{time/driver => runtime/time}/entry.rs (87%) create mode 100644 third_party/rust/tokio/src/runtime/time/handle.rs rename third_party/rust/tokio/src/{time/driver => runtime/time}/mod.rs (66%) create mode 100644 third_party/rust/tokio/src/runtime/time/source.rs rename third_party/rust/tokio/src/{time/driver => runtime/time}/tests/mod.rs (53%) rename third_party/rust/tokio/src/{time/driver => runtime/time}/wheel/level.rs (98%) rename third_party/rust/tokio/src/{time/driver => runtime/time}/wheel/mod.rs (95%) delete mode 100644 third_party/rust/tokio/src/signal/unix/driver.rs create mode 100644 third_party/rust/tokio/src/task/consume_budget.rs delete mode 100644 third_party/rust/tokio/src/time/driver/handle.rs delete mode 100644 third_party/rust/tokio/src/time/driver/wheel/stack.rs rename third_party/rust/tokio/src/time/{driver => }/sleep.rs (87%) delete mode 100644 third_party/rust/tokio/src/time/tests/mod.rs delete mode 100644 third_party/rust/tokio/src/time/tests/test_sleep.rs create mode 100644 third_party/rust/tokio/src/util/markers.rs create mode 100644 third_party/rust/tokio/src/util/memchr.rs create mode 100644 third_party/rust/tokio/src/util/once_cell.rs delete mode 100644 third_party/rust/tokio/src/util/pad.rs create mode 100644 third_party/rust/tokio/src/util/rand/rt.rs create mode 100644 third_party/rust/tokio/src/util/rand/rt_unstable.rs create mode 100644 third_party/rust/tokio/src/util/rc_cell.rs delete mode 100644 third_party/rust/tokio/src/util/vec_deque_cell.rs create mode 100644 third_party/rust/tokio/tests/dump.rs create mode 100644 third_party/rust/tokio/tests/fs_canonicalize_dir.rs create mode 100644 third_party/rust/tokio/tests/fs_open_options.rs create mode 100644 third_party/rust/tokio/tests/fs_open_options_windows.rs create mode 100644 third_party/rust/tokio/tests/fs_remove_dir_all.rs create mode 100644 third_party/rust/tokio/tests/fs_remove_file.rs create mode 100644 third_party/rust/tokio/tests/fs_rename.rs create mode 100644 third_party/rust/tokio/tests/fs_symlink_dir_windows.rs create mode 100644 third_party/rust/tokio/tests/fs_symlink_file_windows.rs create mode 100644 third_party/rust/tokio/tests/fs_try_exists.rs create mode 100644 third_party/rust/tokio/tests/io_panic.rs create mode 100644 third_party/rust/tokio/tests/macros_rename_test.rs rename third_party/rust/tokio/tests/{named_pipe.rs => net_named_pipe.rs} (78%) create mode 100644 third_party/rust/tokio/tests/net_panic.rs create mode 100644 third_party/rust/tokio/tests/net_unix_pipe.rs create mode 100644 third_party/rust/tokio/tests/rt_handle.rs create mode 100644 third_party/rust/tokio/tests/rt_panic.rs create mode 100644 third_party/rust/tokio/tests/rt_time_start_paused.rs create mode 100644 third_party/rust/tokio/tests/signal_panic.rs create mode 100644 third_party/rust/tokio/tests/support/leaked_buffers.rs create mode 100644 third_party/rust/tokio/tests/support/panic.rs create mode 100644 third_party/rust/tokio/tests/sync_mpsc_weak.rs create mode 100644 third_party/rust/tokio/tests/sync_panic.rs create mode 100644 third_party/rust/tokio/tests/task_id.rs create mode 100644 third_party/rust/tokio/tests/task_panic.rs create mode 100644 third_party/rust/tokio/tests/task_yield_now.rs create mode 100644 third_party/rust/tokio/tests/time_panic.rs diff --git a/Cargo.lock b/Cargo.lock index 6fc787e1e1cc..d3863644b077 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,6 +351,10 @@ dependencies = [ name = "autocfg" version = "1.1.0" +[[package]] +name = "backtrace" +version = "0.3.999" + [[package]] name = "base64" version = "0.13.999" @@ -5544,18 +5548,19 @@ dependencies = [ [[package]] name = "tokio" -version = "1.17.0" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ + "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio 0.8.8", "num_cpus", "pin-project-lite", "socket2", - "winapi", + "windows-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9ca98a042aaf..943918dabc19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,11 @@ base64 = { path = "build/rust/base64" } # Patch wasi 0.10 to 0.11 wasi = { path = "build/rust/wasi" } +# tokio 0.29.0 includes an experimental "tracing" feature which requires +# backtrace ^0.3.58 and the `tokio_unstable` flag. We don't use it, and nothing +# else we do use requires backtrace, so dummy it out for now. +backtrace = { path = "build/rust/backtrace" } + # Patch bindgen 0.63, 0.64 and 0.66 to 0.68 bindgen_0_63 = { package = "bindgen", path = "build/rust/bindgen-0.63" } bindgen_0_64 = { package = "bindgen", path = "build/rust/bindgen-0.64" } diff --git a/build/rust/backtrace/Cargo.toml b/build/rust/backtrace/Cargo.toml new file mode 100644 index 000000000000..a251228e7155 --- /dev/null +++ b/build/rust/backtrace/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "backtrace" +version = "0.3.999" +edition = "2018" +license = "MPL-2.0" + +[lib] +path = "lib.rs" diff --git a/build/rust/backtrace/lib.rs b/build/rust/backtrace/lib.rs new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index fb413e9647a4..348d7cf13f2c 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -1310,6 +1310,12 @@ criteria = "safe-to-run" version = "0.7.1" aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/refs/heads/main/cargo-vet/audits.toml?format=TEXT" +[[audits.google.audits.tokio]] +who = "Vovo Yang " +criteria = "safe-to-run" +version = "1.29.1" +aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/refs/heads/main/cargo-vet/audits.toml?format=TEXT" + [[audits.google.audits.tokio-stream]] who = "David Koloski " criteria = "safe-to-deploy" diff --git a/third_party/rust/tokio/.cargo-checksum.json b/third_party/rust/tokio/.cargo-checksum.json index f15d517a74dd..9ebc8be4d5e6 100644 --- a/third_party/rust/tokio/.cargo-checksum.json +++ b/third_party/rust/tokio/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"b6a95c0150a61774987525cfc386fea67df149d4e2fbc53dc50acd1d45192e19","Cargo.toml":"70624bf691cd4ffb004cc2622e01250c3d42b9e043ed68248151ce3848556ce1","LICENSE":"697fc7385b1b0593f77d00db6e3ae8c146c2ccef505c4f09327bbedf952bfe35","README.md":"390b17bca7f2aac8e0ae1d76893708dd79c93f367d377ab28744d38587a7e274","docs/reactor-refactor.md":"f7dcc42be8ae33446be399add73f234fc217a5d67478cfcac65601058ab50d47","src/blocking.rs":"8e62b2cdc512fedbca4b4c4f983629af035afea4ee7e918bb1a3e9851c8e034e","src/coop.rs":"ef7e148a1e2023657f9e4e2c927760a28b3e7d0684e8e5ccaebcebe90d7033e6","src/doc/mod.rs":"313fe6b3eb0e0e7a9d9cb070cc18e8749f5dd44724b651a90af46acf02a4f7af","src/doc/os.rs":"4af4a78a0af962f94e5e8f67858350e4fedb3a5f990db277a0aa8bf4f7b923b1","src/doc/winapi.rs":"f8b16da97e4822116be6a59ef0d9549a857861cecd584353a7fb02df173dde9c","src/fs/canonicalize.rs":"93c64b72abdca17877d6ab61d50a43765d6aef9e0a9f7aaf41b6b0b7d9a8a380","src/fs/copy.rs":"262180fadc66e5ac2bf1e8389628fdd039f14788e66a4f8b10e10633e7310f20","src/fs/create_dir.rs":"233cbab2579a787614aeee88845a57f1578e9c2e064a3456c799f61430e911ad","src/fs/create_dir_all.rs":"56081d541caadca0fc59e84d55e78e702fe9373679598016224ad0b072b189a7","src/fs/dir_builder.rs":"b5b21229c7bf15e2074afab1accfbc392f3c69e546c7b652cfdc8e90d5a4a301","src/fs/file.rs":"193078a2addd11de302024959c79984f13933e5e420098641fc71327bdd90d78","src/fs/file/tests.rs":"b00dfdb1cf5347e773d268447b0cb1ae4aafd036abb7703ba0d6bf404e155cf9","src/fs/hard_link.rs":"98cccbbb3719baee11c232e79723ab1cb3d6c8056bddb109c4990fe2c236c1fb","src/fs/metadata.rs":"782a1a5dbc2cd6c40e928579fbfcf39e5f1de28def78781590c0280acdf02960","src/fs/mocks.rs":"d380cd15d2644ed240f66a2acc7168b5a51f1357a65eb764cfadc98c42676013","src/fs/mod.rs":"eced6fd2a09a6e458556caafbd444753a3577a5dbbe11688f31c0e1e122879c2","src/fs/open_options.rs":"e1f78d96005082e6886de6fc8e6545cdb72e4e4fd444549b53c5bd7addd8510a","src/fs/open_options/mock_open_options.rs":"f08d384602bbd165eed4cf8811a19512d62c4d9573457f8d22c78c77b1091629","src/fs/read.rs":"055ae8b6ae96ebae2d05f8780e7592bb587a742506c6df8ee8b380fc7d2820ef","src/fs/read_dir.rs":"76a90e7a465da2e977392b65265dfa766c19d8b25967a278a5ac20f6a0543243","src/fs/read_link.rs":"93c104a21253372fef7056ab82e6065b0a1a8fc9be8b7329dfd5a8dd07c618b0","src/fs/read_to_string.rs":"9e5b2d476a6084e32a92c5421a8abc9d4f335f4ec677beec4bf8bfa109d7d106","src/fs/remove_dir.rs":"96475771e9c52678768288d8df6814216e0801bebc848481597ad34e829a5854","src/fs/remove_dir_all.rs":"b85abd05c7ab64ee8dc6cf5663a11e713aa51b357759ef660ef3cae3365ccc42","src/fs/remove_file.rs":"1cdf8bf16b3a164c594dac8773b7d1f9ebb28de169343184d34d6aac3b3a7eaa","src/fs/rename.rs":"a97875e92626fa46e23fece7b8698c9c4cea2bae8f1be8726f30ae6fe80ae0c7","src/fs/set_permissions.rs":"8adccafa475dcfc1bc3989af73374d90683c1be4953ef812e5fd606f968d7b7a","src/fs/symlink.rs":"32cf3e906531d30ebe6d8be7ee3bfe049949759b566015b56d0851f51abcff50","src/fs/symlink_dir.rs":"5fbd05365555ba7942ffc1c2dfddf201ddad2cf9b005be2ea99849a473fe982b","src/fs/symlink_file.rs":"a1170fd40a000dc449de972267f579a0d14f50dbb39466f985f183fdcd1d3438","src/fs/symlink_metadata.rs":"f5ce1e05f137da995e3e0d9582bae0a5f7ef4251285c64e912b0eedbb068b395","src/fs/write.rs":"1ffb734d31748bd879ad398b0fe99bdec569782b42677022957db2cae95c4d2d","src/future/block_on.rs":"ef7fd744d2727f169b6d8997e2202af88a256bf41a894c93a4490ade65fa0e57","src/future/maybe_done.rs":"9042619f2a907e81763ac6082c080faa28af5d571dd49e82b60f3f14d58598f3","src/future/mod.rs":"6f28857347c50cd6467f1ca7d1b1028b633de7d142c3f3ca710a8693d1e839f9","src/future/poll_fn.rs":"e1a8393f18d6cfa9d7d0c8912b10b5b65cc9a2b507d8d1ac1c5c3be4e2ba057a","src/future/trace.rs":"c42712a8d372922eba7e05cd21382fe5df5eec02cbcc870062100b59ab99654f","src/future/try_join.rs":"0ea5a069b17a34bbc091acbd74b9d51794a55a85dfa63fe2404d5ee91a4f0038","src/io/async_buf_read.rs":"b37caa8f6c974b3c97327c635218803e573c531d4197950840549aa794357c99","src/io/async_fd.rs":"b3eb76f964bed4ec0b3d20059a038e178c13e462f25860b824d7b754109ede2f","src/io/async_read.rs":"f52c8d2f4a283c0dc8d06dc974484749973125b0b691bc0c3d100972ac67cb92","src/io/async_seek.rs":"a9a0df389ff2be3d79208ec475fcfede46a86f6ea0b822b1a4ce8273ec714b0b","src/io/async_write.rs":"198ed6a475b6f093fd2ff15e618d816d7e33cb8bc28d2e2299533e5df0bd78d6","src/io/blocking.rs":"d6613950dc10f17287ff396d6acc7f386bed1ffaaeb71dcbb10ad4ffcfb29ed7","src/io/bsd/poll_aio.rs":"893aab3d616e33fbb6ea61e6d590e80d2871e676c0647a58041b16e4698ca34e","src/io/driver/interest.rs":"726c813211a8606f68e58a2693cccdf3788f252aa1e9700af088ba756ba36c1d","src/io/driver/mod.rs":"66325853609db8e4cf51de3f4887a1de0a5c42879128aac66d1aa2cf39da6577","src/io/driver/platform.rs":"023acd3f2703d241b3e91ab6e4d4c0bc5ccc3451655fffd9f37938224f915494","src/io/driver/ready.rs":"e4bac7c50bf7c2d29b3516e9184c2a46cc4631366ae940c41cc5320bb35a9250","src/io/driver/registration.rs":"b3a62b64014181839c8d13aa5d13da27d8d173c18feb18c3f03246b08404c0d2","src/io/driver/scheduled_io.rs":"d8da8185d1c72af922746737769a612457f053eb529a465d242de641fde1c731","src/io/mod.rs":"1d32445c95577065827eb98ef842bf4c9be842da376ef95c565c0a65820c55f6","src/io/poll_evented.rs":"e3b9765a8630b497be71acfbee58fb00174399e4755e04859d418ed82350094b","src/io/read_buf.rs":"8a79bdd31b7d77dd585db87dde45f4208f7f131a6614b51f5a776d512b435c2c","src/io/seek.rs":"e9e346fc926c3360601b80a8319a25fd0567dd6f77fab666694e9787deaef633","src/io/split.rs":"2c6982598a86620b76826ba6065953e9fc78a0434652fe97d82d79f7bbcb66b3","src/io/stderr.rs":"8f4d9fc5e596180171b68919bde951c49d134446d30af06dbbbe413ff2882ff5","src/io/stdin.rs":"5c15de00a7fe2ffee2f48f2ecb2a4fa765f3de891f8cdbc240e2988db7cc0d13","src/io/stdio_common.rs":"6e54019e53594a632dd3536e575ca11ee7b279977ee92c556ef39c46255f6b1f","src/io/stdout.rs":"2c1b11ae547b0eec06e2b21039655a2acf34125b789548c7bd362babf5b05cd2","src/io/util/async_buf_read_ext.rs":"7f7fde910ecd9e526bb85882a318f457dedf6ccc2cdbc693112562186dfb78de","src/io/util/async_read_ext.rs":"f70825f0be7ebecce5473c53c716cc57f1123e5818f7b917444c3892639a5e2c","src/io/util/async_seek_ext.rs":"804ea19c98a28aacedc38852f692f59c52f1327807a0620c1740182ac6b01e02","src/io/util/async_write_ext.rs":"6256ed8320aa225fbe9ba159cdc20b08e1e22bcab4c090fdb1dca1cdbc2d97b4","src/io/util/buf_reader.rs":"670a58f404e5689daf1f2b3070b0b9e95fef96ad19f0e8487f294e8a2afe558d","src/io/util/buf_stream.rs":"2246fe71b707c15d7168c5da5ee158cec6e854d4fd11b685531c16a9c3cf2c6a","src/io/util/buf_writer.rs":"f9c3e018c9f9177fb6d910096503caee727bebd3c36f5f67dca2c4c55044408a","src/io/util/chain.rs":"5cd8df2cc7bbcd18ca2336a78507fa8009c0a9e595f81730a8c16cadb8f731a2","src/io/util/copy.rs":"34f164f2169caa58fc9c43415108e61c662aece2e2f3010470d89b3fc3e2aedd","src/io/util/copy_bidirectional.rs":"0f72d957fa2d1b3517224e34c65f06016f5e04857d45b01c368398fb95e555f5","src/io/util/copy_buf.rs":"b029d0ee8da5301a06722d425828418027234a8b111a209fa49ead161263aa8e","src/io/util/empty.rs":"48f23da224ff6c5d3803b24e4d5e6a18c75b31344a4570081bd0b3c3e0244fa6","src/io/util/fill_buf.rs":"223725d828071e923f25d2d49a0f6e470c411a6d9ba225700f2dd8d5793601bb","src/io/util/flush.rs":"fe3b4ff226e294843b8cbea9dc4e02d581582b78ddaafce137c96e290699c718","src/io/util/lines.rs":"1d9f9b99567111c911e72a4caa2abb19b277f2cdd0ca3268ac5ca6df5276259f","src/io/util/mem.rs":"222519afeb028305f7d7165e8ded58aceeabc5935b223ae73bb2413703565ddd","src/io/util/mod.rs":"6a9012d78fe2bed8240e7a628e9421cbef45433551522624065fdcbb329f3594","src/io/util/read.rs":"01c1113b6171c83ba2a0ad774703c3139b0ff11f47953f1b50861334d44f86ec","src/io/util/read_buf.rs":"a87be2d115c09a6782ec8cadeafc92fb1fbe534580e71540087c3298a03bfca2","src/io/util/read_exact.rs":"4a8650fd7a885963a0fef2bec24c17046c5624e4dd7fe229ab3f33c4a92fc66c","src/io/util/read_int.rs":"49da230796335df584832cd7deb8370b4d1e0350d743046389a9d9ae17dbd94f","src/io/util/read_line.rs":"9cdb2d778b81bc50098a6851981ed9f541bd0c7896c0763b811971b5a598b7e8","src/io/util/read_to_end.rs":"7aca8032ee911467c24ce435b7bcf023c72ca6f19c2663269ff50d31913d6e01","src/io/util/read_to_string.rs":"fafb5463b013cc8f76def3a505dbebd179afc95bde0e2ca9388e428265788924","src/io/util/read_until.rs":"b2a2a7c434439fd2c9269494e59edbe1f75eb45449dd6be384663a6ceaf137ac","src/io/util/repeat.rs":"d4effcd81338831eb373cf2db972a99218b8379b91066940a732edcf4524c7c2","src/io/util/shutdown.rs":"971454342b4636fbd68e123d59d87017d81f72afb410c385846069b11def8efe","src/io/util/sink.rs":"0dcb794e48ca9b1c28e5f9f2051073ea0951a54c9c7dfc903ce9e5489d3d8cd7","src/io/util/split.rs":"03a59adccda29608886e38a1f484fbd4d6a6019180c4cfa851372d250796aa5a","src/io/util/take.rs":"ef080ce27d23cc23b9f99fe14ddd56349a3cb1442aba18f8405c30d20017b074","src/io/util/vec_with_initialized.rs":"06f3a452c0158b4b72c6fdbddd6505eec8ce1893abce0420db1e79897d63380b","src/io/util/write.rs":"20d14ee545ab1f67732915522e97808d1ddde13d151505c1289b596be519f7c8","src/io/util/write_all.rs":"906ff3fb24c6a979b104598f9a8229421bcaf2a4218c28069504b34a218241f6","src/io/util/write_all_buf.rs":"5911bf673ef89097938f4e2e38d9012865b28a0ce5ebb217ebe0e2507de6c1e3","src/io/util/write_buf.rs":"ab51d6174de24cbb729ce77dbaeea27e16059b8253e4830d8243ec5f08a08a8c","src/io/util/write_int.rs":"f321e69b0c7c01728b079e9fdeedb96c26475667e8b259d0c5f4a83d060990d1","src/io/util/write_vectored.rs":"7a335a9f796daa048fa9708dc44d32d2567b36461a6d88f07893eb31f304b69d","src/lib.rs":"6a9ed401d3d763c6d13faf5966d6a8b9b5ec8866956e2838190c18b5dc2c831f","src/loom/mocked.rs":"6db5ed42cd469ac1f98d04c535e59aea174d6c06aed5f9f0c5b5254dc6e476b9","src/loom/mod.rs":"b14b9333a7a21bd125a4ae82f01e5ea9c9ed2f78d7d1ad49a13d9b176f1fe8ab","src/loom/std/atomic_ptr.rs":"16d7e6f841697020aa315a303f26cca714c35a96c4912ae45b90ad3ab0715e28","src/loom/std/atomic_u16.rs":"8793a4927367270305a6a3a68423ccc838cead0960ab1a7cb345681182916c14","src/loom/std/atomic_u32.rs":"39889c3295f5a201ecbd4ce3f5f942d88931fa9988071f771935dfd0c6c3443c","src/loom/std/atomic_u64.rs":"2f8314f2ef093a8701d1ddfcd88278da52448c0ea9f0505bfd33c05e3e26f8fe","src/loom/std/atomic_u8.rs":"98f6baee74059eea2fc950dabc273a2fcb3518950015fb3a5acb3dbc58ffac03","src/loom/std/atomic_usize.rs":"ce7e840ac029a91adb90d7834c2bec3a8ef5fcf0b311de0bb6900c0be199301f","src/loom/std/mod.rs":"0503d16f472852d2fdb1bf9edced988a264ec933e9ba678de65f9d72b5e88d6c","src/loom/std/mutex.rs":"83938246987904d2cf0fbd76566170e62ab89a2e10dc309f8aa4e149cdaba74e","src/loom/std/parking_lot.rs":"89c0b87687dcfe5b200a82f78ab9517b40c42f07da4b9ced59067b76a8c391f6","src/loom/std/unsafe_cell.rs":"05e2b4d8e78b0435650c6da6239ce8c53b4de2573e64ba098354b049f23429ec","src/macros/cfg.rs":"2b624d6d114560809f839b49491dc36e0108977dd2131a8f02bd20c07be8b654","src/macros/join.rs":"54fb8935246c4524e958a5bd80b9286f3c523c3a613adecfe204bfd4504f925a","src/macros/loom.rs":"80d2e4af9fc50d0bda1b20b95f8873b2f59c3c0e70f2e812a6207855df76204e","src/macros/mod.rs":"913c824be0242590352d952a2ec445ef1d3d77e1512cdea4e2be7f4956cb9bf5","src/macros/pin.rs":"294e5644061e41801dcee5494b7334439e09af0b6219ce164090feb624864631","src/macros/ready.rs":"6efd4c866c4718c3a9a7b5564b435e2d13e9c1ae91fd98b1313d5e7c182942d6","src/macros/scoped_tls.rs":"d598cf0c6e57d91d56432898ebab741aec3fa07b08e0ab5f91dd151127f4ae3e","src/macros/select.rs":"4832b4aacaff0f7c79c97f3c707e613086accadec6a3f15857c94832ee35aeb3","src/macros/support.rs":"0cf0789ba2dc5a196ccbabba9ca9f1861a71284032b98dc7c372078c12c5d5ce","src/macros/thread_local.rs":"8602495ed102b63e3048a261eda7483dc9a24b15a74d7059c31635e8f45de19a","src/macros/trace.rs":"33befd4533a3b2b4b22e246033f2bea8930174a7da58adaa57dbf20931275bcd","src/macros/try_join.rs":"fdeb6c3d4b56b81cd62f2532bf2373494fecd301e3c3a4afc66e4f1651bd7a9e","src/net/addr.rs":"dea03c41778eae06672d294bb05b41e31cee4efae7059bb9cd83a9c6f4664a7f","src/net/lookup_host.rs":"c7a21d735225e316253d822f6b11a17107e6a8db004f947a54d8bc12ec782baf","src/net/mod.rs":"b75d65aee9871a9b5ead894a01199d0af48f993e12a5e49041a91148739876bc","src/net/tcp/listener.rs":"c0f9d5e115da4127ce6792122e48d39dba1888928411241a3e4982ea68f38767","src/net/tcp/mod.rs":"d33297e086d7fcc6c46584782d7b24f65578bf9895844f1ec6cde95cc5b8902d","src/net/tcp/socket.rs":"df83f65dc8ec7de1619217697d43c13f3260c7b64f7b41c42ee850b0d5b4760a","src/net/tcp/split.rs":"e477c847baf20c5f0235a40720a60fd3564cab66bef37f48c2a17cdf95de96ad","src/net/tcp/split_owned.rs":"55f393899efa77a41571e64d96507a374c413050ee866144c8f57b3d2935b2d4","src/net/tcp/stream.rs":"3dec8d775f1cf050ea90359902f8c4aa444a9cc04660247d48762b33e7db1b04","src/net/udp.rs":"60943a9703084926639a4ac3bfbc62a5276a8cf9ebde479f11e9992f0f82ea93","src/net/unix/datagram/mod.rs":"fc48924e5d1e551405b0708a2d122473cdafeeee802a5722f64b4cf41a1c01da","src/net/unix/datagram/socket.rs":"f888d8c1499693e1b4fd92e17596a310c81679a0dcb5a3f528d151ec36158665","src/net/unix/listener.rs":"eb9f0deb2f6f292b3224f5fc4c6aa12fd4e842e867e75ededcf4859276dc1b8a","src/net/unix/mod.rs":"2c18ab3fef385be4b1603cdd391f557f36a3188b8bebc3949647b27bbd6d84b4","src/net/unix/socketaddr.rs":"66bf18321a81baeb394a7094567632b113e44e12d2643109b6a97c89d919bf3a","src/net/unix/split.rs":"9f6c51cc59c8b2c6bd8e7b8f918486ccd8d200973e240c58ce26490569db2182","src/net/unix/split_owned.rs":"e0640e4fd2b6be95139f7f1aa014c149fc2dc834e7264382b7ff84750b93e10b","src/net/unix/stream.rs":"3a9073f8c6ee7dabe170c19b06e09db0bb280c7e3f20aea8a6ee49325f60d7e1","src/net/unix/ucred.rs":"836fe68abe151abcaaf4d9bc92025a12030f1c2b7beb314cc453d0d88aa316d1","src/net/windows/mod.rs":"a1525f35c1acb92b15bec788a625b76acb42f9424f392074db697102ce79760d","src/net/windows/named_pipe.rs":"670aa7d368c84b92c29ceb3cd21776cf1e8dc310aa4f46600ab0f4cb9d5637a6","src/park/either.rs":"251c6255768445ca2bcfb037b1fcffa05e9ca0f4f31923ffcf341eb3cd794180","src/park/mod.rs":"0e8d343742e6e498c4f4d44a837d01da2e75012aada2117641937c56f85abc8f","src/park/thread.rs":"f5c4885e6c7e3f6e33e87774e79ef3bbef13fc7a4e7f9c5f27a86d74fec8f275","src/process/kill.rs":"2f98bd1bd28ab37bedc34ab7b737760407ab5315420538acbd18da31d2662d94","src/process/mod.rs":"dd4724fa9dae1c875d1ad8c7b6aa41f4a386851dfadbf0bcb11d29204ae13583","src/process/unix/driver.rs":"073baa2a7b1065745793a219cb7045907f339f71945943dcb3c52a2cfeb7ea45","src/process/unix/mod.rs":"ab58ab44dfd8ec1da5dbd064faad0ffdb10c2c70c56e741cb5726648dec12f4e","src/process/unix/orphan.rs":"9927c3c918bead0b56b1a9256b8c8eba9f9f16f87f0c81866278b65bb96a1f72","src/process/unix/reap.rs":"62868319849b8482d8d927dcd00cc8a74b9af61fd47494b39bd41fe2f4dcf0b6","src/process/windows.rs":"feb605a16ea975886b38a66f30b3aad97f7e9ae73d35eb2ad45ccaae36d0e4e8","src/runtime/basic_scheduler.rs":"f45e55922b4eee2dff049496eef226ccca17ec1c308adaa394982fbd0c97e0c1","src/runtime/blocking/mod.rs":"26ce33f7e51e3afc1f18c2e2e110420f5d9956f9c8738cf171b38d4f853bb8e4","src/runtime/blocking/pool.rs":"47cd86a446428bf7fd1e44c5b7fb0410ce00cd9281002d26601bda644ec62393","src/runtime/blocking/schedule.rs":"fef31b442b9e743a4e0f3d490cc320ecab17b475f2be7c995a57fd2af7b34b88","src/runtime/blocking/shutdown.rs":"964c67d028152e4daa3d424d90297f4589973114a527dd081c90cbc6c80f6fec","src/runtime/blocking/task.rs":"60b68f1dc13c84a64376f078a0eed4cd9e9f160e9ac0a069b2a8b11eb748eb5b","src/runtime/builder.rs":"4380af8e914969e4c9a092c45110e856d95dd9b7e37c93723dc1866f5bc3b7be","src/runtime/context.rs":"fcaf27b7a953435fd0c3204f259d229a881c66e89eb0c0293d40750dc19a0571","src/runtime/driver.rs":"763324ba05d5901acd060532551cc25c539ee9c1c9a5a87b72450fe74f2df394","src/runtime/enter.rs":"76d42510a6f0bf159746268c316071d44daebf7523347c032917f55031f7880f","src/runtime/handle.rs":"8f0ddf7da5106950d77d3c6ca64fac88798f6e13cc9c84ce6f427360a3b0abe7","src/runtime/metrics/batch.rs":"fbe9c32fb1c5643b7cc3a2f563e2bd1582ea068a5c1215a151a50f1855c2719b","src/runtime/metrics/mock.rs":"0daeb89a321a7922ff28432122ece55b818ba60469d63531e9a3b29eae110647","src/runtime/metrics/mod.rs":"4a97af964e1b3b0cc1866336ee42e9aaff64f9b87e8bfad691af9ecae65c1d0f","src/runtime/metrics/runtime.rs":"628ca61eac20eccbe0eeaa3e1bdbfc855005e635971e5fd0fcb011b145a2ec69","src/runtime/metrics/scheduler.rs":"27ac9b6efc0d7bee874c270d5e6d46b8f2bc7a78e620b474a6b00034082842c0","src/runtime/metrics/worker.rs":"aeb298a0a517bcabe19d097b9b4500193e2b0bf974e4c6e90a51c3c39e3bf42a","src/runtime/mod.rs":"7cb4660e62db48b8af4657477d702ad82efee34114fe90ce14942c02be2daa4c","src/runtime/park.rs":"e1d66e859e402068e70580277b4b128b48bda145de77ddab90c55b8a19799083","src/runtime/queue.rs":"6f4a02d98f7784979511bc3a36995f3cd6b6bdd412c036f2b06c49d22a1cccae","src/runtime/spawner.rs":"05ee54be6a380640aa7d7a9107d81b9a80c5c479f65f70be99059c6f4062841d","src/runtime/task/core.rs":"801361712c9a930478dfd7d55f738d0c7374e0676366bf6dfa9891a3ca64737c","src/runtime/task/error.rs":"eb3ef53fbcd235b44bb25c759f0fd8fd46a0548e299d36fdf7785dd55d4e816a","src/runtime/task/harness.rs":"c0182d7f6e4cf8256367aa5ad5e77fb16f9314a987e8e4a0504751e013734e9a","src/runtime/task/inject.rs":"34cd85d0d9d241ad1bc34d710bac708ee2f3fbe43bb68732123a30ed99ca2e36","src/runtime/task/join.rs":"358c88e23e62f4577c5534a54732e2981c6f7daba05860550ad296a466c9e3f7","src/runtime/task/list.rs":"0c671142c97b3223fc060f8c4ad281c96d5e8edcd5142bc704e612dcb8820d97","src/runtime/task/mod.rs":"5d13730713d45db479d9f28da2bb59bd24d4e58798ff5cb5da26468304d5ed17","src/runtime/task/raw.rs":"78fa51cd21505a9df06ba1f35d8e3dbc88093e7b2e066cdda9b65fbd8827ed7e","src/runtime/task/state.rs":"6b872d5c489daa68091f9b17be8dee645b5c960a1eba8df8852f27646202d20a","src/runtime/task/waker.rs":"f06c05fa37d1974218a44922edade90e0fe533e06e2bdd111fb3219e2393cfe7","src/runtime/tests/loom_basic_scheduler.rs":"9ec324867d9d123bf93c8f1c9feba8f9b57689122a1efa784adff425962b2d65","src/runtime/tests/loom_blocking.rs":"4110de132b975fbfff2172457ed0e75ab84e9e643eb8099770f86fa173e12215","src/runtime/tests/loom_join_set.rs":"b4d8701e530376ea62249a6329af7fd0c25d6539fa8a530acb0b26acd0e5b54a","src/runtime/tests/loom_local.rs":"69cf7fa67da9e3efe3eee722791f811efb395dcf821780051e62842654de09e3","src/runtime/tests/loom_oneshot.rs":"cb0cb66e1014708a10a2236c2dbbac9441b6a6e36a49c419fa3a51af62b054ce","src/runtime/tests/loom_pool.rs":"30ea07a942c39bf1eff70cc6d7ae10cc5f0162c0389029ef0bf8b14ebaf4533d","src/runtime/tests/loom_queue.rs":"1906ba6c36659a48236245baeedb109c58ce2331e36041c3abfcda5e0b285a80","src/runtime/tests/loom_shutdown_join.rs":"2f48626eb0a4135d480aa5bf72e514c4abf39aa17711c96d3829f93c03758557","src/runtime/tests/mod.rs":"0432bc4633080e7a85de00df15831f277c94f04a5829a761d0818a31ed47f01b","src/runtime/tests/queue.rs":"d1cae3668a51719c6facceef669846f5ba532e0006ef1bf29a187e1d91bc6b03","src/runtime/tests/task.rs":"52e9bdd03d931bd24020214ddd2390dcb9ce5aca36c039853f4257b42796ab1d","src/runtime/tests/task_combinations.rs":"9ec3bfdd62aa8beed6738b36870a383be9db452b67dd4a08a6992bb23fba8564","src/runtime/thread_pool/idle.rs":"b048a6ed70ddfe08670b0f99535566be7eacafd75df9c3a9447ad14c835ca326","src/runtime/thread_pool/mod.rs":"d4fd42bebe199b63969bd69318e7bdb3295c2c24f73d436c603355169c44a5bf","src/runtime/thread_pool/worker.rs":"2e8520b49caa661fd2439ff2db72cf41de861fb4bd87eeb62170850d19fd50e8","src/signal/ctrl_c.rs":"3e1d98df851b9ea1418f607c260434e906ada4e3d6c14cdfa5c81a0c4d55cdd3","src/signal/mod.rs":"e326c839b0b8d549aff9e4e9204b99c544ab49f32ab2c5066fe3c7b8bc7767eb","src/signal/registry.rs":"d4eb2577277a9909b4d6f79d53ca76774a1d0601c31609bf1929a829027fd4c1","src/signal/reusable_box.rs":"5793ac41840ac8cafce7db78cf6c1fa94943829d8ebc2653c7ff07de87aadb0d","src/signal/unix.rs":"c88b98373a9eec9f509e45ad94de9e725de3b0d1c1552d3c78ee0aa943ebb4e7","src/signal/unix/driver.rs":"78ead5545500535c20e061b0e5809fcec15369286083ce9208a0a1fad771aa37","src/signal/windows.rs":"18680b96548ce1269615587b803186eda94521f9af8519f5ef6aa04cd706c816","src/signal/windows/stub.rs":"e080a121465461e6de2b7756956975cc755820bb7c54248b903d7d18aa79e255","src/signal/windows/sys.rs":"8a7178e52c33689420e54bbfc2db78f1b15e3c7f16632594d6d668ca1d865fb3","src/sync/barrier.rs":"4dfc962897d02d7a71ce16e2cd134a72dcc10e5efb79abb66743a081a1cbac18","src/sync/batch_semaphore.rs":"628ad5e2b2dc4bd1166519e9beacd34319e1d9c18f04833c7c909804aad0b06b","src/sync/broadcast.rs":"123abe24e57759871d42b6dbdd90809ff47abd06908a140e14870cdc58c0e0b0","src/sync/mod.rs":"339338d51c28f55b70c2050d0dfb527d5111fa321c7b0be065bff41150cda215","src/sync/mpsc/block.rs":"c8ee862dabbecaa67b848abf62e97a5c445ab99e4fa7ec0a8c024e3c7bec0bf8","src/sync/mpsc/bounded.rs":"69e63335f55717a1e396f39893c74e5404f97a852ecceb5e76ca98613664bacb","src/sync/mpsc/chan.rs":"2c3564e8217fcbaf6d69e0217e71f8196d8617a158c7c8d9aa8195a1ae50d242","src/sync/mpsc/error.rs":"f445a7c3894799ca3be60f714bf4431766eb27e159209eeafd99e398ec6c7976","src/sync/mpsc/list.rs":"45b165d0965695248d63bcadc17e571bc81dc2e9d7448e86517d5a9a818401ed","src/sync/mpsc/mod.rs":"abe1eca4a868e485016406ec59b46a7251bdf97ac0dc6c0a8d2985937b65b8a3","src/sync/mpsc/unbounded.rs":"01d047a3e2fe57a25b2fb13e489765ccfca15cce110242565087afd913cc8afc","src/sync/mutex.rs":"3885f93e3e5cc7051ee62c66c94582097cebbf693494e232726bf65aabc127b1","src/sync/notify.rs":"654d4f804874fd220f3f5a28752220f031f5c9c15dc6c8bc11d3073641b9a3ad","src/sync/once_cell.rs":"935cdcb05ac86fe10731c6946f3c89b52851da61e42297fc6a98eb1418eb6a53","src/sync/oneshot.rs":"67e0504b7ba54768d7926ae7cfa4884350412344269b0deb44d282118a5fb540","src/sync/rwlock.rs":"eac7ae4ee97bb4962bcbd8f56a1256933be30db6db1394e97d30d866526dd3d5","src/sync/rwlock/owned_read_guard.rs":"6f1f39e8a84188cd3a57683c46da492257c4752d9204a158d04cab7acb26ff69","src/sync/rwlock/owned_write_guard.rs":"ffee768eb69cdddb35e51fc6b9547762e9131b46c236e44d87de86ddd7d65f8b","src/sync/rwlock/owned_write_guard_mapped.rs":"ae79b73500cbdbc3f8acd9a456bb292fa97cf0c3d509434895c906d84c0ce101","src/sync/rwlock/read_guard.rs":"edbeb6fd9fa848ff8e4749c5e3a4dfcdb8c82842ca8c91edb86c139af854ea79","src/sync/rwlock/write_guard.rs":"d2c7fbc1b32ee6de2f4aecf546034a620c4427fe244a9fb4e8e506e4ef9f5a43","src/sync/rwlock/write_guard_mapped.rs":"cb632f2653efadb3717d1cac0112891f676218431155753640f775705e33bd53","src/sync/semaphore.rs":"477b93fdd093a1e76954e97d3681074b28a1f1fe2a5dec751ad35a57960e4e15","src/sync/task/atomic_waker.rs":"41e9e05522254afbacec9895390d6f95498e413a61d8f654f9c421c808e7f83f","src/sync/task/mod.rs":"f5e38105c7f8a942c0e49b973bad0a8c2a1df81deea19f3c5228edc4896c1725","src/sync/tests/atomic_waker.rs":"388254c09ba995b46918980a61d440a1eb3f81db3c8abec3e1843301ac543547","src/sync/tests/loom_atomic_waker.rs":"984b52699c47383b9b62e6c4ff93fd458bbe64cb4ca836463adbaf94d27c38ee","src/sync/tests/loom_broadcast.rs":"b2c6f138707fc389ee7d91109bc38093af9060b3465e68d3543cb652e0070406","src/sync/tests/loom_list.rs":"f0ce15a0f965fe558a21bca24863c712156eaeb10feb8ef91031a6d6e3cc5dba","src/sync/tests/loom_mpsc.rs":"4883352b9d75a81c878609613545ae14910eca4f2e7f3718053dfdb792aa0760","src/sync/tests/loom_notify.rs":"c571a81799e617222f1d07e2b9b0ae9b21ead982aea90205757b6038859abd04","src/sync/tests/loom_oneshot.rs":"c3596c15692b16e7cb8cd6957362adb3a98b3d7f16c4a4262f19a3a27f262b03","src/sync/tests/loom_rwlock.rs":"80ec00bdcac838806d4d9f711cb154e02f22913ba68711855c39ca92028d3e4e","src/sync/tests/loom_semaphore_batch.rs":"c6f69b8d5b2e6842287ed34638a9045095d9f94c86ba6bb84c1224bbe10026ff","src/sync/tests/loom_watch.rs":"558ad9bab3f18b79a0d46548aff24288f1c6c5d0581e51559cc2a82cccd0696c","src/sync/tests/mod.rs":"1ef2026ac3dfbb70a437b8be441deca4b7b5e72638b71daf753b89808cd3c9ee","src/sync/tests/notify.rs":"d43bac90fa8a5dae3fad76d33f622e55767205fef2b4e8667f0be84d838a9146","src/sync/tests/semaphore_batch.rs":"0046207b0711f4d6cca228a3abb6523fcb5a1e6642255db4d469cb5246c86f7a","src/sync/watch.rs":"1b5fb2ef0108c4e8a9393943d50fe3b1aeb1ff3374c5d59ddf6524d001a20565","src/task/blocking.rs":"34ed9c27dac697ea9911cf5818be7589b3fa0dc9e2594ee1e3a06ea16c34f41c","src/task/builder.rs":"ec7f53b165ec941e2066ab01d00f68a2ea23074b85a894204a4ceda5ca76c21d","src/task/join_set.rs":"56afa4dbf1a618839da837090618cf262b629495776635d65d6db969ea70a8cc","src/task/local.rs":"6698564a6f3f7991ae17cc5d11e6bc0311cdaa9396caccfaead53a5116151e27","src/task/mod.rs":"e3d51a6f3be1ee1ae75b7b05bada2912893189474e2cbf5f09d91c64f5e18f3c","src/task/spawn.rs":"c315a3cdf3d8e493f85a3693a65478619a75a86c7e563cfa6820e93bf44850c0","src/task/task_local.rs":"5ea65d92a4766760807b1729b61e92fa34d280deebebc19fe486dcc18341d03e","src/task/unconstrained.rs":"a39131550205cbc0b0ad5e4cdc1f3326ab243774a31fc986cc4ab0f27f356459","src/task/yield_now.rs":"2734c883ebd307ffb6e1b43c336f3537be96074886de3d362b1b7a3e1cc67120","src/time/clock.rs":"5a653de458a61c1c6343167b5fb30dafdb5fcf29d1103869a03cbc228d6b15c2","src/time/driver/entry.rs":"bd073d1392fce2f128206957c1b09fcf571d95e104bdc39423d5267d4362d492","src/time/driver/handle.rs":"655393617dc953ffe08e34e7662b20c64339f4cf96092ac763beff1d8b06f4aa","src/time/driver/mod.rs":"a8f2dfb5aa31df491310fcf213715b782f5bfd14034565e345fb5f2bfc7c70dd","src/time/driver/sleep.rs":"52644c616131d1ee617a8c2aa41ab7b5a30433d960a1d808e551a1d34fcdaa58","src/time/driver/tests/mod.rs":"bc6b7a8d7ea5002f82e77bb1d848ae390c56e293e6ae271e8eee59ec6a8d1acf","src/time/driver/wheel/level.rs":"6319e3537ab9470ff3404a3043ce4404b478728a4b48f884dc9db11c34ce4867","src/time/driver/wheel/mod.rs":"138edd95c6a095d2edb53754bbc40bc719ae3a8bb10b4d4fa2daf67c27f6a508","src/time/driver/wheel/stack.rs":"3adb5316b814656d6b271c85b99ecf2f23431e966ab4cfff26ba999ebdd8915e","src/time/error.rs":"2ca1e032254ee67087aa15255d9434067c65ef84d143cbd01280f037bea8c19c","src/time/instant.rs":"056bbebc2f9e3d8572c0a1ab48610613415b34db4b2e6494787d14834d4f06ce","src/time/interval.rs":"ef703ae199aab4fc6ae719a0db830ec57a08b8245a1dc29043adb16c02e9a30f","src/time/mod.rs":"e1602cbfa9a93479b580fb1e205751e0551a0233140390dcfe5177b2cca252ee","src/time/tests/mod.rs":"d5a5971423d84c60593baa494e71bec40655266afcd532545d70a680fc014cbc","src/time/tests/test_sleep.rs":"78c97ec7816ef02b89749b0c0384e2bb2f34a3e5f5daa1480729fee91e69bec7","src/time/timeout.rs":"43adefcd4bbd41cd261996007e5667747e4ab0486aa62bc7cd9c78e44e797712","src/util/atomic_cell.rs":"6e156b730f69c698cece358ff5b206e8dd88083230fd49b2fc4a66a539b92553","src/util/bit.rs":"ad3117c9a8b1e312114a72e9b315c37137af61da51ea458832d46d2c1b88879b","src/util/error.rs":"d13d6f901192c38f4f3656ff0f2fe81b82d37ffc2a10644ea9ac4c2b0d2119b2","src/util/idle_notified_set.rs":"9c6c9bff4a2c66c83bd9a70800a29772aff688286183a36aebb1da2303e65d0a","src/util/linked_list.rs":"824ae1872cc389bbf95ad746d46229051ddf90db320751e3531c0fbe4029a2e5","src/util/mod.rs":"3462a9f130045d2fda886efaaf21e664b1e8e31111f6cdfc1d4cd72d761bee05","src/util/pad.rs":"5dc99dbb3d3d16fecd6228fd2e2f67e5301b2d426e6149f79f93c1af1b4d1d90","src/util/rand.rs":"85d46431198a8c07d08c969e3556245f66ab7062552e3717d100c6d6769408f8","src/util/slab.rs":"7821a370afb5449c7347c54456f18087a7d00a1b59984a1ca2e2aa8250eafb2b","src/util/sync_wrapper.rs":"8f1ab76280171c33c0bafaec9c1cb1d48cfb0d19a1ab2b768cdf7d7e40f07d00","src/util/trace.rs":"f00f5e3c7e603ee7d700b40994976e8798440fc6bb4463bd2b89abddd436d048","src/util/try_lock.rs":"c4ee49e1751ee0a7df1a8cbd4f8d36ea1d7355e3ac584fdb8697a94cd7a7a8f8","src/util/vec_deque_cell.rs":"ff7d49cc4f660f55cb6ece97d1660bc5c36dee1eeb42643187ded07a33a0a6f0","src/util/wake.rs":"27046cb8116d4aef4b9296024d1a9b822121c0813b8390acca45878c9a13c85e","src/util/wake_list.rs":"c3443e695fd0b5c41c06d66cab96549064e9b645043559d01f82f365dcc4ff6a","tests/_require_full.rs":"d99eb68c2efd1e77f66689f805d89bb25f74e7aa589444ef7da71ba87dd1595a","tests/async_send_sync.rs":"de9c6aaab8a806f3a334489bde421c3dfc1d7186abd8545d0affee3161c8e5ab","tests/buffered.rs":"3ca857823d8073fecd4175dcb886558a3c6f3aec81803a90c5f786fc3edb4a96","tests/fs.rs":"b4902aaff2c28ef4d2676462381b04559fb4f7cdc0ecf46c46bccbb6276feb5d","tests/fs_copy.rs":"83448b19bdc332ec315024d4903b0a2ae81221895725a8b750025b47a43b0e79","tests/fs_dir.rs":"d9a837aacc072620c621e7e9799261b44873feea51cc6d25770bd3ff93abac00","tests/fs_file.rs":"d5ce8931976a6909eb656f90a3f9721a9788fa1e4965d7400bf196372925d1cc","tests/fs_link.rs":"01689c5c69aaa33543399164e2036ed96ae6e09b006c8dbe8af59df8c63df47e","tests/io_async_fd.rs":"ed56435564501e1cadd284f180dddfbd747df72e74c4b7b7a77bc166383bab04","tests/io_async_read.rs":"a590efe9bb01986f067118640a3b55f6760186e554f8e8f2404231d4e96f61b9","tests/io_buf_reader.rs":"f5a322dea6fe9f40c18a085a865919c1bbfe8653203b37d1e18b77d259c6211d","tests/io_buf_writer.rs":"4327940f81c0591e5c7b4a2583af1d42169c9427bcdeb88327d9a351f02d43fb","tests/io_chain.rs":"f5d3ddc9f6e8152ceb08b5dda2ca3168b174f1f67ff28a4c5983bcbad69d8af6","tests/io_copy.rs":"0683dee400710c1696a6634ecee64c39e7027344e66bfdd6b2a78de8ca913555","tests/io_copy_bidirectional.rs":"8ec51c37395f273de8ace88e7203ce78af2c4acf00b33fd968239a644d28a8cc","tests/io_driver.rs":"76db3259c69f66ac07e5b8bcdb1e2e46d42e06e202831d8ccbee835b2dfc9714","tests/io_driver_drop.rs":"31e7002a20ab65b6a91904b4784047bdf17b8cfa134edef04b394430a43573eb","tests/io_fill_buf.rs":"a520725811d266e8833dbfb62999d4a32ad8aefc6403e12a51f8b0fd62530436","tests/io_lines.rs":"f5b1599ffff44819e269519ff0a08635ea1c5d7c541293e63ee33d98f25f0e3b","tests/io_mem_stream.rs":"24be85e8d95a896f0398459d140df1de24acbb8e70132d32893bf7fc0c24f1ed","tests/io_poll_aio.rs":"165f80ebc81e8ccb4d335c9b9a89d960f097de9b17e92bc964effa3c76ce5f98","tests/io_read.rs":"528a2146495c53b271af460c0dcba28d54cc71821811002a7f8de213b4cbb385","tests/io_read_buf.rs":"3c1a7820fc5e486fe34c6fb61b3e1bf8d18587790a4897a732f325bdd438721d","tests/io_read_exact.rs":"b6387dbeb0baceb7a1f74a9a3a8b4a654894465368be27c3bbf4352b79fc4314","tests/io_read_line.rs":"8296624b4f5e162c79024f3beab2f561f4195a244cfd4c53e4d06282f56a31bf","tests/io_read_to_end.rs":"b5478431bf61dd66cedeb8e0ef588ed8ecd6d9e0feea3655512019b4abe9d451","tests/io_read_to_string.rs":"c9ebfee5cb262d822119c2881ea1cc0c73598b13c517c297663e35bb120a089d","tests/io_read_until.rs":"b6c0df9e4852766910ec68affcd92fbfbc280018b7f9c16cf5f4830f9b8389f0","tests/io_split.rs":"b1253c9f3e733e6bd1b6e7e4a1515606902992e15e57c8dd2ef28ccfcb7618d5","tests/io_take.rs":"f44ac14d94493f2346f5b0b2e23f4147ed51ff466165a5eb63bd525bee6719df","tests/io_util_empty.rs":"32dff601a78e46e12339bf1577463c7ce1070d71d78a2fb33318112a111dc120","tests/io_write.rs":"98668a8c8feae0f85714df1dfecfcd94fba4ba347bdc3d8aaa4ea8b175055c69","tests/io_write_all.rs":"e171af1ecab45a439b384c3bae7198959c3f5e2e998967dbd9296760b52951b7","tests/io_write_all_buf.rs":"2c037f07ac464eaa4e0b87e4e4968b28a0f2f1b1d1e218546c9d5dac7a75d145","tests/io_write_buf.rs":"331d3b54c7664386bb87585f39910d1fe31bfbdfa012a2dc2120e535dcdac329","tests/io_write_int.rs":"3f4b50345f7d7d558e71ac7f2a8c1c4b7b771dad09fe2e1fbf9a17d4fb93c001","tests/join_handle_panic.rs":"97169881a5fb93ba044b86d7d81ec6d358be61e7e34e03af174fccaf8e0aa572","tests/macros_join.rs":"a963ce60e82f247bf034e275db5bf25a406b28872df52243eed8763671edafd3","tests/macros_pin.rs":"37569f60c199103b167a4979b61e8cee2fd63e39273d4c94eaebfa281a6f0556","tests/macros_select.rs":"e9ec258ded92767961cdf42ec2f19feb98b2acc3e48023e758cd5a54ed8ea879","tests/macros_test.rs":"748b157b3be8c5bcf49ce28c7263dd8df31c59548951f8e0654e4cee8e3c36e0","tests/macros_try_join.rs":"faec183aded3ad042abdc23a08d7147461c9ec47c08269c6742bb1d094ea2e04","tests/named_pipe.rs":"fba6c0d454a78ff938056d1573f07ebcf1844de386f7331c6750681ce16feeca","tests/net_bind_resource.rs":"3abdf9457ebc9f8262c03fa5834f1ceb6312d4a1573b6bdd4e2f584e3cf76b66","tests/net_lookup_host.rs":"436d541bff6a6775244f9989694ef4b40457c44d847fd2514e8915516a711113","tests/no_rt.rs":"eb19939398940c56f5bf27f9da66e29894f9b2957bf724828767b52eaddc1c6d","tests/process_arg0.rs":"785d801cf281230e3208512820e7d71b4d8362b8cf33fc72235da370e10983a8","tests/process_issue_2174.rs":"66c56d8dfda4e1723928da713dddea8d15774d6730398effadf0ec28f4c6f1e1","tests/process_issue_42.rs":"26043f8246b00046137551f7a9f638652c70f527f10b4d91e4286643120ca41d","tests/process_kill_on_drop.rs":"56d908ccead7fd23d1f73976524cbd0b0f05806d342818095f2c6803469cc3ad","tests/process_raw_handle.rs":"54498de544b61e95d75bf6e92779fbd9b805a434ffcd1ea18496a1c5cd7c7522","tests/process_smoke.rs":"3554f4a8c8b76ec0a3b79f4ae0b5900edd34255dd3adc561e8e9a77f217b1956","tests/rt_basic.rs":"c43409b5c9300c3cf8b14521ede32355124637558143dc4f25ff02dfdb7de1e5","tests/rt_common.rs":"f09915824604a33c8e21098c84c648ec09cf4e26ccb3a8e43fda7f58df10f31b","tests/rt_handle_block_on.rs":"76b49a32abb7ed59c16977a119cb41ee7459221510eb919cdebdbde378db95f2","tests/rt_metrics.rs":"fa0b32715c2749380924469c125a97b9950a0678c524b3dcf241f6903939dca7","tests/rt_threaded.rs":"921499dd76186e70f1a88a546327398e1979c6237143f9f5090565f6289e0218","tests/signal_ctrl_c.rs":"9b53065781b37f3db5f7c67938239b0f3b0ebbc5938c14a5b730ad7ec07415d2","tests/signal_drop_recv.rs":"d1ec97213d9c6fd9fb25ea8c2b015c9e9ee1a62fe0853fc558bc8801e5a3a841","tests/signal_drop_rt.rs":"f968c1154262a4427b5aad2d9fb36d3b7d47084312d0b5527a8eb3d589381d8b","tests/signal_drop_signal.rs":"041940550863250f359630dc67ef133874d809ddaf0a6c1238cee1565a19efec","tests/signal_multi_rt.rs":"a1c50c25f4707fda7665da61b3317dd61fc32c63c61db2bbdb56065bd9c591ce","tests/signal_no_rt.rs":"99714bf488a26b6b394d93e61639c4b6807f9d756c8d5836f31111a30d42609b","tests/signal_notify_both.rs":"bf0b9def20f530d146ee865305833d8e9bee07a0515e66573d7ff30e2c631123","tests/signal_twice.rs":"bce33093eed151955d13c334d6d8a5bc5ca67cf5b37c246e435a24c15bc166a0","tests/signal_usr1.rs":"86ad07594b09d35e71011d1e12a1fa2c477bfbc4a2a36df1421b6594a0930074","tests/support/io_vec.rs":"9b3001e120138ead4a63720019c669ff00f8455a74dea2fb231633b3b58c9b09","tests/support/mpsc_stream.rs":"00d48122fa2ccbf1fe0b110ce3cf22590eda54b3ddec0134b1f9376eb1169645","tests/support/signal.rs":"83531afa2e8e71cfd90cd4e1fc821490ffa824f0f9f0c9c4a027c08fed6b8712","tests/sync_barrier.rs":"37401fe3de3010c2767ce0b1105c1ce991bd9c59ddf409403faa8a4dcdf30ffd","tests/sync_broadcast.rs":"0e13d330ce8932205998df5c7f4b0d613f88bd37b97d491083224d7a4a1d6fac","tests/sync_errors.rs":"438469440a1910457faadf9386757cae6e4f14807aaa513c930a8716ff68bc70","tests/sync_mpsc.rs":"989798056a9031d35cf01c6dec051b2cfe21b98b2e8f9edbf315853c2b057f66","tests/sync_mutex.rs":"e9ebbf1261204f0eb7ccf013e7455a1e11df8ebbef6105967e342559d4197a01","tests/sync_mutex_owned.rs":"911464f1aec2d5916fdde3746e9534705d1232f4161d4be56d2673d9d360b6cf","tests/sync_notify.rs":"e4689c9f735643b5993014df051db7057fd140b39975755a7434a64aa88f3ac0","tests/sync_once_cell.rs":"639d55248775f2cba0709919a3b2ff4317e70f081e09444d0a67791d65dfb3fa","tests/sync_oneshot.rs":"41e6966f3741ed848e71e973e65e630d549c6f9568f02feba737fbe3dea353bd","tests/sync_rwlock.rs":"1ddc830bfb961be4b05b1af549dacfad78ed2ae84eb2d9d12f8a201bee6cf929","tests/sync_semaphore.rs":"9796ca2c67c56676b4dc01055a55765f81f00e164b5af546ce2f0134af08cd4c","tests/sync_semaphore_owned.rs":"6d0a1067fdd6350ab1778e346fc105e9d17d9f40bba72700245f9184f0f6b5a0","tests/sync_watch.rs":"1d41976a05392901399f3d16c92544e835bc8f6c2ed2b5db2e42a8270e5fff83","tests/task_abort.rs":"a12920188752719d557a77c96586b1ef5917c424eb2b1237f90e017f01d51e25","tests/task_blocking.rs":"241574dfc0793008f80d64c3f6da64842fd1d46cc88964b59380e6796e0b116b","tests/task_builder.rs":"9f0212ae4d7071f2e491effca992ed8884b2ef2e3493a6d7ee3fa659cc926338","tests/task_join_set.rs":"e362f7d17ce4373db59b01e0c590e8ed231e4099220a4838039a9bc1d4a7d612","tests/task_local.rs":"eb6fbfc6ba024885863dd5d633611f21bffdd2f96a7fbb596598dfc0c151bf9d","tests/task_local_set.rs":"dfeb90355de7ffd59aa567bbae5539f859b55d64255e6c71785118364c9da8d3","tests/tcp_accept.rs":"a17b374e8bf590633d5c3ba434ca0a8545ad96d8c01e140bb8418be968bd8cc1","tests/tcp_connect.rs":"3e457719e2ecc877090e039465d492706c9d0318f483c70fc325866ef84108f5","tests/tcp_echo.rs":"68d67368f1d9f9dffbaa9fb1a0ef71e4f3700a76aa5a2c3714c8cf7aedb0f1bc","tests/tcp_into_split.rs":"9636970f8450139462d7b88a4994ffb6c4d77c92f89750ad9a479ee70b0ed5ce","tests/tcp_into_std.rs":"9fab18181788d46e787e87fcdbbb2d1a24c1a60bbe8bcb1bb1e63cf43326c643","tests/tcp_peek.rs":"ea904d05f9684e6108a698bdbbd856c9849c1a51eb334cf5bd45ef74c8fe585c","tests/tcp_shutdown.rs":"68f3a5b9d166064f63819a53bd07bd1c0988c1c84832ac4a35ac9b1ad6f447f4","tests/tcp_socket.rs":"bbc3f467f6d2c13e975d28198d9e186f6d4a46dd893049084845ee5b4d468095","tests/tcp_split.rs":"e967b01bb90e2081e5e08e8cfd619cbaf0d1dcd08c4e77dbd5bae893698cae85","tests/tcp_stream.rs":"0b597cfb53d00e2af278b64d74b7de6a1436b36b3a9a4966de3976c52d6f37e1","tests/test_clock.rs":"d5c9bf7bb5d926e2b29b43e47b5bb051b0f761bfe44d5fef349ed442ea7b416f","tests/time_interval.rs":"cd1e459be096c33f6d2b25030326dcdbff3eca5f83ac514d236ea95ad5d12283","tests/time_pause.rs":"aadbea51adba2f437bffb3a01f444e634f629651b1bdd89134354e5ae4782ba8","tests/time_rt.rs":"2981f4a4c276b306c4d4bf5684bafbaffb0789ce7bf8d2db3556e5208bab52df","tests/time_sleep.rs":"61556bc4882dfa111bc8a2eab566046d547c2f886f58be8b16d0d9523d9c5f7f","tests/time_timeout.rs":"ba53f80e8d3be2cd8c72388ca6b2e9d444504861febcd077cae5d7a02dc59425","tests/udp.rs":"ad6dfeda7a4721b7bd00692a05c9d668013c7f5e86714e6ed1005b99eacabaab","tests/uds_cred.rs":"146c6e9bdbb82a268f178b03575f94a40db81ef67a740dd16ead5b9e4a447f1b","tests/uds_datagram.rs":"06964ba9c88e49296da2e27dc8ae0d2bc85a55875a31f3875cfcf11f55ae6851","tests/uds_split.rs":"79d54d6ce35e5d15138299091871ecbdb6492ae6863fe406021fd7359f1ed7fd","tests/uds_stream.rs":"d672ed046b6e6ad9e2fd118be91c788680972e3f5271e91857f29c2e25d1b9d8","tests/unwindsafe.rs":"dab3dd646625a878ce979cb54d1a68672444d105af3337b06b5dbb169283521b"},"package":"2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"} \ No newline at end of file +{"files":{"CHANGELOG.md":"bc77dcdf89fcc44f69865ff2f76d259059c53a86748a3718f1d79448f58dad10","Cargo.toml":"4b9acdc35c6a29194af890d53794224c79b6cf4389749c8356d6958f45873686","LICENSE":"1a594f153f129c2de7b15f3262394bdca3dcc2da40058e3ea435c8473eb1f3a0","README.md":"c797f1898c291ce3af69f04ce51832b341341d6b46aabf77a9d4c26db4fe5f69","build.rs":"48941a3d0e097f960e6616bd5dd22d9959f03e415e48a6dbc9c271804411d58d","docs/reactor-refactor.md":"24d4f3ec6d8828bb96afe0084df9e7739bbdf1995283dbd2cd76740311485b75","external-types.toml":"d808de4bbff94d5ea56f1b1783497b0c6cd107534853aca8c79ee8404d8c2399","src/blocking.rs":"8e62b2cdc512fedbca4b4c4f983629af035afea4ee7e918bb1a3e9851c8e034e","src/doc/mod.rs":"f54f0d17974e68bd93035137a36b216dad05402cd4f3f64ea7e0008e7c309858","src/doc/os.rs":"727a5e3dfd3b57609f6054c32965d6825949d5e7a5c751f14b50dac3b7ca065b","src/fs/canonicalize.rs":"93c64b72abdca17877d6ab61d50a43765d6aef9e0a9f7aaf41b6b0b7d9a8a380","src/fs/copy.rs":"262180fadc66e5ac2bf1e8389628fdd039f14788e66a4f8b10e10633e7310f20","src/fs/create_dir.rs":"233cbab2579a787614aeee88845a57f1578e9c2e064a3456c799f61430e911ad","src/fs/create_dir_all.rs":"56081d541caadca0fc59e84d55e78e702fe9373679598016224ad0b072b189a7","src/fs/dir_builder.rs":"b5b21229c7bf15e2074afab1accfbc392f3c69e546c7b652cfdc8e90d5a4a301","src/fs/file.rs":"a18108174484261dfff5816f9be63517377e5c83c9d996887df04c01ac05f652","src/fs/file/tests.rs":"014b0bd871a58d42d8b97cf2d860f83693294f935c808f1406f4b26e5d09afa2","src/fs/hard_link.rs":"98cccbbb3719baee11c232e79723ab1cb3d6c8056bddb109c4990fe2c236c1fb","src/fs/metadata.rs":"782a1a5dbc2cd6c40e928579fbfcf39e5f1de28def78781590c0280acdf02960","src/fs/mocks.rs":"d3718127d1d0362253bc7c7ca4d6abd21548fe69296291d1ae12a97ccf7fc196","src/fs/mod.rs":"8743addab538366cbe90b43cb233c54e1be524c173b9e659fda4fe9fd78060b2","src/fs/open_options.rs":"cd8acd3deaa339f38cbe14b1537ca4b07b9830105d3b385d965585330c824304","src/fs/open_options/mock_open_options.rs":"60ebc4bea76d742ead8fc5b5f4079450eb2450ecfed5f83d7d89cfe148f4ca54","src/fs/read.rs":"055ae8b6ae96ebae2d05f8780e7592bb587a742506c6df8ee8b380fc7d2820ef","src/fs/read_dir.rs":"3d67be6f69299cffe1b6403a0cb182f76bc576ebd6ae8722a2e0ef47a8f912ad","src/fs/read_link.rs":"93c104a21253372fef7056ab82e6065b0a1a8fc9be8b7329dfd5a8dd07c618b0","src/fs/read_to_string.rs":"9e5b2d476a6084e32a92c5421a8abc9d4f335f4ec677beec4bf8bfa109d7d106","src/fs/remove_dir.rs":"96475771e9c52678768288d8df6814216e0801bebc848481597ad34e829a5854","src/fs/remove_dir_all.rs":"b85abd05c7ab64ee8dc6cf5663a11e713aa51b357759ef660ef3cae3365ccc42","src/fs/remove_file.rs":"1cdf8bf16b3a164c594dac8773b7d1f9ebb28de169343184d34d6aac3b3a7eaa","src/fs/rename.rs":"a97875e92626fa46e23fece7b8698c9c4cea2bae8f1be8726f30ae6fe80ae0c7","src/fs/set_permissions.rs":"8adccafa475dcfc1bc3989af73374d90683c1be4953ef812e5fd606f968d7b7a","src/fs/symlink.rs":"32cf3e906531d30ebe6d8be7ee3bfe049949759b566015b56d0851f51abcff50","src/fs/symlink_dir.rs":"66a6655f5306854a7b6ed3748598bbe737055da9635bded715a04c3abfacda7c","src/fs/symlink_file.rs":"ec5816344f8c0b79c4d84e0ef2a987d753c719afd3bbc0d0a739224477a9edd2","src/fs/symlink_metadata.rs":"f5ce1e05f137da995e3e0d9582bae0a5f7ef4251285c64e912b0eedbb068b395","src/fs/try_exists.rs":"c666196690c4d45991374b03b27f0838b33e98d0be2838638d4de4ff856894b7","src/fs/write.rs":"1ffb734d31748bd879ad398b0fe99bdec569782b42677022957db2cae95c4d2d","src/future/block_on.rs":"30bad79b005a1ba7f696bec5639a5886b1f653a152b8948ae5fcd287c05ab8db","src/future/maybe_done.rs":"9042619f2a907e81763ac6082c080faa28af5d571dd49e82b60f3f14d58598f3","src/future/mod.rs":"6f28857347c50cd6467f1ca7d1b1028b633de7d142c3f3ca710a8693d1e839f9","src/future/poll_fn.rs":"b3c0eaeb442991d3fe27f53f2c2849f5f40b0f974035036c26661bcdaffa09df","src/future/trace.rs":"c42712a8d372922eba7e05cd21382fe5df5eec02cbcc870062100b59ab99654f","src/future/try_join.rs":"0ea5a069b17a34bbc091acbd74b9d51794a55a85dfa63fe2404d5ee91a4f0038","src/fuzz.rs":"db998767c729f35320b1d489f43f4d7788e989c8d16c0793188a2868207d8e25","src/io/async_buf_read.rs":"b37caa8f6c974b3c97327c635218803e573c531d4197950840549aa794357c99","src/io/async_fd.rs":"3d08b8c1d6e2435331245f7ace0b79f0cd47c72ad9da1959f17af3c8297415a4","src/io/async_read.rs":"f52c8d2f4a283c0dc8d06dc974484749973125b0b691bc0c3d100972ac67cb92","src/io/async_seek.rs":"a9a0df389ff2be3d79208ec475fcfede46a86f6ea0b822b1a4ce8273ec714b0b","src/io/async_write.rs":"198ed6a475b6f093fd2ff15e618d816d7e33cb8bc28d2e2299533e5df0bd78d6","src/io/blocking.rs":"b94dcac1e0ca315b356e7a71e5d71696e79363b92480603e699051e390b62873","src/io/bsd/poll_aio.rs":"45e818a201c8dce93f612c7856ba4259724d2c9fcf341001eb9833f821f16f3f","src/io/interest.rs":"2bfb67ee66f3a802922b6ad34a60d635f2409df2d8d23e7c06af06dfa5866f3c","src/io/mod.rs":"b0ba3446cce66a7c51654e22e7386ce6242e4109f994c9af4cdae7890ff30b8e","src/io/poll_evented.rs":"67655d44018479bcc17626f01331739c1d2edabbd097da30c992cde6b54a9061","src/io/read_buf.rs":"14f82f701d5056d1d5581c70aec98f7c6d287899df45e4b2e82388e7c97cca57","src/io/ready.rs":"465fe520f7c876f675ed078e22aca482f4dd6dcfc50ebf40567ae29ec9fb0047","src/io/seek.rs":"e9e346fc926c3360601b80a8319a25fd0567dd6f77fab666694e9787deaef633","src/io/split.rs":"a8cde1b7b2dacccf31647183db475d34e15b000978f7d0207f34ced2506167c8","src/io/stderr.rs":"76cfb4eea6ba478ab158fa269c11ec56ac5ba91d2813417cf0b7a838a45599af","src/io/stdin.rs":"54b989a7a057fb784950fff882b331f7860326cf5adb1b66748df98bc93ab08e","src/io/stdio_common.rs":"f8c72bddf74e48af264676b36470636c16e227b46740c46d44f6b21b2a44bef8","src/io/stdout.rs":"ba7468285659fce3105fd3e02fc8f2ec63a83e01e3081520b6912534614756e9","src/io/util/async_buf_read_ext.rs":"010e722194c93d56dccc8794db9f73954ff58e87364599474c4d97398f4bdc99","src/io/util/async_read_ext.rs":"f70825f0be7ebecce5473c53c716cc57f1123e5818f7b917444c3892639a5e2c","src/io/util/async_seek_ext.rs":"0535ec6139be7ea68becafc0d374aaf9d996ce9cbe5dca0b8e70b133a661c1ea","src/io/util/async_write_ext.rs":"b8ef64cd0b8b57036297cc90f66aeab6d97ba20f33dda0cab9dd364b4509c123","src/io/util/buf_reader.rs":"670a58f404e5689daf1f2b3070b0b9e95fef96ad19f0e8487f294e8a2afe558d","src/io/util/buf_stream.rs":"2246fe71b707c15d7168c5da5ee158cec6e854d4fd11b685531c16a9c3cf2c6a","src/io/util/buf_writer.rs":"f9c3e018c9f9177fb6d910096503caee727bebd3c36f5f67dca2c4c55044408a","src/io/util/chain.rs":"5cd8df2cc7bbcd18ca2336a78507fa8009c0a9e595f81730a8c16cadb8f731a2","src/io/util/copy.rs":"f4c7d59736f9a6d8a1efdabf9685dce2ccc3053c5f9578d3b3aab4be58dd5c8b","src/io/util/copy_bidirectional.rs":"89a5fb3f77e9863c9496c02afc2613fce039f47a1b2322e6221dccda8323f80f","src/io/util/copy_buf.rs":"9d83771a6edcc6d0d32f066072e375634e21a13517968ec9b21a4edddbcc2605","src/io/util/empty.rs":"32cbcfa8bc033e7ae1cd62505a84125750d1abccd0c37b2a433c5eb80db0f0f6","src/io/util/fill_buf.rs":"223725d828071e923f25d2d49a0f6e470c411a6d9ba225700f2dd8d5793601bb","src/io/util/flush.rs":"fe3b4ff226e294843b8cbea9dc4e02d581582b78ddaafce137c96e290699c718","src/io/util/lines.rs":"1d9f9b99567111c911e72a4caa2abb19b277f2cdd0ca3268ac5ca6df5276259f","src/io/util/mem.rs":"5089e03d72b2873ab3f56cd86944e6bd29439c427219cb090610b5b4bf032fca","src/io/util/mod.rs":"6a9012d78fe2bed8240e7a628e9421cbef45433551522624065fdcbb329f3594","src/io/util/read.rs":"58988c3fbcf5ede633dc224d7df5a372495c4485757dec9bdbd825138bb7f5d4","src/io/util/read_buf.rs":"a87be2d115c09a6782ec8cadeafc92fb1fbe534580e71540087c3298a03bfca2","src/io/util/read_exact.rs":"4a8650fd7a885963a0fef2bec24c17046c5624e4dd7fe229ab3f33c4a92fc66c","src/io/util/read_int.rs":"49da230796335df584832cd7deb8370b4d1e0350d743046389a9d9ae17dbd94f","src/io/util/read_line.rs":"9cdb2d778b81bc50098a6851981ed9f541bd0c7896c0763b811971b5a598b7e8","src/io/util/read_to_end.rs":"1c061665818a4ca9632403aa95955bc435f4438c6d6317370ca728fb47acdbc5","src/io/util/read_to_string.rs":"fafb5463b013cc8f76def3a505dbebd179afc95bde0e2ca9388e428265788924","src/io/util/read_until.rs":"d9a932dfb5ef3d0d5e8faa72a2b3b9d1765c85599f3bc77741f69e7fe9c0d037","src/io/util/repeat.rs":"d4effcd81338831eb373cf2db972a99218b8379b91066940a732edcf4524c7c2","src/io/util/shutdown.rs":"971454342b4636fbd68e123d59d87017d81f72afb410c385846069b11def8efe","src/io/util/sink.rs":"0dcb794e48ca9b1c28e5f9f2051073ea0951a54c9c7dfc903ce9e5489d3d8cd7","src/io/util/split.rs":"03a59adccda29608886e38a1f484fbd4d6a6019180c4cfa851372d250796aa5a","src/io/util/take.rs":"ef080ce27d23cc23b9f99fe14ddd56349a3cb1442aba18f8405c30d20017b074","src/io/util/vec_with_initialized.rs":"f8673705967021b5a3cb819d672df89ec81eb5baabb48de7bb598d53352b62f8","src/io/util/write.rs":"20d14ee545ab1f67732915522e97808d1ddde13d151505c1289b596be519f7c8","src/io/util/write_all.rs":"906ff3fb24c6a979b104598f9a8229421bcaf2a4218c28069504b34a218241f6","src/io/util/write_all_buf.rs":"5911bf673ef89097938f4e2e38d9012865b28a0ce5ebb217ebe0e2507de6c1e3","src/io/util/write_buf.rs":"ab51d6174de24cbb729ce77dbaeea27e16059b8253e4830d8243ec5f08a08a8c","src/io/util/write_int.rs":"f321e69b0c7c01728b079e9fdeedb96c26475667e8b259d0c5f4a83d060990d1","src/io/util/write_vectored.rs":"7a335a9f796daa048fa9708dc44d32d2567b36461a6d88f07893eb31f304b69d","src/lib.rs":"65ca594d01af3324cfbd84dbc7e8714ee5d47d9ac1da2f41cc5159feea92b2c6","src/loom/mocked.rs":"f0322f61b997ed86bd5f729a55cd862d56ec7602af711772094fb4005c7268f8","src/loom/mod.rs":"b14b9333a7a21bd125a4ae82f01e5ea9c9ed2f78d7d1ad49a13d9b176f1fe8ab","src/loom/std/atomic_u16.rs":"5aba33a798e8a95129e3fd039e4412322e25880003ba782031df9faac3dc93e1","src/loom/std/atomic_u32.rs":"7fbaa448621371cbb9b8dd882701236b41bed599d69eeeb0a1e25b2a9a4645ad","src/loom/std/atomic_u64.rs":"2d98d1608058f3248d79b1e9c96118627fef6362ccfefa77047be2e5617d7571","src/loom/std/atomic_u64_as_mutex.rs":"1a6b9f8417e968a78dcddd8c7c320c447b9f27d5f84245cae86c3ef68425c749","src/loom/std/atomic_u64_native.rs":"559e6fd21e678b904f5a905f2756a4a8e32ca68a8c7a9b4344af86979373cfa0","src/loom/std/atomic_u64_static_const_new.rs":"a1c0e7d2ea28904ae76227baa7da5da33b68257f4c1a7a456f8d3171529bd934","src/loom/std/atomic_u64_static_once_cell.rs":"92a25654dd8232f6c02c8ec58d4f706030db442fb2a5ba07f1aec09149559725","src/loom/std/atomic_usize.rs":"698c0a65ea4d1ad05d49770cef2614cee643794be4b6208ee071a3e381201349","src/loom/std/barrier.rs":"1539e5773ad0ab1936e8a9a6cf5c20bc697915abde15b3e0898615194eb37fb0","src/loom/std/mod.rs":"7f15210848cba1e8f382508acf948da79499a14b69e0d8694312db8642b97d56","src/loom/std/mutex.rs":"0899bdc1302f1406aedf96ba8b1a890abf743188b442d242b1667d998952d17f","src/loom/std/parking_lot.rs":"e29739d7894c61a005ba29b0ea88dcf34ba24d9ea8fcf5f19b9e009d81095952","src/loom/std/unsafe_cell.rs":"05e2b4d8e78b0435650c6da6239ce8c53b4de2573e64ba098354b049f23429ec","src/macros/addr_of.rs":"cbd020a07ffba2b1c608856081499645cf606cb45444dc53d94457381a94bc33","src/macros/cfg.rs":"44dbd194ce13ae0afd3b77966021fc10b8740bce2b6b676c4768e52677f3da77","src/macros/join.rs":"076dbeed1cbbb292e39012d18c0d1c97848bce48c4609b23f327025989b34a8e","src/macros/loom.rs":"bee8a86b0b96697cc21e4b5e9f1a3403c1f9dbc8c4e591a2ea91f51c2469d3d0","src/macros/mod.rs":"913b1382779ceedbd13606e429f59b9c7ad929324f121452f47773c232c3303f","src/macros/pin.rs":"294e5644061e41801dcee5494b7334439e09af0b6219ce164090feb624864631","src/macros/ready.rs":"6efd4c866c4718c3a9a7b5564b435e2d13e9c1ae91fd98b1313d5e7c182942d6","src/macros/select.rs":"4d9588dd514cf9f25b8c3c76ab08ad141d8f7ed1acdd856c6418946edd08f055","src/macros/support.rs":"da976befe4ba4d29e96f0de24cafe6e1f7f97364274e240556a01f2af77314de","src/macros/thread_local.rs":"5e759658d18035af5aaf907237d399c05d967f94c71c7bb33f36f78ca768cf78","src/macros/trace.rs":"33befd4533a3b2b4b22e246033f2bea8930174a7da58adaa57dbf20931275bcd","src/macros/try_join.rs":"2233cb76a6ce35d020ebbb8a0b0d713f01c0b6fd20fd878442bcca688c97ed6a","src/net/addr.rs":"0ed3d72ef6679e3ad76c6830143713c611d357ca4ece88c3ee5ceb2787217224","src/net/lookup_host.rs":"c7a21d735225e316253d822f6b11a17107e6a8db004f947a54d8bc12ec782baf","src/net/mod.rs":"301307d5a5f8c783418646ef9f7712414226a73061a691b468e0e3ebb7eb4df9","src/net/tcp/listener.rs":"76dc043e1272d040a8a338ad23f3e27e27d7a47030a708bff538e0b21c1f8d59","src/net/tcp/mod.rs":"347182e4f0381896bf3b7ab578c4564f1730ae25a108076ec65b8e855683fbf6","src/net/tcp/socket.rs":"d8bc43dcd5f16baf8ed5d8b96addc6a8140d49c684fa99b6fa75d0ca5c402c4d","src/net/tcp/split.rs":"75d1f6c4afa23397164a30c4409a73c8d3df022c780aa4f91fabfa90549cc606","src/net/tcp/split_owned.rs":"0c5a9f48e7e49c257c25e138f74c0c5d3402edf447a860cbfc1a36439d997295","src/net/tcp/stream.rs":"2397ef910c6d04af277de6064ab70c20aa0abb7c131c3850bf9bbd354e33c066","src/net/udp.rs":"c2a5319f3e02eb515d1980083effba2545c25820e4bb6c5fba5fa328a3cd4ec7","src/net/unix/datagram/mod.rs":"fc48924e5d1e551405b0708a2d122473cdafeeee802a5722f64b4cf41a1c01da","src/net/unix/datagram/socket.rs":"5b736136422911db1f5aac2e541432850ec9f9f57bdc784f61f818868aaf75e2","src/net/unix/listener.rs":"e1c242e96856cf878e9cb5514f97cd8f5a65b284093a0cf47f94bf349f66d707","src/net/unix/mod.rs":"72008ca5352432de2203321244da5377afe5779ac15f5f6a577998c037e7e4ae","src/net/unix/pipe.rs":"0881edb516df3443f190d5dc84d6b40b31b517775cca1bf0393f2e473e15ea8f","src/net/unix/socketaddr.rs":"66bf18321a81baeb394a7094567632b113e44e12d2643109b6a97c89d919bf3a","src/net/unix/split.rs":"bd0388de8b6035b8cce2a34989ff56304697ed6864615ae13474f4e9c504faa0","src/net/unix/split_owned.rs":"5eccba5c1d589af4e5b52d17410c2290c289db5583676cf016de93ccabdace9e","src/net/unix/stream.rs":"bbb7e287385ac47817fd266393bf83bc4b28d83a689aef4e280bf5c3c321dac6","src/net/unix/ucred.rs":"567e59bdf47e6f607d3699a7880dc4ca2d5625559069cf3e8cb0d35f162a3718","src/net/windows/mod.rs":"a1525f35c1acb92b15bec788a625b76acb42f9424f392074db697102ce79760d","src/net/windows/named_pipe.rs":"d607bdada51daada04c863d39d16e2e66f63bc141cdb017e2457573aa34cc048","src/process/kill.rs":"2f98bd1bd28ab37bedc34ab7b737760407ab5315420538acbd18da31d2662d94","src/process/mod.rs":"60e77366338e19c2491e20ffd313322251ac859a31a9413cd11d34f72858525e","src/process/unix/mod.rs":"09112d79de5d25fc1d1a2c32435199ba480ce186d8516d5d1fc6bdb09b80f439","src/process/unix/orphan.rs":"d641d9dcf41b0ce26a3ccc38e2e8d3cd505c060a87a137fa761132e2290bd886","src/process/unix/reap.rs":"62868319849b8482d8d927dcd00cc8a74b9af61fd47494b39bd41fe2f4dcf0b6","src/process/windows.rs":"bf98b7c716c03c07b3341f124ceab93de45fd595af8fd061b1b0f3f4782e8d50","src/runtime/blocking/mod.rs":"3a1e04d2fc5590c7c0a19c85ecbd893108f9a81b197162378c525133c3bbc7aa","src/runtime/blocking/pool.rs":"b50454e68d2e60e5360c4aa5fe1f8289d21fbfa2f6e77b176931f9d370301af3","src/runtime/blocking/schedule.rs":"206f8a9972dfd2c1c18d29c8b8129a7d7165f5df898946015f6d4dc72c61a777","src/runtime/blocking/shutdown.rs":"6eaf2a1418b885bcc1ce3f257b99382de9c16f848ed53d9e00dc97389bb6b816","src/runtime/blocking/task.rs":"08571bce8a99dd6296ba7099c1f6da8c832d13a8c43db22044c683470147d7d4","src/runtime/builder.rs":"9b4af3c9a5bdcee3c0cd8e96d28588463712930d1ba22047a5738674e9fedf34","src/runtime/config.rs":"ffc5bac854fc94e429a53bbe73c494c39f1c6b99adbb55088571394a111df80f","src/runtime/context.rs":"b2c974ae63995a4b7f81f5f2fcad429487659fd7df462492f52f5fcef76afca6","src/runtime/context/blocking.rs":"82ec6b5c6b0ce7b18582cc931e9cdd6d5946366cd676c9e6bb0235e6e8471ed3","src/runtime/context/current.rs":"51023fb843227d82a4c8b7a093148a2b36306f398e2770026d5d1e0f296e08dd","src/runtime/context/runtime.rs":"df7b8deca59426ac39d6950b518172ff7a7ef306c1681187a4d3b1f22168984d","src/runtime/context/runtime_mt.rs":"911b95f39dbed005a02fde0394bd74c02e14e5b9f5c98bf49511f6c68ac5dac8","src/runtime/context/scoped.rs":"16bfa1da16baa831cc926c41df869b99bfbc6d96cf39a04aa4914a9e4b19a2f0","src/runtime/coop.rs":"911565ce644a15b3815772dbd8d996bb42175fa0497aa3b459868f556f3dbbb9","src/runtime/driver.rs":"00c9e7f1a33119612ab122cbd0ec12d4d4264fbbf03d69ea40ad44a530a305d0","src/runtime/dump.rs":"de53e25c28c362cc8bd4f321d23a7c5c8a3956ed3a2ffa4acd4d62f6397ab9fd","src/runtime/handle.rs":"629bac5f9c98a4b4016c4c183ab8a825cfcb8973102657ae7e7361a2fd1f5b52","src/runtime/io/metrics.rs":"e711c65da155ef73e567125c0c9db33b2f511982a0c0a6b4f3b37c2d87c791a6","src/runtime/io/mod.rs":"b412b1c499dc535809ba89dad92b2d0eac570620536803dc62c0f40569e52c51","src/runtime/io/registration.rs":"3b9ab9ca192e90d041008d7ff40be77fdae14d5029a047d8a949b9de05c49fbe","src/runtime/io/scheduled_io.rs":"39f12635cb1ea028bd82040982ecfc38d31daa106c9dce0a39d81b326c84e163","src/runtime/metrics/batch.rs":"2f4dbac94c8da8feb5b78c5b4b40c23d2c014567c571877b4d4540d2ce37fe05","src/runtime/metrics/histogram.rs":"1d85cc2e6e1ef02b2cc4f2c627d5405d603742b995dd3ff23909a3a2fbedc8bb","src/runtime/metrics/io.rs":"960f44573cd16e2ec43ed872f235ece09db9f4c87e217002ae0d180e20ce9838","src/runtime/metrics/mock.rs":"b91e1531419af9f8f99b0b796db4dc1760b2836f1a55f6cc2c80f690b611bfcb","src/runtime/metrics/mod.rs":"6c690fffe34bac1267eb3dc0ed93481874795d2fd2fe9f495859fab0866fb814","src/runtime/metrics/runtime.rs":"f34f9ba43e5c1959fef485f98afda44f4a0fcdd5c6ed282b7505de61928fbbfc","src/runtime/metrics/scheduler.rs":"997ba0339af84ef878881fdb6f35d8f6b39dc81e8f0b5cafce94476d50675c91","src/runtime/metrics/worker.rs":"e13ee455c2643f71b5a8c61a07023afc7341c1320fafcdd32c74dd553d81f3ad","src/runtime/mod.rs":"d7c76867cc1ddd517564a624f3ea1df64b29f72e553a7bdc2514f8670faea769","src/runtime/park.rs":"dda2acc53529d8169bb2c4fc08ecb8896bd675515b89eea6110d73fb5bc93b10","src/runtime/process.rs":"f40854b46e5851d7f5ee10f50d839aac933343c2178508f419c8e41dc60557db","src/runtime/runtime.rs":"3247dcc560c8471dbf5a79062ee9a58e08d20b0aaa8de27ebb64f2c9ac5752c9","src/runtime/scheduler/current_thread.rs":"b3bd715635b19320d3faaaccae233a00dfa822d29ca2893a1e5a9230619614e4","src/runtime/scheduler/defer.rs":"6124d19b90b56db1aa0688ac0e4af964a6882a51814b0065fc80d3cb6398e98c","src/runtime/scheduler/inject.rs":"fea0cf9dba0e72efaea4671a1693fa5bc68aef376d9773407855551951810426","src/runtime/scheduler/inject/metrics.rs":"26ee4f9d34c9d670eb75ee8d02a6e268952a1ef698ab3213d71873a97ea2c851","src/runtime/scheduler/inject/pop.rs":"335b42278217a3020ec1419d070132f8f5a0ce832c504dc7806c42821848be83","src/runtime/scheduler/inject/rt_multi_thread.rs":"de370e710a9c9010ac1c2b29ddc3d4ca7de9d538bd40a4329f5d64be5924d7d3","src/runtime/scheduler/inject/shared.rs":"7d5c67a41156e559a5c91d39f89ab1d3f5e217bad079e46f0065535ae186ac53","src/runtime/scheduler/inject/synced.rs":"99e5508570ba2cf8facfbfe5a54276876c5ff71d53522c0976c4a79c1b9f117b","src/runtime/scheduler/lock.rs":"2aa5a50067dcc79d66ca39d1e4982c28f55bdcec43ddd7bc91414de5f794019c","src/runtime/scheduler/mod.rs":"9dece4afaaf3412d55206f1f1d8b4558dee7645d0108486eb07397cc4b31c996","src/runtime/scheduler/multi_thread/counters.rs":"e5a6a1a1d01a50890dabbeb7998c7b64a954ed94bbb7cf9505ff917ed63633f6","src/runtime/scheduler/multi_thread/handle.rs":"c17a2644ee53990376ecd4276ace8fe90eb2cd446b9ac8115f1062af1c7662d4","src/runtime/scheduler/multi_thread/handle/metrics.rs":"d5858d459700380a372821c2a5f03255f96e58e2fdfe8367e6cf355fb29c7b72","src/runtime/scheduler/multi_thread/handle/taskdump.rs":"20768dab2f9683ecb10849fd3a4a2862a0f9942fba97d6ac29c8bf5ba7017c62","src/runtime/scheduler/multi_thread/idle.rs":"c2be2afa82ddbbe36eb6e231c8a01cd7c233d9c6e197d2feae32200098e974e9","src/runtime/scheduler/multi_thread/mod.rs":"2fae4c9f66b57d15f63c485ee6ebe724f1d41de126e641418ed6ec40c8fb3e0a","src/runtime/scheduler/multi_thread/overflow.rs":"c70ff49356fa6948f9dc53a19014a5dbf7f6bf52d4c00d9c2e20b0d23b7b3bf0","src/runtime/scheduler/multi_thread/park.rs":"86aa7fb6f52b16237e9465fe572ff61c7aa13e5fe9654ce52dab435e23df9781","src/runtime/scheduler/multi_thread/queue.rs":"1b7ace7cdf3ae20b80980aeab84ca4493d6ecd0247c808340a8b62fc3f7ee77f","src/runtime/scheduler/multi_thread/stats.rs":"3686e6b7fd6ba31c36f78d5e5ef58b81e19f6920c764184f9e4c5497125f8460","src/runtime/scheduler/multi_thread/trace.rs":"f8ac6b0db76c45057c0061908d5acd63eb1463c397984bfd9a80f4bbc2748f30","src/runtime/scheduler/multi_thread/trace_mock.rs":"3ce6a3d19b53b608cba34fda213e498691ed55ce4a542806f9080bd2dbdb054a","src/runtime/scheduler/multi_thread/worker.rs":"9c7e9f33f4a043f39ad3075105faddf86111e5233e6c09c4ce5f72a9c88e0814","src/runtime/scheduler/multi_thread/worker/metrics.rs":"14e15c7315e1e1a6829b67beda6a858570b70eb9ecf9f0f3d7530881ff7799ea","src/runtime/scheduler/multi_thread/worker/taskdump.rs":"ea2bd932ef0a3b999fd8a16e15f9e7dfe8c30f3589569b7f7ab334a63ad3bede","src/runtime/scheduler/multi_thread/worker/taskdump_mock.rs":"f8664a755109ddc87f7ab82c4815cb12eb37c0bee3a1808907ac10534b9867c4","src/runtime/signal/mod.rs":"1498f18f55e730270f8c92906857d856de5ae264d80a96699c8b8b2f9439b77f","src/runtime/task/abort.rs":"81004b45834ab44f1b673b31379d37c68c5ce8acb7d223ea58207f412745d3d5","src/runtime/task/core.rs":"77134284a7da7157c1f4bfc6b2d5d4569c3a7079cf24f15416a3185b0bcf0bc9","src/runtime/task/error.rs":"0367095439c51ce94f53b13aa64404e13e85b114f77cdcd7315d4cb3d4fb9344","src/runtime/task/harness.rs":"996330140acc964d5bf31398b994283d997a9dc5390a7c6983b0a38723d3f1f6","src/runtime/task/id.rs":"4cade5f2c02e5d3a0c516625aa8def2da110db3b83d13a6d9e26f2a1ad8b31c5","src/runtime/task/join.rs":"db92ae97f715e36bc77fb6d8d668479472f5454406aaf4c1191623e59a893989","src/runtime/task/list.rs":"ff04f5303e59913e651f36c6be1189cbb2955de6f1acc94815608a23a3579612","src/runtime/task/mod.rs":"b096464db86360785446c67e3a9da8d35169283f3bbd57f1d65c0ce872a5c7b5","src/runtime/task/raw.rs":"4c30fe4cb3c57361b0f28a4acd25877b1f6e6507076b5933ad91f098ab2d7f71","src/runtime/task/state.rs":"2a75a1af1e2e6bfbbb6e33c8421c0841d0680f1fbbf42d2b555d9d33ef9f0c8b","src/runtime/task/trace/mod.rs":"e0175011ea04df0ae64060f36d4a91b3222cdab4ec4aaa8220e3bfb6542d81fc","src/runtime/task/trace/symbol.rs":"bde671f445b351d388e5fcb80e8cb8be4424a4fac4b8ec3aee356eb86082b422","src/runtime/task/trace/tree.rs":"b8f9aa018147d539841e5716394086709fec6a8d0f95f6b2fc011ed131063dd6","src/runtime/task/waker.rs":"784f7f251df481b7b7390561a6601e4b8e7fc26a2fc4207873519462185ad5cb","src/runtime/tests/inject.rs":"97b68f6c22f6861b2a1649a8f919bae3310d6fddeae107aee4dcbf00de526cdc","src/runtime/tests/loom_blocking.rs":"fb4bf871ae59cd567ae4a2cb568d7cba31ad92579bea17c598d273a3a749f74a","src/runtime/tests/loom_current_thread_scheduler.rs":"9ec324867d9d123bf93c8f1c9feba8f9b57689122a1efa784adff425962b2d65","src/runtime/tests/loom_join_set.rs":"d6612a19c695907a8bb41b24df3b404cb521cae6f30226f2449dc32f18700b69","src/runtime/tests/loom_local.rs":"69cf7fa67da9e3efe3eee722791f811efb395dcf821780051e62842654de09e3","src/runtime/tests/loom_oneshot.rs":"cb0cb66e1014708a10a2236c2dbbac9441b6a6e36a49c419fa3a51af62b054ce","src/runtime/tests/loom_pool.rs":"7904461bb6affb1bb46f9ddbc11d1cd6a57e4d3c980a34f0ee6beb62c1ca496c","src/runtime/tests/loom_queue.rs":"5cbed693390bae17cc9c1b2e11c4cf183d88a746ab35802a85f4aebbd81befc7","src/runtime/tests/loom_shutdown_join.rs":"2f48626eb0a4135d480aa5bf72e514c4abf39aa17711c96d3829f93c03758557","src/runtime/tests/loom_yield.rs":"fac610e464660c196a35ce15a46c9ebf537d3f3475cfc633e58c76c25eb558fe","src/runtime/tests/mod.rs":"625205f9c1f42b3ca4329593cee7d266497620a611c5f7f65fbfddf7d8185f82","src/runtime/tests/queue.rs":"56ea2b3c8f3a1a01091c3e624259b03a57ab555e52ee77931ad9edaf5d6db329","src/runtime/tests/task.rs":"e98f94f976cee07f9722f82f11e9d9486e978310a5314367fcb9580564f613c8","src/runtime/tests/task_combinations.rs":"0ebf6682728fc5ede9374b6353a8fe5d0cb1d27e511f338e779607d8fcaebcf6","src/runtime/thread_id.rs":"ad85f1057b2c61838620f1323fa8a710e140bf448fb2aa1d9d1f87a7fad5f01d","src/runtime/time/entry.rs":"7f07396d64c008f1d3b9a7d496564a07bb976837c3e1f0b21ba02ddfb6306515","src/runtime/time/handle.rs":"1ecbebdc070477d61b782b2f4d3a5b9b5cc06dbe447188f23eef3257fd16ea8b","src/runtime/time/mod.rs":"28f78f2bfce3b7cd61a893b19b93a285d55f084299401816f3d9fec03d87de70","src/runtime/time/source.rs":"c03f9eb4d229a582826164031541f028e26b95c75fd02fabfde3668cc81b5e8a","src/runtime/time/tests/mod.rs":"5be4c9b935df11411670c4bb59e071a513f4488dfed78f48ef20371cecee1d07","src/runtime/time/wheel/level.rs":"974a230b26f6d4fc9f4046741f1e8ef90acd2d53132f135515bc37914afc276e","src/runtime/time/wheel/mod.rs":"1104133c6d50ea708c5733f7bf05b2ad1f027608a7495d18e649c9b97f31204c","src/signal/ctrl_c.rs":"3e1d98df851b9ea1418f607c260434e906ada4e3d6c14cdfa5c81a0c4d55cdd3","src/signal/mod.rs":"4da598cb028d5716628bcfc06c0fb0bb8fb8f0f0986c6a328a655e76979cd85d","src/signal/registry.rs":"c9e9d1df123c6069c2ad80824162a5de8c0163a9c63714d26ba2128538e34cfa","src/signal/reusable_box.rs":"3d3b710b1794d9f8f5463e9ca380ece60c426b58786a5cb5f40add627da01aeb","src/signal/unix.rs":"0504cf52a28e9e8e528fab0ccd4e9c3dc86fdcfb807ecc9506fb6adca186c531","src/signal/windows.rs":"826a2a16fc9ff37ecaf7eef88f09d8b5f57adb6095dc10b981a4c0bb67b70a2d","src/signal/windows/stub.rs":"a6d3b11aa3a7247f7a8365c3241857bcde48d8c390d569e6bc9529c9b1f8ecd1","src/signal/windows/sys.rs":"3e4882762ac424cb3b3caf71d5a69f571fbe14e6b298889ccb227c58b213c257","src/sync/barrier.rs":"45a8ca0fec02e6a6fa77cfeeb09e3a142f931d413c00962aa0d0036ff8ff37bb","src/sync/batch_semaphore.rs":"ddf48ea17d5f911a05c86ef45388cab848f710068d4ebd9139342f8ad83a7ac0","src/sync/broadcast.rs":"e2c2f003fe871b51dd3b827e9a1e685365163fa733bc62da7da2ad07b8fc9289","src/sync/mod.rs":"bd94e0b941f6c61623bf9ed6317c545c5a076d3431471f014f0448984c2bb82c","src/sync/mpsc/block.rs":"70a034a62ea88a95e970352a7a3e004c2755c75afa34d71f3653537389e6e2d6","src/sync/mpsc/bounded.rs":"fe08d3d153053f54c51c16f0cd67b74141c14b7d4e852cf2577a156e92b4707b","src/sync/mpsc/chan.rs":"1f5b7c5dfda94156b366cfb2b25262c85a0c77e31d415099653afecddcb59139","src/sync/mpsc/error.rs":"67db03385fa385cb459a830dae7e05c818c995d1f48e3bb8308f76f9f37bd7c7","src/sync/mpsc/list.rs":"2c99cec4d3d179f6e668b8de37e58525477be3f069a9a748f7f8ed2f8a40f817","src/sync/mpsc/mod.rs":"3699ff2b4c5315f0a1cde40a1d3a45bc312febe1cb612cd68c6176ed7590bf1f","src/sync/mpsc/unbounded.rs":"5cd0694dbd4a2f33d44756a382c02780e1951dae19d246fe1af3e026ad6da5ca","src/sync/mutex.rs":"c8dbfa89fbb2484fb4d15b814a52db5f4b00f54daf464194001aa733ec657a63","src/sync/notify.rs":"bde3d5cfe7d417687bc79bd971c07b97e916919f133c89d1e63fce5a6be83d0d","src/sync/once_cell.rs":"56e0722bd81143351e780646404d55da907bca866dc72bc3a0fbfcbd8b176754","src/sync/oneshot.rs":"2900d5d4ac81c3056c04b38ba7252eb4e98437943cba1bd6f06eafe78873cf93","src/sync/rwlock.rs":"a89b7a8ffed53c9f91d07a9dff966544567fd90da9219e9099d81393970edee0","src/sync/rwlock/owned_read_guard.rs":"135d81271f0028263ce99c3246cd335a08912b50a43f125b3e8f7739ca353cfb","src/sync/rwlock/owned_write_guard.rs":"ef56443fac210bc3d0c2b66beda8436154c2ab726f1dc208758b54b2b087762d","src/sync/rwlock/owned_write_guard_mapped.rs":"3038187dc3a791a89505621230d3edd8629287ec11490ff37f2c624b593c5325","src/sync/rwlock/read_guard.rs":"0f7e52adacad0fb1529a453eb14f8207a8caabd8f5fca4aeb2031a9e9b0981f1","src/sync/rwlock/write_guard.rs":"f3855cdd09940c86a9bd10b91d3ebb37b2219b60c727a3bed377404ef2483c35","src/sync/rwlock/write_guard_mapped.rs":"8c839c4ac4b73faa1c1fc697753e23fce113ba6e576d2abc022ddadb98a34af4","src/sync/semaphore.rs":"801c98bdcf3003964c3441702cec952b89e914475653f1d6a3901ad7e8c91941","src/sync/task/atomic_waker.rs":"41e9e05522254afbacec9895390d6f95498e413a61d8f654f9c421c808e7f83f","src/sync/task/mod.rs":"f5e38105c7f8a942c0e49b973bad0a8c2a1df81deea19f3c5228edc4896c1725","src/sync/tests/atomic_waker.rs":"34b1bfa8fa3b392b89bad36fc4a2a17e42d84c60635c6e7727b428cfb693feae","src/sync/tests/loom_atomic_waker.rs":"984b52699c47383b9b62e6c4ff93fd458bbe64cb4ca836463adbaf94d27c38ee","src/sync/tests/loom_broadcast.rs":"b2c6f138707fc389ee7d91109bc38093af9060b3465e68d3543cb652e0070406","src/sync/tests/loom_list.rs":"f0ce15a0f965fe558a21bca24863c712156eaeb10feb8ef91031a6d6e3cc5dba","src/sync/tests/loom_mpsc.rs":"4883352b9d75a81c878609613545ae14910eca4f2e7f3718053dfdb792aa0760","src/sync/tests/loom_notify.rs":"cd401c73084df551043b7d96b9f2c51e9c1929a9038eb899381bd4ecafe04ec8","src/sync/tests/loom_oneshot.rs":"c3596c15692b16e7cb8cd6957362adb3a98b3d7f16c4a4262f19a3a27f262b03","src/sync/tests/loom_rwlock.rs":"80ec00bdcac838806d4d9f711cb154e02f22913ba68711855c39ca92028d3e4e","src/sync/tests/loom_semaphore_batch.rs":"c6f69b8d5b2e6842287ed34638a9045095d9f94c86ba6bb84c1224bbe10026ff","src/sync/tests/loom_watch.rs":"d451c914e94c5a672abec3232e58aff6218581f0c0b1802434ddbe771b1be6a1","src/sync/tests/mod.rs":"1ef2026ac3dfbb70a437b8be441deca4b7b5e72638b71daf753b89808cd3c9ee","src/sync/tests/notify.rs":"cdbdc080eb9e653fbd56501a9897b9dcc6298ef81f7a88390976f9fb5deb034f","src/sync/tests/semaphore_batch.rs":"291d6c615058aa28a26ef17260cf5f27be286b20da9dba36ec95ca3a7d895ea9","src/sync/watch.rs":"abe32dffffc6105f806238b64fedcc40807a48fb19f35db2120286cad1568de4","src/task/blocking.rs":"d2ece044d7cbefd4a07aad1a5824231d4ccd35f9ef7197deaa80f0e5de5396d6","src/task/builder.rs":"6704158d8adb0e1a1a2b1b66548fb295c6c1e32b7387e6f3c9c6ebde9a0c084c","src/task/consume_budget.rs":"f1000b9628938a5964f70793506f51579849284a4acb10d5e9c70080c5caea47","src/task/join_set.rs":"e178485ba52bdeeaf0f9320e150637770db8313cbbfd442440dd61a98fb5c402","src/task/local.rs":"5753e6182a647547e3a80e3fcdc5a34433dce809ea9c4fe5d99a313054e30e3a","src/task/mod.rs":"68c4e77221655cc3cd208e20fd9f8b3035d5f6176e30a90c5ec67423f8ff8ec4","src/task/spawn.rs":"1b4546ba873255b9ba28b755e2349200cb299c253ed6539c9fe0045ce48b7f0a","src/task/task_local.rs":"a0597c6ae9c304709f8c770c184d40d0d3d609b674c8881e32fa890fe14dc734","src/task/unconstrained.rs":"8e75239ae694f296136fbacadb232ae0a255b0b2013e1eb0390bfbb82af36692","src/task/yield_now.rs":"ce1f9417f650142488a8c27856e995676c5e74b8965832966bedff6a99b8165c","src/time/clock.rs":"839139149402a4ab18f3144a36249f57d36536ca44a646de0bd6eba374a71ef9","src/time/error.rs":"de761055f6596528fa158eb2179aa662fb1510d93aaa2102f8de3192c6a6d1ae","src/time/instant.rs":"164ee00dabfa6bb9fe4c9854f937ee92ae454eeaaeaa7a447e09434ea95bf722","src/time/interval.rs":"3d9d49b644546f95ca3deb2ac7ed920368e597a84da564e0590b59ade2e857f0","src/time/mod.rs":"c790bbe9356ceb56fd20e762c9002906aa4eee1d1d297fa0c520130f72d3be3a","src/time/sleep.rs":"74fbbc406caa686ba100b4e399b6bccc8659ff2f20f1414bb40cf473a415c4a7","src/time/timeout.rs":"67f5396cad468dac3381b323734739914052a0f72c3df018eeab14a855f6c8d8","src/util/atomic_cell.rs":"6e156b730f69c698cece358ff5b206e8dd88083230fd49b2fc4a66a539b92553","src/util/bit.rs":"ad3117c9a8b1e312114a72e9b315c37137af61da51ea458832d46d2c1b88879b","src/util/error.rs":"de1a0d662e9359da3e19739125b1450f571abadf30e34f50f085b23d8da28d80","src/util/idle_notified_set.rs":"7c54b9cd9490a085e224047825bc6f79827ce32d39e16547ab624bd113a7f3d7","src/util/linked_list.rs":"688cbd8e8176460f9a342b0d2b3da039260f853495dec8db12e4bd6eb104761a","src/util/markers.rs":"fb9b674d9a48c97b411db8a3264fa2781026a0eafa74cf276cd47d8ad62cbf02","src/util/memchr.rs":"8cb5e0a0da9b7d94d29a994dd4e725e547ce31f87b5bf24a1a9b53b0ae65cc08","src/util/mod.rs":"2a3c39fbf7643e34344d5ab3a769a4b31c88eecb356c07297eeee163f4310ca7","src/util/once_cell.rs":"bafbbff82e595a0b8f41f3d5273dcfcacd282c51a24f14666105391a456c5db7","src/util/rand.rs":"c36ca3d9e971e857b132f29ce3ce0ca3fb7c82a4f413354e7acc91cb6e8ee285","src/util/rand/rt.rs":"9790a2b6d822edbffde723f38b75ded408dba6954d711f285bd17924322b246b","src/util/rand/rt_unstable.rs":"bdb69915fe12a9e0b8f2ab4d72466663b7a3e87e671343a0a623028690f4e452","src/util/rc_cell.rs":"362a7a7b776389688299628a4678378fa02f88fbc6ed7f9fe006054d9b62643a","src/util/slab.rs":"fd21a2927f658c3fa7bad377e964702c4a85b5d995f44b90740101c19d820b97","src/util/sync_wrapper.rs":"8f1ab76280171c33c0bafaec9c1cb1d48cfb0d19a1ab2b768cdf7d7e40f07d00","src/util/trace.rs":"3b4e7ed40a16f9806a40f56268ec5daa519565c899216ab08a64c2f152f85c84","src/util/try_lock.rs":"c4ee49e1751ee0a7df1a8cbd4f8d36ea1d7355e3ac584fdb8697a94cd7a7a8f8","src/util/wake.rs":"27046cb8116d4aef4b9296024d1a9b822121c0813b8390acca45878c9a13c85e","src/util/wake_list.rs":"c3443e695fd0b5c41c06d66cab96549064e9b645043559d01f82f365dcc4ff6a","tests/_require_full.rs":"869a93f914df3d1c941b5787ab52dc55440d8d9d78dcaa0354b8d2a1d30119c9","tests/async_send_sync.rs":"bbd4c385d982840e5afe4cc3d8de580d23afefc278ccb9e7195ea57ea89a31ea","tests/buffered.rs":"b953415c67c406677816332e66d46ed677fdd504119d9a1ac4b5df15acd168db","tests/dump.rs":"4eb9887bd06cf82dca679d6e27a37770f8a8bcf44c8f5b06cd3cbad835848788","tests/fs.rs":"da61358e4dfaa8bdf499fd453f68b26276ba50a721e79fd1e472b9eea889f58c","tests/fs_canonicalize_dir.rs":"c39e85391dc738db31ef5c7607956a97dc32cabdf09e5dc0bd4edd3a6e354aa3","tests/fs_copy.rs":"b28a67ea4722ed346eb4c035bb03c2785ed94c796191c1eafb29975bf1f66781","tests/fs_dir.rs":"e519be77200d54984f6088dfd1ab0e61c8284be54a4fd8c41060e1743533b4f4","tests/fs_file.rs":"61df8af26212e38ed0de9a8377a3b44a51871c52f9232ab2e130a12a70c7ee4c","tests/fs_link.rs":"ab0f68b65101b9e1d26216179d11bb637d968a81322e96721136881ec4796287","tests/fs_open_options.rs":"039a58348ca9d94dae8bc42e0e85f4c79f25f0ef07102e904564273ce2cd8ece","tests/fs_open_options_windows.rs":"1561afac0671988ee0bf4bb1a34dcc485e53fdeb11e75a87a6166a8f49a6a392","tests/fs_remove_dir_all.rs":"2b5294d43fa608a412d2d6a3c2d843fa1164524e680d86c6c216d62066da54e5","tests/fs_remove_file.rs":"da32874c72fa6b031176ffd4e8102faa19dbb9c92b3a43c1348fd9bb206ab801","tests/fs_rename.rs":"ec78709cae9e6d5afd7422b06303a192946845e8f5f78465eac8b3b7ddb8480e","tests/fs_symlink_dir_windows.rs":"5622869c393f27bd20ae44e7e38748a6d466844228c2c25038b9d1171c85b72b","tests/fs_symlink_file_windows.rs":"bee0d4b180d9edf7ff98a688e68b05e012ce78eff7d7af82c27c2ee63a5ebecf","tests/fs_try_exists.rs":"8ed69037a876312bc3d0a0ffe3e970349a3827369db5fe60d8123315f99fefae","tests/io_async_fd.rs":"ee05f0acc0b93f3f78abaf5221af2c031f5bd7e34878e780642ca637591f03ef","tests/io_async_read.rs":"a590efe9bb01986f067118640a3b55f6760186e554f8e8f2404231d4e96f61b9","tests/io_buf_reader.rs":"f5a322dea6fe9f40c18a085a865919c1bbfe8653203b37d1e18b77d259c6211d","tests/io_buf_writer.rs":"3bdabe9ac26f3189929ab3e8957150f5262d5b426fd3cb6c4761a45014b1c1fa","tests/io_chain.rs":"f5d3ddc9f6e8152ceb08b5dda2ca3168b174f1f67ff28a4c5983bcbad69d8af6","tests/io_copy.rs":"0683dee400710c1696a6634ecee64c39e7027344e66bfdd6b2a78de8ca913555","tests/io_copy_bidirectional.rs":"855f720881c624562897c7e8f9cc15ef11498ddd11dbd803f21fd587c84eb2bd","tests/io_driver.rs":"01b89cded4f482dfcaf5cf1715a493c932aa0c72eb0563950b4b5fcdadf86ce2","tests/io_driver_drop.rs":"d25d432740f5eef3f214fff6d30646ffa6a3dba77658c76c1fd2172f32a0169a","tests/io_fill_buf.rs":"82a83d0d4d5a60bc312f2d77722607a8bd28d465fdeec3fd072815cd0cb4ee70","tests/io_lines.rs":"f5b1599ffff44819e269519ff0a08635ea1c5d7c541293e63ee33d98f25f0e3b","tests/io_mem_stream.rs":"7b20d86c02c8a868cfa3aa4d228519090d156fdd9a8a19a3be263f264fc9d33c","tests/io_panic.rs":"48f3f842f55afefa04824cee79383639c9d9465c8a6641c9eb8850f1c47d3bc7","tests/io_poll_aio.rs":"165f80ebc81e8ccb4d335c9b9a89d960f097de9b17e92bc964effa3c76ce5f98","tests/io_read.rs":"4ce62e123e687f38efb09a14246be86018c4772b10d492f9e512c99bbb5f0818","tests/io_read_buf.rs":"6dc454751af0e9cccb44fdfb12b9f0311fa8afa482886aa56e3e15d49ae1e06c","tests/io_read_exact.rs":"b6387dbeb0baceb7a1f74a9a3a8b4a654894465368be27c3bbf4352b79fc4314","tests/io_read_line.rs":"8296624b4f5e162c79024f3beab2f561f4195a244cfd4c53e4d06282f56a31bf","tests/io_read_to_end.rs":"722e2922a40080d391a83b4903f47823779740090624a98ac30e9f164b20a3bb","tests/io_read_to_string.rs":"c9ebfee5cb262d822119c2881ea1cc0c73598b13c517c297663e35bb120a089d","tests/io_read_until.rs":"b6c0df9e4852766910ec68affcd92fbfbc280018b7f9c16cf5f4830f9b8389f0","tests/io_split.rs":"a3228f84142732d624e062bc933ac26267a10f71202123b3a9d5eebaba82eb96","tests/io_take.rs":"8790584a344102da70e669d07b9aa39b94d5d76fd87ded8a684971b06d5e7367","tests/io_util_empty.rs":"32dff601a78e46e12339bf1577463c7ce1070d71d78a2fb33318112a111dc120","tests/io_write.rs":"98668a8c8feae0f85714df1dfecfcd94fba4ba347bdc3d8aaa4ea8b175055c69","tests/io_write_all.rs":"e171af1ecab45a439b384c3bae7198959c3f5e2e998967dbd9296760b52951b7","tests/io_write_all_buf.rs":"2c037f07ac464eaa4e0b87e4e4968b28a0f2f1b1d1e218546c9d5dac7a75d145","tests/io_write_buf.rs":"331d3b54c7664386bb87585f39910d1fe31bfbdfa012a2dc2120e535dcdac329","tests/io_write_int.rs":"3f4b50345f7d7d558e71ac7f2a8c1c4b7b771dad09fe2e1fbf9a17d4fb93c001","tests/join_handle_panic.rs":"5f81436d1e7c18847392c744c25c14518c41f24558861584f5ff759b147ce4eb","tests/macros_join.rs":"d91b865047afe490c2db84112c2fce1c953e1bf26f80c82d9c8dd5b73769c693","tests/macros_pin.rs":"6162681c8b31c35ea6d88d09187b77e1fcc96ca56a1304b5cb39b4051f601f3d","tests/macros_rename_test.rs":"c478f11266527a166bd8f7ca3446db94c7e7a1bcfa0e74b6597b6087e5749436","tests/macros_select.rs":"20044ec9cb0d92c04bb08e0636ecad749b06dccc9e8ccd09905f88fe7dcfa63b","tests/macros_test.rs":"dd28c1f90676230285ed2aad7e760e43d7df38d18ebba3fc8c7c7957d309fd72","tests/macros_try_join.rs":"08d723892038e36c39291b112a4eb16e4c1d021522cdee569c1209e02f1f4b44","tests/net_bind_resource.rs":"3fb8060945e26c6b38d28115a984ed17ec89e8313568e88014d3bb3dbad60bb9","tests/net_lookup_host.rs":"a1ba4c6100584f3ec706810c9a4313875e8cd0bca13531bc7759d8ecdf0189fa","tests/net_named_pipe.rs":"dfb791caa5a4ed1bae0605da27ef599abd8cbe745c2fa95a31853bd46465b042","tests/net_panic.rs":"e7382fd76c8e1f9744e9f2e442ef259cc9cb801429998ec3b00126cd8739911e","tests/net_unix_pipe.rs":"6b6956dd7d584fa18309fc8c86a74f5270c9fc2faed2f5c3b722c11e93419abe","tests/no_rt.rs":"228964501f3a7b0eb3ff12adb29dbf6296006d05d04fd2f418fb1f1f015dfe54","tests/process_arg0.rs":"785d801cf281230e3208512820e7d71b4d8362b8cf33fc72235da370e10983a8","tests/process_issue_2174.rs":"66c56d8dfda4e1723928da713dddea8d15774d6730398effadf0ec28f4c6f1e1","tests/process_issue_42.rs":"26043f8246b00046137551f7a9f638652c70f527f10b4d91e4286643120ca41d","tests/process_kill_on_drop.rs":"9b4bf8a73769fb268ef3687141e8df6122814855d3bbc4e63295c66b0ee6cff7","tests/process_raw_handle.rs":"aaf0dc242ce69f1a8c614e04515c41584f7e213f84ebf23238521eb881e0a6de","tests/process_smoke.rs":"61cfdea779ea749df9a30d136543874e0f1b979b85931728c75a44b70c55658e","tests/rt_basic.rs":"6e40762f7b772f4aede9197b9e82261282ec3fc6e0b55bc473a535104f99669f","tests/rt_common.rs":"e9b582fc530284192e6edd79409b5e1b03caa4f057e5203332cb74a3b01f9cd6","tests/rt_handle.rs":"3df11420510cd2a7218e17a78f320a746e65d3f2bfd10292ecb1460fb82a49b1","tests/rt_handle_block_on.rs":"2afa6806468f070c5b38d47bf51dd51d25e75d3f9098cdc804a7c2ca533199fa","tests/rt_metrics.rs":"e2388929c1d80464372dcfb7e39ee9c23109f7eb920b9e87a5d244426f516d18","tests/rt_panic.rs":"d7f54f60d0305b344b2c30b940662cb75f9f000c2ecfed58c6e8b427e936e787","tests/rt_threaded.rs":"e10c56d73f6af20e70e340ca2132844d1772c8d6abfebf03b5ca611e57a03c80","tests/rt_time_start_paused.rs":"76549719f16e9cbdd9a84b9d97929ed7c8ca95b8d4d715608eb99f950cdda7a3","tests/signal_ctrl_c.rs":"9b53065781b37f3db5f7c67938239b0f3b0ebbc5938c14a5b730ad7ec07415d2","tests/signal_drop_recv.rs":"d1ec97213d9c6fd9fb25ea8c2b015c9e9ee1a62fe0853fc558bc8801e5a3a841","tests/signal_drop_rt.rs":"f968c1154262a4427b5aad2d9fb36d3b7d47084312d0b5527a8eb3d589381d8b","tests/signal_drop_signal.rs":"041940550863250f359630dc67ef133874d809ddaf0a6c1238cee1565a19efec","tests/signal_multi_rt.rs":"a1c50c25f4707fda7665da61b3317dd61fc32c63c61db2bbdb56065bd9c591ce","tests/signal_no_rt.rs":"55b9d96355e1c4177c047ea01b78a5514c34e1eb75e67d8d99b65605c2df206c","tests/signal_notify_both.rs":"bf0b9def20f530d146ee865305833d8e9bee07a0515e66573d7ff30e2c631123","tests/signal_panic.rs":"0b26845bbc7f388346e0fe3af98a2257dbfd0eb84168368994a0857e06a42398","tests/signal_twice.rs":"bce33093eed151955d13c334d6d8a5bc5ca67cf5b37c246e435a24c15bc166a0","tests/signal_usr1.rs":"86ad07594b09d35e71011d1e12a1fa2c477bfbc4a2a36df1421b6594a0930074","tests/support/io_vec.rs":"9b3001e120138ead4a63720019c669ff00f8455a74dea2fb231633b3b58c9b09","tests/support/leaked_buffers.rs":"85ddbce3ff6b0c5cc8cb30449dd37e5bfedc94f8357e741ec88ff04462bc60c3","tests/support/mpsc_stream.rs":"00d48122fa2ccbf1fe0b110ce3cf22590eda54b3ddec0134b1f9376eb1169645","tests/support/panic.rs":"7bcaf8ea2d249b5c1acb3c1b769cbe479834630dc71b3ce1114b8b46277f44da","tests/support/signal.rs":"83531afa2e8e71cfd90cd4e1fc821490ffa824f0f9f0c9c4a027c08fed6b8712","tests/sync_barrier.rs":"e6d44e5c2da6e055598513781bdee6a42261e3f7ff538c925e07a52866662bd0","tests/sync_broadcast.rs":"eae02f0ad50141c36590a01113be82de382592273a55584a545ffd1eeb431533","tests/sync_errors.rs":"40c3ff2413dae2efc7abca7e84eed5023f2919108a5ab96bee1bc46b98b42449","tests/sync_mpsc.rs":"754ae8ad77f38fae2478ac016239c4c9b6e64e68e4764ab721f6a513dcf17482","tests/sync_mpsc_weak.rs":"868efe2c82c8c3232c1686bad41101486bc74a552bc7dd0b32e1633f272526d0","tests/sync_mutex.rs":"4355a738f2868f09ba0b376a82dbf0e8ab9a83225bd21e80e16ea7c34392fae8","tests/sync_mutex_owned.rs":"38ee9abc16c71a9c9eee5b74a17bcda588e93cb17a60dfa9a4d7fa6e93c8700a","tests/sync_notify.rs":"f2b6077a47bba8fe285970fc3e11b46fb9be1024bd3ba5b152b9c199c2da905f","tests/sync_once_cell.rs":"dc68e3643ab6a95f37e12dad9e8eca757dd3f2448e74e5463686c9a7e8b981bd","tests/sync_oneshot.rs":"93cdd67891fe501c2b86c1f7f1814b6f0b3f16b2877778af2e36cac141ec99ce","tests/sync_panic.rs":"5d0b4515bde6bae58131ea3f7c792b1faec7284924161918acf4ceb768bf4917","tests/sync_rwlock.rs":"f06683b9a95f3f9f12be06afefabc7cb0de790d9d79b4a1d7f3f7967aef1bade","tests/sync_semaphore.rs":"ac9102c51d2bfedb7afabe5a751ea58895ba35816639ec977f82a26f3d9ae56d","tests/sync_semaphore_owned.rs":"98547cb4b4e33409b7cf30e9db0c30e06377e77463dd31d03d3d3fe35f9a662f","tests/sync_watch.rs":"6718eedd56bc3107b1c7bc01ea059197800a6cebe1a61e74bafb0c450d137e7a","tests/task_abort.rs":"dc10ad424d4be12a86722ca181c3e2d93cdc41ffb7f27c1615f1c93ac1b11e94","tests/task_blocking.rs":"307bf3c4764936712b456c271f15b9c527001fdebee58822e74744c8f3e951dd","tests/task_builder.rs":"784830b868e9fd7b0d9413cbd7719eea7606c62dffaaf59986ed3e7902e9a3e5","tests/task_id.rs":"118bde53222009e4277fe4747ffde18f5acb3db17db3036b4d793200fa8f2a18","tests/task_join_set.rs":"f5a5708d4ef3bbf291b1de41ff54c30f37d7479b6087089a6c3a463e841828e1","tests/task_local.rs":"60cf924729be40a7f281def7911f6f6969c349af3a8a10c22fa4c94dd60b7e86","tests/task_local_set.rs":"5ee1c2fb426b69ad682567d91110c6ba43f04ebc6665a03ba4fb84a21c6c7a70","tests/task_panic.rs":"f4867fdde7918312ee493d263ae809ae3903c8c73a7e0ca6b8f9951ad55ef23f","tests/task_yield_now.rs":"dbd0f342129b2d06edc0daee360731e5adcb2e3a19f8d658a2cb510490d7fa25","tests/tcp_accept.rs":"5d91dbca6d78e176d62a316bd40818efa37e73e6253d1c9c31d5620ddc3b33df","tests/tcp_connect.rs":"a592530fd995ae5d119ffbf16c488875c944518c2c7b8c6ba7356ccd9d9963b5","tests/tcp_echo.rs":"5263afeb2499f4ed80e05bb19a68c9c407b798686715d53595c10beac6229193","tests/tcp_into_split.rs":"78df4144ed605dc65d20737281e6de10c9d4b750cd51ca8d4b03ef4d5aaef7e7","tests/tcp_into_std.rs":"3dd28d2a7ccfaa1eb74e321b08c05e1ce5d015d3f05f22a95f949b3acc5280ba","tests/tcp_peek.rs":"fa7868cee7ab85e5fd657c8b8fa1d947f63cce0e328d40c36ce6e5e450a13bc8","tests/tcp_shutdown.rs":"ddcae943626b44b74739f66f409092b05d303154041580c1bab319e02ecbb051","tests/tcp_socket.rs":"0c42454cf4246557daaef5d56e396aa6066cd4cd91ebb1cbc84be740f371bc0a","tests/tcp_split.rs":"691d578e4ee527674a6b800246dd48b12f1145a478b5e07f12f91c4a00c3ad2e","tests/tcp_stream.rs":"417cda921374da15359d7097daa24e8d32df2417efab2cce58f0705979a95a29","tests/test_clock.rs":"d5c9bf7bb5d926e2b29b43e47b5bb051b0f761bfe44d5fef349ed442ea7b416f","tests/time_interval.rs":"5e36011d97bdd27aad7c9c3a3da59190197f290543f0bb91cde331ffa67047b5","tests/time_panic.rs":"86430db173ddedd858bc77abd8d2a1394fdc74dde01beca7ecaa95f1f4274af9","tests/time_pause.rs":"457913a8b3d801291a516db1a6a31ea83e54cc89628b08fa7e49f7ffc5df7836","tests/time_rt.rs":"1687c1635695791fdcc4b536fa7b69410c4e82675b693f486ad48909f8f55b67","tests/time_sleep.rs":"2c57177a7fdf108e9a88f6ba3c07230bdae6df745856d01a5da09a217d2032d9","tests/time_timeout.rs":"1c1b44da8e8483f5df2e85892f3e88b075674a956ce91ea7a637772b8e6ac8e7","tests/udp.rs":"af5c73e42205aa9c5ecfc32a0c9923b8d92bb9735b5b100f6654150569a0a835","tests/uds_cred.rs":"146c6e9bdbb82a268f178b03575f94a40db81ef67a740dd16ead5b9e4a447f1b","tests/uds_datagram.rs":"f8d886adc6c3947960991392cd8540a0b4a7f7b0110a24d10805237b385b49f0","tests/uds_split.rs":"79d54d6ce35e5d15138299091871ecbdb6492ae6863fe406021fd7359f1ed7fd","tests/uds_stream.rs":"287b7d5e297df3e326488d98f9645ebfcf8be0205ba6a8e5ddaadac58240e113","tests/unwindsafe.rs":"86b71da9e70b53df8be1b80188b38cea4693f96f2a3006c0143c03555a5ff12a"},"package":"532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"} \ No newline at end of file diff --git a/third_party/rust/tokio/CHANGELOG.md b/third_party/rust/tokio/CHANGELOG.md index 3f69f09364f2..aea22c6d9850 100644 --- a/third_party/rust/tokio/CHANGELOG.md +++ b/third_party/rust/tokio/CHANGELOG.md @@ -1,3 +1,901 @@ +# 1.29.1 (June 29, 2023) + +### Fixed + +- rt: fix nesting two `block_in_place` with a `block_on` between ([#5837]) + +[#5837]: https://github.com/tokio-rs/tokio/pull/5837 + +# 1.29.0 (June 27, 2023) + +Technically a breaking change, the `Send` implementation is removed from +`runtime::EnterGuard`. This change fixes a bug and should not impact most users. + +### Breaking + +- rt: `EnterGuard` should not be `Send` ([#5766]) + +### Fixed + +- fs: reduce blocking ops in `fs::read_dir` ([#5653]) +- rt: fix possible starvation ([#5686], [#5712]) +- rt: fix stacked borrows issue in `JoinSet` ([#5693]) +- rt: panic if `EnterGuard` dropped incorrect order ([#5772]) +- time: do not overflow to signal value ([#5710]) +- fs: wait for in-flight ops before cloning `File` ([#5803]) + +### Changed + +- rt: reduce time to poll tasks scheduled from outside the runtime ([#5705], [#5720]) + +### Added + +- net: add uds doc alias for unix sockets ([#5659]) +- rt: add metric for number of tasks ([#5628]) +- sync: implement more traits for channel errors ([#5666]) +- net: add nodelay methods on TcpSocket ([#5672]) +- sync: add `broadcast::Receiver::blocking_recv` ([#5690]) +- process: add `raw_arg` method to `Command` ([#5704]) +- io: support PRIORITY epoll events ([#5566]) +- task: add `JoinSet::poll_join_next` ([#5721]) +- net: add support for Redox OS ([#5790]) + + +### Unstable + +- rt: add the ability to dump task backtraces ([#5608], [#5676], [#5708], [#5717]) +- rt: instrument task poll times with a histogram ([#5685]) + +[#5766]: https://github.com/tokio-rs/tokio/pull/5766 +[#5653]: https://github.com/tokio-rs/tokio/pull/5653 +[#5686]: https://github.com/tokio-rs/tokio/pull/5686 +[#5712]: https://github.com/tokio-rs/tokio/pull/5712 +[#5693]: https://github.com/tokio-rs/tokio/pull/5693 +[#5772]: https://github.com/tokio-rs/tokio/pull/5772 +[#5710]: https://github.com/tokio-rs/tokio/pull/5710 +[#5803]: https://github.com/tokio-rs/tokio/pull/5803 +[#5705]: https://github.com/tokio-rs/tokio/pull/5705 +[#5720]: https://github.com/tokio-rs/tokio/pull/5720 +[#5659]: https://github.com/tokio-rs/tokio/pull/5659 +[#5628]: https://github.com/tokio-rs/tokio/pull/5628 +[#5666]: https://github.com/tokio-rs/tokio/pull/5666 +[#5672]: https://github.com/tokio-rs/tokio/pull/5672 +[#5690]: https://github.com/tokio-rs/tokio/pull/5690 +[#5704]: https://github.com/tokio-rs/tokio/pull/5704 +[#5566]: https://github.com/tokio-rs/tokio/pull/5566 +[#5721]: https://github.com/tokio-rs/tokio/pull/5721 +[#5790]: https://github.com/tokio-rs/tokio/pull/5790 +[#5608]: https://github.com/tokio-rs/tokio/pull/5608 +[#5676]: https://github.com/tokio-rs/tokio/pull/5676 +[#5708]: https://github.com/tokio-rs/tokio/pull/5708 +[#5717]: https://github.com/tokio-rs/tokio/pull/5717 +[#5685]: https://github.com/tokio-rs/tokio/pull/5685 + +# 1.28.2 (May 28, 2023) + +Forward ports 1.18.6 changes. + +### Fixed + +- deps: disable default features for mio ([#5728]) + +[#5728]: https://github.com/tokio-rs/tokio/pull/5728 + +# 1.28.1 (May 10th, 2023) + +This release fixes a mistake in the build script that makes `AsFd` +implementations unavailable on Rust 1.63. ([#5677]) + +[#5677]: https://github.com/tokio-rs/tokio/pull/5677 + +# 1.28.0 (April 25th, 2023) + +### Added + +- io: add `AsyncFd::async_io` ([#5542]) +- io: impl BufMut for ReadBuf ([#5590]) +- net: add `recv_buf` for `UdpSocket` and `UnixDatagram` ([#5583]) +- sync: add `OwnedSemaphorePermit::semaphore` ([#5618]) +- sync: add `same_channel` to broadcast channel ([#5607]) +- sync: add `watch::Receiver::wait_for` ([#5611]) +- task: add `JoinSet::spawn_blocking` and `JoinSet::spawn_blocking_on` ([#5612]) + +### Changed + +- deps: update windows-sys to 0.48 ([#5591]) +- io: make `read_to_end` not grow unnecessarily ([#5610]) +- macros: make entrypoints more efficient ([#5621]) +- sync: improve Debug impl for `RwLock` ([#5647]) +- sync: reduce contention in `Notify` ([#5503]) + +### Fixed + +- net: support `get_peer_cred` on AIX ([#5065]) +- sync: avoid deadlocks in `broadcast` with custom wakers ([#5578]) + +### Documented + +- sync: fix typo in `Semaphore::MAX_PERMITS` ([#5645]) +- sync: fix typo in `tokio::sync::watch::Sender` docs ([#5587]) + +[#5065]: https://github.com/tokio-rs/tokio/pull/5065 +[#5503]: https://github.com/tokio-rs/tokio/pull/5503 +[#5542]: https://github.com/tokio-rs/tokio/pull/5542 +[#5578]: https://github.com/tokio-rs/tokio/pull/5578 +[#5583]: https://github.com/tokio-rs/tokio/pull/5583 +[#5587]: https://github.com/tokio-rs/tokio/pull/5587 +[#5590]: https://github.com/tokio-rs/tokio/pull/5590 +[#5591]: https://github.com/tokio-rs/tokio/pull/5591 +[#5607]: https://github.com/tokio-rs/tokio/pull/5607 +[#5610]: https://github.com/tokio-rs/tokio/pull/5610 +[#5611]: https://github.com/tokio-rs/tokio/pull/5611 +[#5612]: https://github.com/tokio-rs/tokio/pull/5612 +[#5618]: https://github.com/tokio-rs/tokio/pull/5618 +[#5621]: https://github.com/tokio-rs/tokio/pull/5621 +[#5645]: https://github.com/tokio-rs/tokio/pull/5645 +[#5647]: https://github.com/tokio-rs/tokio/pull/5647 + +# 1.27.0 (March 27th, 2023) + +This release bumps the MSRV of Tokio to 1.56. ([#5559]) + +### Added + +- io: add `async_io` helper method to sockets ([#5512]) +- io: add implementations of `AsFd`/`AsHandle`/`AsSocket` ([#5514], [#5540]) +- net: add `UdpSocket::peek_sender()` ([#5520]) +- sync: add `RwLockWriteGuard::{downgrade_map, try_downgrade_map}` ([#5527]) +- task: add `JoinHandle::abort_handle` ([#5543]) + +### Changed + +- io: use `memchr` from `libc` ([#5558]) +- macros: accept path as crate rename in `#[tokio::main]` ([#5557]) +- macros: update to syn 2.0.0 ([#5572]) +- time: don't register for a wakeup when `Interval` returns `Ready` ([#5553]) + +### Fixed + +- fs: fuse std iterator in `ReadDir` ([#5555]) +- tracing: fix `spawn_blocking` location fields ([#5573]) +- time: clean up redundant check in `Wheel::poll()` ([#5574]) + +### Documented + +- macros: define cancellation safety ([#5525]) +- io: add details to docs of `tokio::io::copy[_buf]` ([#5575]) +- io: refer to `ReaderStream` and `StreamReader` in module docs ([#5576]) + +[#5512]: https://github.com/tokio-rs/tokio/pull/5512 +[#5514]: https://github.com/tokio-rs/tokio/pull/5514 +[#5520]: https://github.com/tokio-rs/tokio/pull/5520 +[#5525]: https://github.com/tokio-rs/tokio/pull/5525 +[#5527]: https://github.com/tokio-rs/tokio/pull/5527 +[#5540]: https://github.com/tokio-rs/tokio/pull/5540 +[#5543]: https://github.com/tokio-rs/tokio/pull/5543 +[#5553]: https://github.com/tokio-rs/tokio/pull/5553 +[#5555]: https://github.com/tokio-rs/tokio/pull/5555 +[#5557]: https://github.com/tokio-rs/tokio/pull/5557 +[#5558]: https://github.com/tokio-rs/tokio/pull/5558 +[#5559]: https://github.com/tokio-rs/tokio/pull/5559 +[#5572]: https://github.com/tokio-rs/tokio/pull/5572 +[#5573]: https://github.com/tokio-rs/tokio/pull/5573 +[#5574]: https://github.com/tokio-rs/tokio/pull/5574 +[#5575]: https://github.com/tokio-rs/tokio/pull/5575 +[#5576]: https://github.com/tokio-rs/tokio/pull/5576 + +# 1.26.0 (March 1st, 2023) + +### Fixed + +- macros: fix empty `join!` and `try_join!` ([#5504]) +- sync: don't leak tracing spans in mutex guards ([#5469]) +- sync: drop wakers after unlocking the mutex in Notify ([#5471]) +- sync: drop wakers outside lock in semaphore ([#5475]) + +### Added + +- fs: add `fs::try_exists` ([#4299]) +- net: add types for named unix pipes ([#5351]) +- sync: add `MappedOwnedMutexGuard` ([#5474]) + +### Changed + +- chore: update windows-sys to 0.45 ([#5386]) +- net: use Message Read Mode for named pipes ([#5350]) +- sync: mark lock guards with `#[clippy::has_significant_drop]` ([#5422]) +- sync: reduce contention in watch channel ([#5464]) +- time: remove cache padding in timer entries ([#5468]) +- time: Improve `Instant::now()` perf with test-util ([#5513]) + +### Internal Changes + +- io: use `poll_fn` in `copy_bidirectional` ([#5486]) +- net: refactor named pipe builders to not use bitfields ([#5477]) +- rt: remove Arc from Clock ([#5434]) +- sync: make `notify_waiters` calls atomic ([#5458]) +- time: don't store deadline twice in sleep entries ([#5410]) + +### Unstable + +- metrics: add a new metric for budget exhaustion yields ([#5517]) + +### Documented + +- io: improve AsyncFd example ([#5481]) +- runtime: document the nature of the main future ([#5494]) +- runtime: remove extra period in docs ([#5511]) +- signal: updated Documentation for Signals ([#5459]) +- sync: add doc aliases for `blocking_*` methods ([#5448]) +- sync: fix docs for Send/Sync bounds in broadcast ([#5480]) +- sync: document drop behavior for channels ([#5497]) +- task: clarify what happens to spawned work during runtime shutdown ([#5394]) +- task: clarify `process::Command` docs ([#5413]) +- task: fix wording with 'unsend' ([#5452]) +- time: document immediate completion guarantee for timeouts ([#5509]) +- tokio: document supported platforms ([#5483]) + +[#4299]: https://github.com/tokio-rs/tokio/pull/4299 +[#5350]: https://github.com/tokio-rs/tokio/pull/5350 +[#5351]: https://github.com/tokio-rs/tokio/pull/5351 +[#5386]: https://github.com/tokio-rs/tokio/pull/5386 +[#5394]: https://github.com/tokio-rs/tokio/pull/5394 +[#5410]: https://github.com/tokio-rs/tokio/pull/5410 +[#5413]: https://github.com/tokio-rs/tokio/pull/5413 +[#5422]: https://github.com/tokio-rs/tokio/pull/5422 +[#5434]: https://github.com/tokio-rs/tokio/pull/5434 +[#5448]: https://github.com/tokio-rs/tokio/pull/5448 +[#5452]: https://github.com/tokio-rs/tokio/pull/5452 +[#5458]: https://github.com/tokio-rs/tokio/pull/5458 +[#5459]: https://github.com/tokio-rs/tokio/pull/5459 +[#5464]: https://github.com/tokio-rs/tokio/pull/5464 +[#5468]: https://github.com/tokio-rs/tokio/pull/5468 +[#5469]: https://github.com/tokio-rs/tokio/pull/5469 +[#5471]: https://github.com/tokio-rs/tokio/pull/5471 +[#5474]: https://github.com/tokio-rs/tokio/pull/5474 +[#5475]: https://github.com/tokio-rs/tokio/pull/5475 +[#5477]: https://github.com/tokio-rs/tokio/pull/5477 +[#5480]: https://github.com/tokio-rs/tokio/pull/5480 +[#5481]: https://github.com/tokio-rs/tokio/pull/5481 +[#5483]: https://github.com/tokio-rs/tokio/pull/5483 +[#5486]: https://github.com/tokio-rs/tokio/pull/5486 +[#5494]: https://github.com/tokio-rs/tokio/pull/5494 +[#5497]: https://github.com/tokio-rs/tokio/pull/5497 +[#5504]: https://github.com/tokio-rs/tokio/pull/5504 +[#5509]: https://github.com/tokio-rs/tokio/pull/5509 +[#5511]: https://github.com/tokio-rs/tokio/pull/5511 +[#5513]: https://github.com/tokio-rs/tokio/pull/5513 +[#5517]: https://github.com/tokio-rs/tokio/pull/5517 + +# 1.25.1 (May 28, 2023) + +Forward ports 1.18.6 changes. + +### Fixed + +- deps: disable default features for mio ([#5728]) + +[#5728]: https://github.com/tokio-rs/tokio/pull/5728 + +# 1.25.0 (January 28, 2023) + +### Fixed + +- rt: fix runtime metrics reporting ([#5330]) + +### Added + +- sync: add `broadcast::Sender::len` ([#5343]) + +### Changed + +- fs: increase maximum read buffer size to 2MiB ([#5397]) + +[#5330]: https://github.com/tokio-rs/tokio/pull/5330 +[#5343]: https://github.com/tokio-rs/tokio/pull/5343 +[#5397]: https://github.com/tokio-rs/tokio/pull/5397 + +# 1.24.2 (January 17, 2023) + +Forward ports 1.18.5 changes. + +### Fixed + +- io: fix unsoundness in `ReadHalf::unsplit` ([#5375]) + +[#5375]: https://github.com/tokio-rs/tokio/pull/5375 + +# 1.24.1 (January 6, 2022) + +This release fixes a compilation failure on targets without `AtomicU64` when using rustc older than 1.63. ([#5356]) + +[#5356]: https://github.com/tokio-rs/tokio/pull/5356 + +# 1.24.0 (January 5, 2022) + +### Fixed + - rt: improve native `AtomicU64` support detection ([#5284]) + +### Added + - rt: add configuration option for max number of I/O events polled from the OS + per tick ([#5186]) + - rt: add an environment variable for configuring the default number of worker + threads per runtime instance ([#4250]) + +### Changed + - sync: reduce MPSC channel stack usage ([#5294]) + - io: reduce lock contention in I/O operations ([#5300]) + - fs: speed up `read_dir()` by chunking operations ([#5309]) + - rt: use internal `ThreadId` implementation ([#5329]) + - test: don't auto-advance time when a `spawn_blocking` task is running ([#5115]) + +[#5186]: https://github.com/tokio-rs/tokio/pull/5186 +[#5294]: https://github.com/tokio-rs/tokio/pull/5294 +[#5284]: https://github.com/tokio-rs/tokio/pull/5284 +[#4250]: https://github.com/tokio-rs/tokio/pull/4250 +[#5300]: https://github.com/tokio-rs/tokio/pull/5300 +[#5329]: https://github.com/tokio-rs/tokio/pull/5329 +[#5115]: https://github.com/tokio-rs/tokio/pull/5115 +[#5309]: https://github.com/tokio-rs/tokio/pull/5309 + +# 1.23.1 (January 4, 2022) + +This release forward ports changes from 1.18.4. + +### Fixed + +- net: fix Windows named pipe server builder to maintain option when toggling + pipe mode ([#5336]). + +[#5336]: https://github.com/tokio-rs/tokio/pull/5336 + +# 1.23.0 (December 5, 2022) + +### Fixed + + - net: fix Windows named pipe connect ([#5208]) + - io: support vectored writes for `ChildStdin` ([#5216]) + - io: fix `async fn ready()` false positive for OS-specific events ([#5231]) + + ### Changed + - runtime: `yield_now` defers task until after driver poll ([#5223]) + - runtime: reduce amount of codegen needed per spawned task ([#5213]) + - windows: replace `winapi` dependency with `windows-sys` ([#5204]) + + [#5208]: https://github.com/tokio-rs/tokio/pull/5208 + [#5216]: https://github.com/tokio-rs/tokio/pull/5216 + [#5213]: https://github.com/tokio-rs/tokio/pull/5213 + [#5204]: https://github.com/tokio-rs/tokio/pull/5204 + [#5223]: https://github.com/tokio-rs/tokio/pull/5223 + [#5231]: https://github.com/tokio-rs/tokio/pull/5231 + +# 1.22.0 (November 17, 2022) + +### Added + - runtime: add `Handle::runtime_flavor` ([#5138]) + - sync: add `Mutex::blocking_lock_owned` ([#5130]) + - sync: add `Semaphore::MAX_PERMITS` ([#5144]) + - sync: add `merge()` to semaphore permits ([#4948]) + - sync: add `mpsc::WeakUnboundedSender` ([#5189]) + +### Added (unstable) + + - process: add `Command::process_group` ([#5114]) + - runtime: export metrics about the blocking thread pool ([#5161]) + - task: add `task::id()` and `task::try_id()` ([#5171]) + +### Fixed + - macros: don't take ownership of futures in macros ([#5087]) + - runtime: fix Stacked Borrows violation in `LocalOwnedTasks` ([#5099]) + - runtime: mitigate ABA with 32-bit queue indices when possible ([#5042]) + - task: wake local tasks to the local queue when woken by the same thread ([#5095]) + - time: panic in release mode when `mark_pending` called illegally ([#5093]) + - runtime: fix typo in expect message ([#5169]) + - runtime: fix `unsync_load` on atomic types ([#5175]) + - task: elaborate safety comments in task deallocation ([#5172]) + - runtime: fix `LocalSet` drop in thread local ([#5179]) + - net: remove libc type leakage in a public API ([#5191]) + - runtime: update the alignment of `CachePadded` ([#5106]) + +### Changed + - io: make `tokio::io::copy` continue filling the buffer when writer stalls ([#5066]) + - runtime: remove `coop::budget` from `LocalSet::run_until` ([#5155]) + - sync: make `Notify` panic safe ([#5154]) + +### Documented + - io: fix doc for `write_i8` to use signed integers ([#5040]) + - net: fix doc typos for TCP and UDP `set_tos` methods ([#5073]) + - net: fix function name in `UdpSocket::recv` documentation ([#5150]) + - sync: typo in `TryLockError` for `RwLock::try_write` ([#5160]) + - task: document that spawned tasks execute immediately ([#5117]) + - time: document return type of `timeout` ([#5118]) + - time: document that `timeout` checks only before poll ([#5126]) + - sync: specify return type of `oneshot::Receiver` in docs ([#5198]) + +### Internal changes + - runtime: use const `Mutex::new` for globals ([#5061]) + - runtime: remove `Option` around `mio::Events` in io driver ([#5078]) + - runtime: remove a conditional compilation clause ([#5104]) + - runtime: remove a reference to internal time handle ([#5107]) + - runtime: misc time driver cleanup ([#5120]) + - runtime: move signal driver to runtime module ([#5121]) + - runtime: signal driver now uses I/O driver directly ([#5125]) + - runtime: start decoupling I/O driver and I/O handle ([#5127]) + - runtime: switch `io::handle` refs with scheduler:Handle ([#5128]) + - runtime: remove Arc from I/O driver ([#5134]) + - runtime: use signal driver handle via `scheduler::Handle` ([#5135]) + - runtime: move internal clock fns out of context ([#5139]) + - runtime: remove `runtime::context` module ([#5140]) + - runtime: keep driver cfgs in `driver.rs` ([#5141]) + - runtime: add `runtime::context` to unify thread-locals ([#5143]) + - runtime: rename some confusing internal variables/fns ([#5151]) + - runtime: move `coop` mod into `runtime` ([#5152]) + - runtime: move budget state to context thread-local ([#5157]) + - runtime: move park logic into runtime module ([#5158]) + - runtime: move `Runtime` into its own file ([#5159]) + - runtime: unify entering a runtime with `Handle::enter` ([#5163]) + - runtime: remove handle reference from each scheduler ([#5166]) + - runtime: move `enter` into `context` ([#5167]) + - runtime: combine context and entered thread-locals ([#5168]) + - runtime: fix accidental unsetting of current handle ([#5178]) + - runtime: move `CoreStage` methods to `Core` ([#5182]) + - sync: name mpsc semaphore types ([#5146]) + +[#4948]: https://github.com/tokio-rs/tokio/pull/4948 +[#5040]: https://github.com/tokio-rs/tokio/pull/5040 +[#5042]: https://github.com/tokio-rs/tokio/pull/5042 +[#5061]: https://github.com/tokio-rs/tokio/pull/5061 +[#5066]: https://github.com/tokio-rs/tokio/pull/5066 +[#5073]: https://github.com/tokio-rs/tokio/pull/5073 +[#5078]: https://github.com/tokio-rs/tokio/pull/5078 +[#5087]: https://github.com/tokio-rs/tokio/pull/5087 +[#5093]: https://github.com/tokio-rs/tokio/pull/5093 +[#5095]: https://github.com/tokio-rs/tokio/pull/5095 +[#5099]: https://github.com/tokio-rs/tokio/pull/5099 +[#5104]: https://github.com/tokio-rs/tokio/pull/5104 +[#5106]: https://github.com/tokio-rs/tokio/pull/5106 +[#5107]: https://github.com/tokio-rs/tokio/pull/5107 +[#5114]: https://github.com/tokio-rs/tokio/pull/5114 +[#5117]: https://github.com/tokio-rs/tokio/pull/5117 +[#5118]: https://github.com/tokio-rs/tokio/pull/5118 +[#5120]: https://github.com/tokio-rs/tokio/pull/5120 +[#5121]: https://github.com/tokio-rs/tokio/pull/5121 +[#5125]: https://github.com/tokio-rs/tokio/pull/5125 +[#5126]: https://github.com/tokio-rs/tokio/pull/5126 +[#5127]: https://github.com/tokio-rs/tokio/pull/5127 +[#5128]: https://github.com/tokio-rs/tokio/pull/5128 +[#5130]: https://github.com/tokio-rs/tokio/pull/5130 +[#5134]: https://github.com/tokio-rs/tokio/pull/5134 +[#5135]: https://github.com/tokio-rs/tokio/pull/5135 +[#5138]: https://github.com/tokio-rs/tokio/pull/5138 +[#5138]: https://github.com/tokio-rs/tokio/pull/5138 +[#5139]: https://github.com/tokio-rs/tokio/pull/5139 +[#5140]: https://github.com/tokio-rs/tokio/pull/5140 +[#5141]: https://github.com/tokio-rs/tokio/pull/5141 +[#5143]: https://github.com/tokio-rs/tokio/pull/5143 +[#5144]: https://github.com/tokio-rs/tokio/pull/5144 +[#5144]: https://github.com/tokio-rs/tokio/pull/5144 +[#5146]: https://github.com/tokio-rs/tokio/pull/5146 +[#5150]: https://github.com/tokio-rs/tokio/pull/5150 +[#5151]: https://github.com/tokio-rs/tokio/pull/5151 +[#5152]: https://github.com/tokio-rs/tokio/pull/5152 +[#5154]: https://github.com/tokio-rs/tokio/pull/5154 +[#5155]: https://github.com/tokio-rs/tokio/pull/5155 +[#5157]: https://github.com/tokio-rs/tokio/pull/5157 +[#5158]: https://github.com/tokio-rs/tokio/pull/5158 +[#5159]: https://github.com/tokio-rs/tokio/pull/5159 +[#5160]: https://github.com/tokio-rs/tokio/pull/5160 +[#5161]: https://github.com/tokio-rs/tokio/pull/5161 +[#5163]: https://github.com/tokio-rs/tokio/pull/5163 +[#5166]: https://github.com/tokio-rs/tokio/pull/5166 +[#5167]: https://github.com/tokio-rs/tokio/pull/5167 +[#5168]: https://github.com/tokio-rs/tokio/pull/5168 +[#5169]: https://github.com/tokio-rs/tokio/pull/5169 +[#5171]: https://github.com/tokio-rs/tokio/pull/5171 +[#5172]: https://github.com/tokio-rs/tokio/pull/5172 +[#5175]: https://github.com/tokio-rs/tokio/pull/5175 +[#5178]: https://github.com/tokio-rs/tokio/pull/5178 +[#5179]: https://github.com/tokio-rs/tokio/pull/5179 +[#5182]: https://github.com/tokio-rs/tokio/pull/5182 +[#5189]: https://github.com/tokio-rs/tokio/pull/5189 +[#5191]: https://github.com/tokio-rs/tokio/pull/5191 +[#5198]: https://github.com/tokio-rs/tokio/pull/5198 + +# 1.21.2 (September 27, 2022) + +This release removes the dependency on the `once_cell` crate to restore the MSRV +of 1.21.x, which is the latest minor version at the time of release. ([#5048]) + +[#5048]: https://github.com/tokio-rs/tokio/pull/5048 + +# 1.21.1 (September 13, 2022) + +### Fixed + +- net: fix dependency resolution for socket2 ([#5000]) +- task: ignore failure to set TLS in `LocalSet` Drop ([#4976]) + +[#4976]: https://github.com/tokio-rs/tokio/pull/4976 +[#5000]: https://github.com/tokio-rs/tokio/pull/5000 + +# 1.21.0 (September 2, 2022) + +This release is the first release of Tokio to intentionally support WASM. The +`sync,macros,io-util,rt,time` features are stabilized on WASM. Additionally the +wasm32-wasi target is given unstable support for the `net` feature. + +### Added + +- net: add `device` and `bind_device` methods to TCP/UDP sockets ([#4882]) +- net: add `tos` and `set_tos` methods to TCP and UDP sockets ([#4877]) +- net: add security flags to named pipe `ServerOptions` ([#4845]) +- signal: add more windows signal handlers ([#4924]) +- sync: add `mpsc::Sender::max_capacity` method ([#4904]) +- sync: implement Weak version of `mpsc::Sender` ([#4595]) +- task: add `LocalSet::enter` ([#4765]) +- task: stabilize `JoinSet` and `AbortHandle` ([#4920]) +- tokio: add `track_caller` to public APIs ([#4805], [#4848], [#4852]) +- wasm: initial support for `wasm32-wasi` target ([#4716]) + +### Fixed + +- miri: improve miri compatibility by avoiding temporary references in `linked_list::Link` impls ([#4841]) +- signal: don't register write interest on signal pipe ([#4898]) +- sync: add `#[must_use]` to lock guards ([#4886]) +- sync: fix hang when calling `recv` on closed and reopened broadcast channel ([#4867]) +- task: propagate attributes on task-locals ([#4837]) + +### Changed + +- fs: change panic to error in `File::start_seek` ([#4897]) +- io: reduce syscalls in `poll_read` ([#4840]) +- process: use blocking threadpool for child stdio I/O ([#4824]) +- signal: make `SignalKind` methods const ([#4956]) + +### Internal changes + +- rt: extract `basic_scheduler::Config` ([#4935]) +- rt: move I/O driver into `runtime` module ([#4942]) +- rt: rename internal scheduler types ([#4945]) + +### Documented + +- chore: fix typos and grammar ([#4858], [#4894], [#4928]) +- io: fix typo in `AsyncSeekExt::rewind` docs ([#4893]) +- net: add documentation to `try_read()` for zero-length buffers ([#4937]) +- runtime: remove incorrect panic section for `Builder::worker_threads` ([#4849]) +- sync: doc of `watch::Sender::send` improved ([#4959]) +- task: add cancel safety docs to `JoinHandle` ([#4901]) +- task: expand on cancellation of `spawn_blocking` ([#4811]) +- time: clarify that the first tick of `Interval::tick` happens immediately ([#4951]) + +### Unstable + +- rt: add unstable option to disable the LIFO slot ([#4936]) +- task: fix incorrect signature in `Builder::spawn_on` ([#4953]) +- task: make `task::Builder::spawn*` methods fallible ([#4823]) + +[#4595]: https://github.com/tokio-rs/tokio/pull/4595 +[#4716]: https://github.com/tokio-rs/tokio/pull/4716 +[#4765]: https://github.com/tokio-rs/tokio/pull/4765 +[#4805]: https://github.com/tokio-rs/tokio/pull/4805 +[#4811]: https://github.com/tokio-rs/tokio/pull/4811 +[#4823]: https://github.com/tokio-rs/tokio/pull/4823 +[#4824]: https://github.com/tokio-rs/tokio/pull/4824 +[#4837]: https://github.com/tokio-rs/tokio/pull/4837 +[#4840]: https://github.com/tokio-rs/tokio/pull/4840 +[#4841]: https://github.com/tokio-rs/tokio/pull/4841 +[#4845]: https://github.com/tokio-rs/tokio/pull/4845 +[#4848]: https://github.com/tokio-rs/tokio/pull/4848 +[#4849]: https://github.com/tokio-rs/tokio/pull/4849 +[#4852]: https://github.com/tokio-rs/tokio/pull/4852 +[#4858]: https://github.com/tokio-rs/tokio/pull/4858 +[#4867]: https://github.com/tokio-rs/tokio/pull/4867 +[#4877]: https://github.com/tokio-rs/tokio/pull/4877 +[#4882]: https://github.com/tokio-rs/tokio/pull/4882 +[#4886]: https://github.com/tokio-rs/tokio/pull/4886 +[#4893]: https://github.com/tokio-rs/tokio/pull/4893 +[#4894]: https://github.com/tokio-rs/tokio/pull/4894 +[#4897]: https://github.com/tokio-rs/tokio/pull/4897 +[#4898]: https://github.com/tokio-rs/tokio/pull/4898 +[#4901]: https://github.com/tokio-rs/tokio/pull/4901 +[#4904]: https://github.com/tokio-rs/tokio/pull/4904 +[#4920]: https://github.com/tokio-rs/tokio/pull/4920 +[#4924]: https://github.com/tokio-rs/tokio/pull/4924 +[#4928]: https://github.com/tokio-rs/tokio/pull/4928 +[#4935]: https://github.com/tokio-rs/tokio/pull/4935 +[#4936]: https://github.com/tokio-rs/tokio/pull/4936 +[#4937]: https://github.com/tokio-rs/tokio/pull/4937 +[#4942]: https://github.com/tokio-rs/tokio/pull/4942 +[#4945]: https://github.com/tokio-rs/tokio/pull/4945 +[#4951]: https://github.com/tokio-rs/tokio/pull/4951 +[#4953]: https://github.com/tokio-rs/tokio/pull/4953 +[#4956]: https://github.com/tokio-rs/tokio/pull/4956 +[#4959]: https://github.com/tokio-rs/tokio/pull/4959 + +# 1.20.5 (May 28, 2023) + +Forward ports 1.18.6 changes. + +### Fixed + +- deps: disable default features for mio ([#5728]) + +[#5728]: https://github.com/tokio-rs/tokio/pull/5728 + +# 1.20.4 (January 17, 2023) + +Forward ports 1.18.5 changes. + +### Fixed + +- io: fix unsoundness in `ReadHalf::unsplit` ([#5375]) + +[#5375]: https://github.com/tokio-rs/tokio/pull/5375 + +# 1.20.3 (January 3, 2022) + +This release forward ports changes from 1.18.4. + +### Fixed + +- net: fix Windows named pipe server builder to maintain option when toggling + pipe mode ([#5336]). + +[#5336]: https://github.com/tokio-rs/tokio/pull/5336 + +# 1.20.2 (September 27, 2022) + +This release removes the dependency on the `once_cell` crate to restore the MSRV +of the 1.20.x LTS release. ([#5048]) + +[#5048]: https://github.com/tokio-rs/tokio/pull/5048 + +# 1.20.1 (July 25, 2022) + +### Fixed + +- chore: fix version detection in build script ([#4860]) + +[#4860]: https://github.com/tokio-rs/tokio/pull/4860 + +# 1.20.0 (July 12, 2022) + +### Added +- tokio: add `track_caller` to public APIs ([#4772], [#4791], [#4793], [#4806], [#4808]) +- sync: Add `has_changed` method to `watch::Ref` ([#4758]) + +### Changed + +- time: remove `src/time/driver/wheel/stack.rs` ([#4766]) +- rt: clean up arguments passed to basic scheduler ([#4767]) +- net: be more specific about winapi features ([#4764]) +- tokio: use const initialized thread locals where possible ([#4677]) +- task: various small improvements to LocalKey ([#4795]) + +### Documented + +- fs: warn about performance pitfall ([#4762]) +- chore: fix spelling ([#4769]) +- sync: document spurious failures in oneshot ([#4777]) +- sync: add warning for watch in non-Send futures ([#4741]) +- chore: fix typo ([#4798]) + +### Unstable + +- joinset: rename `join_one` to `join_next` ([#4755]) +- rt: unhandled panic config for current thread rt ([#4770]) + +[#4677]: https://github.com/tokio-rs/tokio/pull/4677 +[#4741]: https://github.com/tokio-rs/tokio/pull/4741 +[#4755]: https://github.com/tokio-rs/tokio/pull/4755 +[#4758]: https://github.com/tokio-rs/tokio/pull/4758 +[#4762]: https://github.com/tokio-rs/tokio/pull/4762 +[#4764]: https://github.com/tokio-rs/tokio/pull/4764 +[#4766]: https://github.com/tokio-rs/tokio/pull/4766 +[#4767]: https://github.com/tokio-rs/tokio/pull/4767 +[#4769]: https://github.com/tokio-rs/tokio/pull/4769 +[#4770]: https://github.com/tokio-rs/tokio/pull/4770 +[#4772]: https://github.com/tokio-rs/tokio/pull/4772 +[#4777]: https://github.com/tokio-rs/tokio/pull/4777 +[#4791]: https://github.com/tokio-rs/tokio/pull/4791 +[#4793]: https://github.com/tokio-rs/tokio/pull/4793 +[#4795]: https://github.com/tokio-rs/tokio/pull/4795 +[#4798]: https://github.com/tokio-rs/tokio/pull/4798 +[#4806]: https://github.com/tokio-rs/tokio/pull/4806 +[#4808]: https://github.com/tokio-rs/tokio/pull/4808 + +# 1.19.2 (June 6, 2022) + +This release fixes another bug in `Notified::enable`. ([#4751]) + +[#4751]: https://github.com/tokio-rs/tokio/pull/4751 + +# 1.19.1 (June 5, 2022) + +This release fixes a bug in `Notified::enable`. ([#4747]) + +[#4747]: https://github.com/tokio-rs/tokio/pull/4747 + +# 1.19.0 (June 3, 2022) + +### Added + +- runtime: add `is_finished` method for `JoinHandle` and `AbortHandle` ([#4709]) +- runtime: make global queue and event polling intervals configurable ([#4671]) +- sync: add `Notified::enable` ([#4705]) +- sync: add `watch::Sender::send_if_modified` ([#4591]) +- sync: add resubscribe method to broadcast::Receiver ([#4607]) +- net: add `take_error` to `TcpSocket` and `TcpStream` ([#4739]) + +### Changed + +- io: refactor out usage of Weak in the io handle ([#4656]) + +### Fixed + +- macros: avoid starvation in `join!` and `try_join!` ([#4624]) + +### Documented + +- runtime: clarify semantics of tasks outliving `block_on` ([#4729]) +- time: fix example for `MissedTickBehavior::Burst` ([#4713]) + +### Unstable + +- metrics: correctly update atomics in `IoDriverMetrics` ([#4725]) +- metrics: fix compilation with unstable, process, and rt, but without net ([#4682]) +- task: add `#[track_caller]` to `JoinSet`/`JoinMap` ([#4697]) +- task: add `Builder::{spawn_on, spawn_local_on, spawn_blocking_on}` ([#4683]) +- task: add `consume_budget` for cooperative scheduling ([#4498]) +- task: add `join_set::Builder` for configuring `JoinSet` tasks ([#4687]) +- task: update return value of `JoinSet::join_one` ([#4726]) + +[#4498]: https://github.com/tokio-rs/tokio/pull/4498 +[#4591]: https://github.com/tokio-rs/tokio/pull/4591 +[#4607]: https://github.com/tokio-rs/tokio/pull/4607 +[#4624]: https://github.com/tokio-rs/tokio/pull/4624 +[#4656]: https://github.com/tokio-rs/tokio/pull/4656 +[#4671]: https://github.com/tokio-rs/tokio/pull/4671 +[#4682]: https://github.com/tokio-rs/tokio/pull/4682 +[#4683]: https://github.com/tokio-rs/tokio/pull/4683 +[#4687]: https://github.com/tokio-rs/tokio/pull/4687 +[#4697]: https://github.com/tokio-rs/tokio/pull/4697 +[#4705]: https://github.com/tokio-rs/tokio/pull/4705 +[#4709]: https://github.com/tokio-rs/tokio/pull/4709 +[#4713]: https://github.com/tokio-rs/tokio/pull/4713 +[#4725]: https://github.com/tokio-rs/tokio/pull/4725 +[#4726]: https://github.com/tokio-rs/tokio/pull/4726 +[#4729]: https://github.com/tokio-rs/tokio/pull/4729 +[#4739]: https://github.com/tokio-rs/tokio/pull/4739 + +# 1.18.6 (May 28, 2023) + +### Fixed + +- deps: disable default features for mio ([#5728]) + +[#5728]: https://github.com/tokio-rs/tokio/pull/5728 + +# 1.18.5 (January 17, 2023) + +### Fixed + +- io: fix unsoundness in `ReadHalf::unsplit` ([#5375]) + +[#5375]: https://github.com/tokio-rs/tokio/pull/5375 + +# 1.18.4 (January 3, 2022) + +### Fixed + +- net: fix Windows named pipe server builder to maintain option when toggling + pipe mode ([#5336]). + +[#5336]: https://github.com/tokio-rs/tokio/pull/5336 + +# 1.18.3 (September 27, 2022) + +This release removes the dependency on the `once_cell` crate to restore the MSRV +of the 1.18.x LTS release. ([#5048]) + +[#5048]: https://github.com/tokio-rs/tokio/pull/5048 + +# 1.18.2 (May 5, 2022) + +Add missing features for the `winapi` dependency. ([#4663]) + +[#4663]: https://github.com/tokio-rs/tokio/pull/4663 + +# 1.18.1 (May 2, 2022) + +The 1.18.0 release broke the build for targets without 64-bit atomics when +building with `tokio_unstable`. This release fixes that. ([#4649]) + +[#4649]: https://github.com/tokio-rs/tokio/pull/4649 + +# 1.18.0 (April 27, 2022) + +This release adds a number of new APIs in `tokio::net`, `tokio::signal`, and +`tokio::sync`. In addition, it adds new unstable APIs to `tokio::task` (`Id`s +for uniquely identifying a task, and `AbortHandle` for remotely cancelling a +task), as well as a number of bugfixes. + +### Fixed + +- blocking: add missing `#[track_caller]` for `spawn_blocking` ([#4616]) +- macros: fix `select` macro to process 64 branches ([#4519]) +- net: fix `try_io` methods not calling Mio's `try_io` internally ([#4582]) +- runtime: recover when OS fails to spawn a new thread ([#4485]) + +### Added + +- net: add `UdpSocket::peer_addr` ([#4611]) +- net: add `try_read_buf` method for named pipes ([#4626]) +- signal: add `SignalKind` `Hash`/`Eq` impls and `c_int` conversion ([#4540]) +- signal: add support for signals up to `SIGRTMAX` ([#4555]) +- sync: add `watch::Sender::send_modify` method ([#4310]) +- sync: add `broadcast::Receiver::len` method ([#4542]) +- sync: add `watch::Receiver::same_channel` method ([#4581]) +- sync: implement `Clone` for `RecvError` types ([#4560]) + +### Changed + +- update `mio` to 0.8.1 ([#4582]) +- macros: rename `tokio::select!`'s internal `util` module ([#4543]) +- runtime: use `Vec::with_capacity` when building runtime ([#4553]) + +### Documented + +- improve docs for `tokio_unstable` ([#4524]) +- runtime: include more documentation for thread_pool/worker ([#4511]) +- runtime: update `Handle::current`'s docs to mention `EnterGuard` ([#4567]) +- time: clarify platform specific timer resolution ([#4474]) +- signal: document that `Signal::recv` is cancel-safe ([#4634]) +- sync: `UnboundedReceiver` close docs ([#4548]) + +### Unstable + +The following changes only apply when building with `--cfg tokio_unstable`: + +- task: add `task::Id` type ([#4630]) +- task: add `AbortHandle` type for cancelling tasks in a `JoinSet` ([#4530], + [#4640]) +- task: fix missing `doc(cfg(...))` attributes for `JoinSet` ([#4531]) +- task: fix broken link in `AbortHandle` RustDoc ([#4545]) +- metrics: add initial IO driver metrics ([#4507]) + + +[#4616]: https://github.com/tokio-rs/tokio/pull/4616 +[#4519]: https://github.com/tokio-rs/tokio/pull/4519 +[#4582]: https://github.com/tokio-rs/tokio/pull/4582 +[#4485]: https://github.com/tokio-rs/tokio/pull/4485 +[#4613]: https://github.com/tokio-rs/tokio/pull/4613 +[#4611]: https://github.com/tokio-rs/tokio/pull/4611 +[#4626]: https://github.com/tokio-rs/tokio/pull/4626 +[#4540]: https://github.com/tokio-rs/tokio/pull/4540 +[#4555]: https://github.com/tokio-rs/tokio/pull/4555 +[#4310]: https://github.com/tokio-rs/tokio/pull/4310 +[#4542]: https://github.com/tokio-rs/tokio/pull/4542 +[#4581]: https://github.com/tokio-rs/tokio/pull/4581 +[#4560]: https://github.com/tokio-rs/tokio/pull/4560 +[#4631]: https://github.com/tokio-rs/tokio/pull/4631 +[#4582]: https://github.com/tokio-rs/tokio/pull/4582 +[#4543]: https://github.com/tokio-rs/tokio/pull/4543 +[#4553]: https://github.com/tokio-rs/tokio/pull/4553 +[#4524]: https://github.com/tokio-rs/tokio/pull/4524 +[#4511]: https://github.com/tokio-rs/tokio/pull/4511 +[#4567]: https://github.com/tokio-rs/tokio/pull/4567 +[#4474]: https://github.com/tokio-rs/tokio/pull/4474 +[#4634]: https://github.com/tokio-rs/tokio/pull/4634 +[#4548]: https://github.com/tokio-rs/tokio/pull/4548 +[#4630]: https://github.com/tokio-rs/tokio/pull/4630 +[#4530]: https://github.com/tokio-rs/tokio/pull/4530 +[#4640]: https://github.com/tokio-rs/tokio/pull/4640 +[#4531]: https://github.com/tokio-rs/tokio/pull/4531 +[#4545]: https://github.com/tokio-rs/tokio/pull/4545 +[#4507]: https://github.com/tokio-rs/tokio/pull/4507 + # 1.17.0 (February 16, 2022) This release updates the minimum supported Rust version (MSRV) to 1.49, the @@ -23,7 +921,7 @@ performance improvements. - time: use bit manipulation instead of modulo to improve performance ([#4480]) - net: use `std::future::Ready` instead of our own `Ready` future ([#4271]) - replace deprecated `atomic::spin_loop_hint` with `hint::spin_loop` ([#4491]) -- fix miri failures in intrusive linked lists ([#4397]) +- fix miri failures in intrusive linked lists ([#4397]) ### Documented @@ -1076,7 +1974,7 @@ Biggest changes are: - Feature flags are simplified - `rt-core` and `rt-util` are combined to `rt` - `rt-threaded` is renamed to `rt-multi-thread` to match builder API - - `tcp`, `udp`, `uds`, `dns` are combied to `net`. + - `tcp`, `udp`, `uds`, `dns` are combined to `net`. - `parking_lot` is included with `full` ### Changes @@ -1574,7 +2472,7 @@ Biggest changes are: - `net::lookup_host` maps a `T: ToSocketAddrs` to a stream of `SocketAddrs` ([#1870]). - `process::Child` fields are made public to match `std` ([#2014]). - impl `Stream` for `sync::broadcast::Receiver` ([#2012]). -- `sync::RwLock` provides an asynchonous read-write lock ([#1699]). +- `sync::RwLock` provides an asynchronous read-write lock ([#1699]). - `runtime::Handle::current` returns the handle for the current runtime ([#2040]). - `StreamExt::filter` filters stream values according to a predicate ([#2001]). - `StreamExt::filter_map` simultaneously filter and map stream values ([#2001]). @@ -1683,7 +2581,7 @@ Biggest changes are: ### Fixes - calling `spawn_blocking` after runtime shutdown ([#1875]). -- `LocalSet` drop inifinite loop ([#1892]). +- `LocalSet` drop infinite loop ([#1892]). - `LocalSet` hang under load ([#1905]). - improved documentation ([#1865], [#1866], [#1868], [#1874], [#1876], [#1911]). diff --git a/third_party/rust/tokio/Cargo.toml b/third_party/rust/tokio/Cargo.toml index 1141c07a8321..da1e4d12c22a 100644 --- a/third_party/rust/tokio/Cargo.toml +++ b/third_party/rust/tokio/Cargo.toml @@ -10,60 +10,73 @@ # See Cargo.toml.orig for the original contents. [package] -edition = "2018" -rust-version = "1.49" +edition = "2021" +rust-version = "1.56" name = "tokio" -version = "1.17.0" +version = "1.29.1" authors = ["Tokio Contributors "] -description = "An event-driven, non-blocking I/O platform for writing asynchronous I/O\nbacked applications.\n" +description = """ +An event-driven, non-blocking I/O platform for writing asynchronous I/O +backed applications. +""" homepage = "https://tokio.rs" readme = "README.md" -keywords = ["io", "async", "non-blocking", "futures"] -categories = ["asynchronous", "network-programming"] +keywords = [ + "io", + "async", + "non-blocking", + "futures", +] +categories = [ + "asynchronous", + "network-programming", +] license = "MIT" repository = "https://github.com/tokio-rs/tokio" + [package.metadata.docs.rs] all-features = true -rustc-args = ["--cfg", "tokio_unstable"] -rustdoc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"] +rustc-args = [ + "--cfg", + "tokio_unstable", +] +rustdoc-args = [ + "--cfg", + "docsrs", + "--cfg", + "tokio_unstable", +] [package.metadata.playground] -features = ["full", "test-util"] +features = [ + "full", + "test-util", +] + [dependencies.bytes] version = "1.0.0" optional = true -[dependencies.memchr] -version = "2.2" -optional = true - [dependencies.mio] -version = "0.8.0" +version = "0.8.6" optional = true +default-features = false [dependencies.num_cpus] version = "1.8.0" optional = true -[dependencies.once_cell] -version = "1.5.2" -optional = true - [dependencies.parking_lot] version = "0.12.0" optional = true [dependencies.pin-project-lite] -version = "0.2.0" - -[dependencies.socket2] -version = "0.4.4" -features = ["all"] -optional = true +version = "0.2.7" [dependencies.tokio-macros] -version = "1.7.0" +version = "~2.1.0" optional = true + [dev-dependencies.async-stream] version = "0.3" @@ -72,10 +85,7 @@ version = "0.3.0" features = ["async-await"] [dev-dependencies.mockall] -version = "0.10.2" - -[dev-dependencies.tempfile] -version = "3.1.0" +version = "0.11.1" [dev-dependencies.tokio-stream] version = "0.1" @@ -83,58 +93,138 @@ version = "0.1" [dev-dependencies.tokio-test] version = "0.4.0" +[build-dependencies.autocfg] +version = "1.1" + [features] default = [] fs = [] -full = ["fs", "io-util", "io-std", "macros", "net", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "sync", "time"] +full = [ + "fs", + "io-util", + "io-std", + "macros", + "net", + "parking_lot", + "process", + "rt", + "rt-multi-thread", + "signal", + "sync", + "time", +] io-std = [] -io-util = ["memchr", "bytes"] +io-util = ["bytes"] macros = ["tokio-macros"] -net = ["libc", "mio/os-poll", "mio/os-ext", "mio/net", "socket2", "winapi/namedpipeapi"] -process = ["bytes", "once_cell", "libc", "mio/os-poll", "mio/os-ext", "mio/net", "signal-hook-registry", "winapi/threadpoollegacyapiset"] +net = [ + "libc", + "mio/os-poll", + "mio/os-ext", + "mio/net", + "socket2", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_Security", + "windows-sys/Win32_Storage_FileSystem", + "windows-sys/Win32_System_Pipes", + "windows-sys/Win32_System_SystemServices", +] +process = [ + "bytes", + "libc", + "mio/os-poll", + "mio/os-ext", + "mio/net", + "signal-hook-registry", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_System_Threading", + "windows-sys/Win32_System_WindowsProgramming", +] rt = [] -rt-multi-thread = ["num_cpus", "rt"] -signal = ["once_cell", "libc", "mio/os-poll", "mio/net", "mio/os-ext", "signal-hook-registry", "winapi/consoleapi"] +rt-multi-thread = [ + "num_cpus", + "rt", +] +signal = [ + "libc", + "mio/os-poll", + "mio/net", + "mio/os-ext", + "signal-hook-registry", + "windows-sys/Win32_Foundation", + "windows-sys/Win32_System_Console", +] stats = [] sync = [] -test-util = ["rt", "sync", "time"] +test-util = [ + "rt", + "sync", + "time", +] time = [] + +[target."cfg(all(any(target_arch = \"wasm32\", target_arch = \"wasm64\"), not(target_os = \"wasi\")))".dev-dependencies.wasm-bindgen-test] +version = "0.3.0" + [target."cfg(loom)".dev-dependencies.loom] version = "0.5.2" -features = ["futures", "checkpoint"] -[target."cfg(not(target_arch = \"wasm32\"))".dev-dependencies.proptest] -version = "1" +features = [ + "futures", + "checkpoint", +] -[target."cfg(not(target_arch = \"wasm32\"))".dev-dependencies.rand] +[target."cfg(not(all(any(target_arch = \"wasm32\", target_arch = \"wasm64\"), target_os = \"unknown\")))".dev-dependencies.rand] version = "0.8.0" -[target."cfg(not(target_arch = \"wasm32\"))".dev-dependencies.socket2] -version = "0.4" -[target."cfg(target_arch = \"wasm32\")".dev-dependencies.wasm-bindgen-test] -version = "0.3.0" +[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dependencies.socket2] +version = "0.4.9" +features = ["all"] +optional = true + +[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dev-dependencies.socket2] +version = "0.4.9" + +[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dev-dependencies.tempfile] +version = "3.1.0" + [target."cfg(target_os = \"freebsd\")".dev-dependencies.mio-aio] -version = "0.6.0" +version = "0.7.0" features = ["tokio"] + +[target."cfg(tokio_taskdump)".dependencies.backtrace] +version = "0.3.58" + [target."cfg(tokio_unstable)".dependencies.tracing] version = "0.1.25" features = ["std"] optional = true default-features = false + [target."cfg(unix)".dependencies.libc] -version = "0.2.42" +version = "0.2.145" optional = true [target."cfg(unix)".dependencies.signal-hook-registry] version = "1.1.1" optional = true + [target."cfg(unix)".dev-dependencies.libc] -version = "0.2.42" +version = "0.2.145" [target."cfg(unix)".dev-dependencies.nix] -version = "0.23" -[target."cfg(windows)".dependencies.winapi] -version = "0.3.8" -optional = true +version = "0.26" +features = [ + "fs", + "socket", +] default-features = false -[target."cfg(windows)".dev-dependencies.ntapi] -version = "0.3.6" + +[target."cfg(windows)".dependencies.windows-sys] +version = "0.48" +optional = true + +[target."cfg(windows)".dev-dependencies.windows-sys] +version = "0.48" +features = [ + "Win32_Foundation", + "Win32_Security_Authorization", +] diff --git a/third_party/rust/tokio/LICENSE b/third_party/rust/tokio/LICENSE index 8af5baf01ea6..8bdf6bd60d30 100644 --- a/third_party/rust/tokio/LICENSE +++ b/third_party/rust/tokio/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2022 Tokio Contributors +Copyright (c) 2023 Tokio Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated diff --git a/third_party/rust/tokio/README.md b/third_party/rust/tokio/README.md index 1cce34aeeff2..fb2c149abddb 100644 --- a/third_party/rust/tokio/README.md +++ b/third_party/rust/tokio/README.md @@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml: ```toml [dependencies] -tokio = { version = "1.17.0", features = ["full"] } +tokio = { version = "1.29.1", features = ["full"] } ``` Then, on your main.rs: @@ -161,11 +161,46 @@ several other libraries, including: [`mio`]: https://github.com/tokio-rs/mio [`bytes`]: https://github.com/tokio-rs/bytes +## Changelog + +The Tokio repository contains multiple crates. Each crate has its own changelog. + + * `tokio` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio/CHANGELOG.md) + * `tokio-util` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-util/CHANGELOG.md) + * `tokio-stream` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-stream/CHANGELOG.md) + * `tokio-macros` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-macros/CHANGELOG.md) + * `tokio-test` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-test/CHANGELOG.md) + ## Supported Rust Versions + + Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at least** 6 months. When increasing the MSRV, the new Rust version must have been -released at least six months ago. The current MSRV is 1.49.0. +released at least six months ago. The current MSRV is 1.56.0. + +Note that the MSRV is not increased automatically, and only as part of a minor +release. The MSRV history for past minor releases can be found below: + + * 1.27 to now - Rust 1.56 + * 1.17 to 1.26 - Rust 1.49 + * 1.15 to 1.16 - Rust 1.46 + * 1.0 to 1.14 - Rust 1.45 + +Note that although we try to avoid the situation where a dependency transitively +increases the MSRV of Tokio, we do not guarantee that this does not happen. +However, every minor release will have some set of versions of dependencies that +works with the MSRV of that minor release. ## Release schedule @@ -180,18 +215,19 @@ warrants a patch release with a fix for the bug, it will be backported and released as a new patch release for each LTS minor version. Our current LTS releases are: - * `1.8.x` - LTS release until February 2022. - * `1.14.x` - LTS release until June 2022. + * `1.18.x` - LTS release until June 2023. (MSRV 1.49) + * `1.20.x` - LTS release until September 2023. (MSRV 1.49) + * `1.25.x` - LTS release until March 2024. (MSRV 1.49) -Each LTS release will continue to receive backported fixes for at least half a -year. If you wish to use a fixed minor release in your project, we recommend -that you use an LTS release. +Each LTS release will continue to receive backported fixes for at least a year. +If you wish to use a fixed minor release in your project, we recommend that you +use an LTS release. To use a fixed minor version, you can specify the version with a tilde. For -example, to specify that you wish to use the newest `1.8.x` patch release, you +example, to specify that you wish to use the newest `1.18.x` patch release, you can use the following dependency specification: ```text -tokio = { version = "~1.8", features = [...] } +tokio = { version = "~1.18", features = [...] } ``` ## License diff --git a/third_party/rust/tokio/build.rs b/third_party/rust/tokio/build.rs new file mode 100644 index 000000000000..b8234dde9a08 --- /dev/null +++ b/third_party/rust/tokio/build.rs @@ -0,0 +1,192 @@ +use autocfg::AutoCfg; + +const CONST_THREAD_LOCAL_PROBE: &str = r#" +{ + thread_local! { + static MY_PROBE: usize = const { 10 }; + } + + MY_PROBE.with(|val| *val) +} +"#; + +const CONST_MUTEX_NEW_PROBE: &str = r#" +{ + static MY_MUTEX: ::std::sync::Mutex = ::std::sync::Mutex::new(1); + *MY_MUTEX.lock().unwrap() +} +"#; + +const AS_FD_PROBE: &str = r#" +{ + #[allow(unused_imports)] + #[cfg(unix)] + use std::os::unix::prelude::AsFd as _; + #[allow(unused_imports)] + #[cfg(windows)] + use std::os::windows::prelude::AsSocket as _; + #[allow(unused_imports)] + #[cfg(target_os = "wasi")] + use std::os::wasi::prelude::AsFd as _; +} +"#; + +const TARGET_HAS_ATOMIC_PROBE: &str = r#" +{ + #[cfg(target_has_atomic = "ptr")] + let _ = (); +} +"#; + +const TARGET_ATOMIC_U64_PROBE: &str = r#" +{ + #[allow(unused_imports)] + use std::sync::atomic::AtomicU64 as _; +} +"#; + +fn main() { + let mut enable_const_thread_local = false; + let mut enable_target_has_atomic = false; + let mut enable_const_mutex_new = false; + let mut enable_as_fd = false; + let mut target_needs_atomic_u64_fallback = false; + + match AutoCfg::new() { + Ok(ac) => { + // These checks prefer to call only `probe_rustc_version` if that is + // enough to determine whether the feature is supported. This is + // because the `probe_expression` call involves a call to rustc, + // which the `probe_rustc_version` call avoids. + + // Const-initialized thread locals were stabilized in 1.59. + if ac.probe_rustc_version(1, 60) { + enable_const_thread_local = true; + } else if ac.probe_rustc_version(1, 59) { + // This compiler claims to be 1.59, but there are some nightly + // compilers that claim to be 1.59 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2021-12-06. + if ac.probe_expression(CONST_THREAD_LOCAL_PROBE) { + enable_const_thread_local = true; + } + } + + // The `target_has_atomic` cfg was stabilized in 1.60. + if ac.probe_rustc_version(1, 61) { + enable_target_has_atomic = true; + } else if ac.probe_rustc_version(1, 60) { + // This compiler claims to be 1.60, but there are some nightly + // compilers that claim to be 1.60 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2022-02-11. + if ac.probe_expression(TARGET_HAS_ATOMIC_PROBE) { + enable_target_has_atomic = true; + } + } + + // If we can't tell using `target_has_atomic`, tell if the target + // has `AtomicU64` by trying to use it. + if !enable_target_has_atomic && !ac.probe_expression(TARGET_ATOMIC_U64_PROBE) { + target_needs_atomic_u64_fallback = true; + } + + // The `Mutex::new` method was made const in 1.63. + if ac.probe_rustc_version(1, 64) { + enable_const_mutex_new = true; + } else if ac.probe_rustc_version(1, 63) { + // This compiler claims to be 1.63, but there are some nightly + // compilers that claim to be 1.63 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2022-06-20. + if ac.probe_expression(CONST_MUTEX_NEW_PROBE) { + enable_const_mutex_new = true; + } + } + + // The `AsFd` family of traits were made stable in 1.63. + if ac.probe_rustc_version(1, 64) { + enable_as_fd = true; + } else if ac.probe_rustc_version(1, 63) { + // This compiler claims to be 1.63, but there are some nightly + // compilers that claim to be 1.63 without supporting the + // feature. Explicitly probe to check if code using them + // compiles. + // + // The oldest nightly that supports the feature is 2022-06-16. + if ac.probe_expression(AS_FD_PROBE) { + enable_as_fd = true; + } + } + } + + Err(e) => { + // If we couldn't detect the compiler version and features, just + // print a warning. This isn't a fatal error: we can still build + // Tokio, we just can't enable cfgs automatically. + println!( + "cargo:warning=tokio: failed to detect compiler features: {}", + e + ); + } + } + + if !enable_const_thread_local { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_const_thread_local" + autocfg::emit("tokio_no_const_thread_local") + } + + if !enable_target_has_atomic { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_target_has_atomic" + autocfg::emit("tokio_no_target_has_atomic") + } + + if !enable_const_mutex_new { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_const_mutex_new" + autocfg::emit("tokio_no_const_mutex_new") + } + + if !enable_as_fd { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_as_fd" + autocfg::emit("tokio_no_as_fd"); + } + + if target_needs_atomic_u64_fallback { + // To disable this feature on compilers that support it, you can + // explicitly pass this flag with the following environment variable: + // + // RUSTFLAGS="--cfg tokio_no_atomic_u64" + autocfg::emit("tokio_no_atomic_u64") + } + + let target = ::std::env::var("TARGET").unwrap_or_default(); + + // We emit cfgs instead of using `target_family = "wasm"` that requires Rust 1.54. + // Note that these cfgs are unavailable in `Cargo.toml`. + if target.starts_with("wasm") { + autocfg::emit("tokio_wasm"); + if target.contains("wasi") { + autocfg::emit("tokio_wasi"); + } else { + autocfg::emit("tokio_wasm_not_wasi"); + } + } +} diff --git a/third_party/rust/tokio/docs/reactor-refactor.md b/third_party/rust/tokio/docs/reactor-refactor.md index 1c9ace15a8f8..77e64f4dfd7e 100644 --- a/third_party/rust/tokio/docs/reactor-refactor.md +++ b/third_party/rust/tokio/docs/reactor-refactor.md @@ -188,12 +188,12 @@ readiness, the driver's tick is packed into the atomic `usize`. The `ScheduledIo` readiness `AtomicUsize` is structured as: ``` -| reserved | generation | driver tick | readinesss | -|----------+------------+--------------+------------| -| 1 bit | 7 bits + 8 bits + 16 bits | +| shutdown | generation | driver tick | readiness | +|----------+------------+--------------+-----------| +| 1 bit | 7 bits + 8 bits + 16 bits | ``` -The `reserved` and `generation` components exist today. +The `shutdown` and `generation` components exist today. The `readiness()` function returns a `ReadyEvent` value. This value includes the `tick` component read with the resource's readiness value. When diff --git a/third_party/rust/tokio/external-types.toml b/third_party/rust/tokio/external-types.toml new file mode 100644 index 000000000000..a5bde8ed10c9 --- /dev/null +++ b/third_party/rust/tokio/external-types.toml @@ -0,0 +1,11 @@ +# This config file is for the `cargo-check-external-types` tool that is run in CI. + +# The following are types that are allowed to be exposed in Tokio's public API. +# The standard library is allowed by default. +allowed_external_types = [ + "bytes::buf::buf_impl::Buf", + "bytes::buf::buf_mut::BufMut", + + "tokio_macros::*", +] + diff --git a/third_party/rust/tokio/src/doc/mod.rs b/third_party/rust/tokio/src/doc/mod.rs index 3a949344900b..7992fc6bc6d0 100644 --- a/third_party/rust/tokio/src/doc/mod.rs +++ b/third_party/rust/tokio/src/doc/mod.rs @@ -21,4 +21,3 @@ pub enum NotDefinedHere {} pub mod os; -pub mod winapi; diff --git a/third_party/rust/tokio/src/doc/os.rs b/third_party/rust/tokio/src/doc/os.rs index 0ddf86959b86..9404d4491f27 100644 --- a/third_party/rust/tokio/src/doc/os.rs +++ b/third_party/rust/tokio/src/doc/os.rs @@ -13,7 +13,7 @@ pub mod windows { /// See [std::os::windows::io::AsRawHandle](https://doc.rust-lang.org/std/os/windows/io/trait.AsRawHandle.html) pub trait AsRawHandle { - /// See [std::os::windows::io::FromRawHandle::from_raw_handle](https://doc.rust-lang.org/std/os/windows/io/trait.AsRawHandle.html#tymethod.as_raw_handle) + /// See [std::os::windows::io::AsRawHandle::as_raw_handle](https://doc.rust-lang.org/std/os/windows/io/trait.AsRawHandle.html#tymethod.as_raw_handle) fn as_raw_handle(&self) -> RawHandle; } @@ -22,5 +22,44 @@ pub mod windows { /// See [std::os::windows::io::FromRawHandle::from_raw_handle](https://doc.rust-lang.org/std/os/windows/io/trait.FromRawHandle.html#tymethod.from_raw_handle) unsafe fn from_raw_handle(handle: RawHandle) -> Self; } + + /// See [std::os::windows::io::RawSocket](https://doc.rust-lang.org/std/os/windows/io/type.RawSocket.html) + pub type RawSocket = crate::doc::NotDefinedHere; + + /// See [std::os::windows::io::AsRawSocket](https://doc.rust-lang.org/std/os/windows/io/trait.AsRawSocket.html) + pub trait AsRawSocket { + /// See [std::os::windows::io::AsRawSocket::as_raw_socket](https://doc.rust-lang.org/std/os/windows/io/trait.AsRawSocket.html#tymethod.as_raw_socket) + fn as_raw_socket(&self) -> RawSocket; + } + + /// See [std::os::windows::io::FromRawSocket](https://doc.rust-lang.org/std/os/windows/io/trait.FromRawSocket.html) + pub trait FromRawSocket { + /// See [std::os::windows::io::FromRawSocket::from_raw_socket](https://doc.rust-lang.org/std/os/windows/io/trait.FromRawSocket.html#tymethod.from_raw_socket) + unsafe fn from_raw_socket(sock: RawSocket) -> Self; + } + + /// See [std::os::windows::io::IntoRawSocket](https://doc.rust-lang.org/std/os/windows/io/trait.IntoRawSocket.html) + pub trait IntoRawSocket { + /// See [std::os::windows::io::IntoRawSocket::into_raw_socket](https://doc.rust-lang.org/std/os/windows/io/trait.IntoRawSocket.html#tymethod.into_raw_socket) + fn into_raw_socket(self) -> RawSocket; + } + + /// See [std::os::windows::io::BorrowedHandle](https://doc.rust-lang.org/std/os/windows/io/struct.BorrowedHandle.html) + pub type BorrowedHandle<'handle> = crate::doc::NotDefinedHere; + + /// See [std::os::windows::io::AsHandle](https://doc.rust-lang.org/std/os/windows/io/trait.AsHandle.html) + pub trait AsHandle { + /// See [std::os::windows::io::AsHandle::as_handle](https://doc.rust-lang.org/std/os/windows/io/trait.AsHandle.html#tymethod.as_handle) + fn as_handle(&self) -> BorrowedHandle<'_>; + } + + /// See [std::os::windows::io::BorrowedSocket](https://doc.rust-lang.org/std/os/windows/io/struct.BorrowedSocket.html) + pub type BorrowedSocket<'socket> = crate::doc::NotDefinedHere; + + /// See [std::os::windows::io::AsSocket](https://doc.rust-lang.org/std/os/windows/io/trait.AsSocket.html) + pub trait AsSocket { + /// See [std::os::windows::io::AsSocket::as_socket](https://doc.rust-lang.org/std/os/windows/io/trait.AsSocket.html#tymethod.as_socket) + fn as_socket(&self) -> BorrowedSocket<'_>; + } } } diff --git a/third_party/rust/tokio/src/doc/winapi.rs b/third_party/rust/tokio/src/doc/winapi.rs deleted file mode 100644 index be68749e00d4..000000000000 --- a/third_party/rust/tokio/src/doc/winapi.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! See [winapi]. -//! -//! [winapi]: https://docs.rs/winapi - -/// See [winapi::shared](https://docs.rs/winapi/*/winapi/shared/index.html). -pub mod shared { - /// See [winapi::shared::winerror](https://docs.rs/winapi/*/winapi/shared/winerror/index.html). - #[allow(non_camel_case_types)] - pub mod winerror { - /// See [winapi::shared::winerror::ERROR_ACCESS_DENIED][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_ACCESS_DENIED.html - pub type ERROR_ACCESS_DENIED = crate::doc::NotDefinedHere; - - /// See [winapi::shared::winerror::ERROR_PIPE_BUSY][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_PIPE_BUSY.html - pub type ERROR_PIPE_BUSY = crate::doc::NotDefinedHere; - - /// See [winapi::shared::winerror::ERROR_MORE_DATA][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_MORE_DATA.html - pub type ERROR_MORE_DATA = crate::doc::NotDefinedHere; - } -} - -/// See [winapi::um](https://docs.rs/winapi/*/winapi/um/index.html). -pub mod um { - /// See [winapi::um::winbase](https://docs.rs/winapi/*/winapi/um/winbase/index.html). - #[allow(non_camel_case_types)] - pub mod winbase { - /// See [winapi::um::winbase::PIPE_TYPE_MESSAGE][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_MESSAGE.html - pub type PIPE_TYPE_MESSAGE = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::PIPE_TYPE_BYTE][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_BYTE.html - pub type PIPE_TYPE_BYTE = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::PIPE_CLIENT_END][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_CLIENT_END.html - pub type PIPE_CLIENT_END = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::PIPE_SERVER_END][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_SERVER_END.html - pub type PIPE_SERVER_END = crate::doc::NotDefinedHere; - - /// See [winapi::um::winbase::SECURITY_IDENTIFICATION][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.SECURITY_IDENTIFICATION.html - pub type SECURITY_IDENTIFICATION = crate::doc::NotDefinedHere; - } - - /// See [winapi::um::minwinbase](https://docs.rs/winapi/*/winapi/um/minwinbase/index.html). - #[allow(non_camel_case_types)] - pub mod minwinbase { - /// See [winapi::um::minwinbase::SECURITY_ATTRIBUTES][winapi] - /// - /// [winapi]: https://docs.rs/winapi/*/winapi/um/minwinbase/constant.SECURITY_ATTRIBUTES.html - pub type SECURITY_ATTRIBUTES = crate::doc::NotDefinedHere; - } -} diff --git a/third_party/rust/tokio/src/fs/file.rs b/third_party/rust/tokio/src/fs/file.rs index 2c38e8059f5f..74f91958d0df 100644 --- a/third_party/rust/tokio/src/fs/file.rs +++ b/third_party/rust/tokio/src/fs/file.rs @@ -399,6 +399,7 @@ impl File { /// # } /// ``` pub async fn try_clone(&self) -> io::Result { + self.inner.lock().await.complete_inflight().await; let std = self.std.clone(); let std_file = asyncify(move || std.try_clone()).await?; Ok(File::from_std(std_file)) @@ -498,6 +499,7 @@ impl AsyncRead for File { cx: &mut Context<'_>, dst: &mut ReadBuf<'_>, ) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); let me = self.get_mut(); let inner = me.inner.get_mut(); @@ -565,34 +567,36 @@ impl AsyncSeek for File { let me = self.get_mut(); let inner = me.inner.get_mut(); - loop { - match inner.state { - Busy(_) => panic!("must wait for poll_complete before calling start_seek"), - Idle(ref mut buf_cell) => { - let mut buf = buf_cell.take().unwrap(); + match inner.state { + Busy(_) => Err(io::Error::new( + io::ErrorKind::Other, + "other file operation is pending, call poll_complete before start_seek", + )), + Idle(ref mut buf_cell) => { + let mut buf = buf_cell.take().unwrap(); - // Factor in any unread data from the buf - if !buf.is_empty() { - let n = buf.discard_read(); + // Factor in any unread data from the buf + if !buf.is_empty() { + let n = buf.discard_read(); - if let SeekFrom::Current(ref mut offset) = pos { - *offset += n; - } + if let SeekFrom::Current(ref mut offset) = pos { + *offset += n; } - - let std = me.std.clone(); - - inner.state = Busy(spawn_blocking(move || { - let res = (&*std).seek(pos); - (Operation::Seek(res), buf) - })); - return Ok(()); } + + let std = me.std.clone(); + + inner.state = Busy(spawn_blocking(move || { + let res = (&*std).seek(pos); + (Operation::Seek(res), buf) + })); + Ok(()) } } } fn poll_complete(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); let inner = self.inner.get_mut(); loop { @@ -628,6 +632,7 @@ impl AsyncWrite for File { cx: &mut Context<'_>, src: &[u8], ) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); let me = self.get_mut(); let inner = me.inner.get_mut(); @@ -694,11 +699,13 @@ impl AsyncWrite for File { } fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); let inner = self.inner.get_mut(); inner.poll_flush(cx) } fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); self.poll_flush(cx) } } @@ -724,6 +731,15 @@ impl std::os::unix::io::AsRawFd for File { } } +#[cfg(all(unix, not(tokio_no_as_fd)))] +impl std::os::unix::io::AsFd for File { + fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> { + unsafe { + std::os::unix::io::BorrowedFd::borrow_raw(std::os::unix::io::AsRawFd::as_raw_fd(self)) + } + } +} + #[cfg(unix)] impl std::os::unix::io::FromRawFd for File { unsafe fn from_raw_fd(fd: std::os::unix::io::RawFd) -> Self { @@ -731,17 +747,32 @@ impl std::os::unix::io::FromRawFd for File { } } -#[cfg(windows)] -impl std::os::windows::io::AsRawHandle for File { - fn as_raw_handle(&self) -> std::os::windows::io::RawHandle { - self.std.as_raw_handle() - } -} +cfg_windows! { + use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle}; + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsHandle, BorrowedHandle}; -#[cfg(windows)] -impl std::os::windows::io::FromRawHandle for File { - unsafe fn from_raw_handle(handle: std::os::windows::io::RawHandle) -> Self { - StdFile::from_raw_handle(handle).into() + impl AsRawHandle for File { + fn as_raw_handle(&self) -> RawHandle { + self.std.as_raw_handle() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsHandle for File { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { + BorrowedHandle::borrow_raw( + AsRawHandle::as_raw_handle(self), + ) + } + } + } + + impl FromRawHandle for File { + unsafe fn from_raw_handle(handle: RawHandle) -> Self { + StdFile::from_raw_handle(handle).into() + } } } @@ -749,8 +780,18 @@ impl Inner { async fn complete_inflight(&mut self) { use crate::future::poll_fn; - if let Err(e) = poll_fn(|cx| Pin::new(&mut *self).poll_flush(cx)).await { - self.last_write_err = Some(e.kind()); + poll_fn(|cx| self.poll_complete_inflight(cx)).await + } + + fn poll_complete_inflight(&mut self, cx: &mut Context<'_>) -> Poll<()> { + ready!(crate::trace::trace_leaf(cx)); + match self.poll_flush(cx) { + Poll::Ready(Err(e)) => { + self.last_write_err = Some(e.kind()); + Poll::Ready(()) + } + Poll::Ready(Ok(())) => Poll::Ready(()), + Poll::Pending => Poll::Pending, } } diff --git a/third_party/rust/tokio/src/fs/file/tests.rs b/third_party/rust/tokio/src/fs/file/tests.rs index 18a4c0785998..7c61b3c4b31b 100644 --- a/third_party/rust/tokio/src/fs/file/tests.rs +++ b/third_party/rust/tokio/src/fs/file/tests.rs @@ -231,12 +231,12 @@ fn flush_while_idle() { #[cfg_attr(miri, ignore)] // takes a really long time with miri fn read_with_buffer_larger_than_max() { // Chunks - let chunk_a = 16 * 1024; + let chunk_a = crate::io::blocking::MAX_BUF; let chunk_b = chunk_a * 2; let chunk_c = chunk_a * 3; let chunk_d = chunk_a * 4; - assert_eq!(chunk_d / 1024, 64); + assert_eq!(chunk_d / 1024 / 1024, 8); let mut data = vec![]; for i in 0..(chunk_d - 1) { @@ -303,12 +303,12 @@ fn read_with_buffer_larger_than_max() { #[cfg_attr(miri, ignore)] // takes a really long time with miri fn write_with_buffer_larger_than_max() { // Chunks - let chunk_a = 16 * 1024; + let chunk_a = crate::io::blocking::MAX_BUF; let chunk_b = chunk_a * 2; let chunk_c = chunk_a * 3; let chunk_d = chunk_a * 4; - assert_eq!(chunk_d / 1024, 64); + assert_eq!(chunk_d / 1024 / 1024, 8); let mut data = vec![]; for i in 0..(chunk_d - 1) { @@ -955,3 +955,24 @@ fn partial_read_set_len_ok() { assert_eq!(n, FOO.len()); assert_eq!(&buf[..n], FOO); } + +#[test] +fn busy_file_seek_error() { + let mut file = MockFile::default(); + let mut seq = Sequence::new(); + file.expect_inner_write() + .once() + .in_sequence(&mut seq) + .returning(|_| Err(io::ErrorKind::Other.into())); + + let mut file = crate::io::BufReader::new(File::from_std(file)); + { + let mut t = task::spawn(file.write(HELLO)); + assert_ready_ok!(t.poll()); + } + + pool::run_one(); + + let mut t = task::spawn(file.seek(SeekFrom::Start(0))); + assert_ready_err!(t.poll()); +} diff --git a/third_party/rust/tokio/src/fs/mocks.rs b/third_party/rust/tokio/src/fs/mocks.rs index b1861726778c..aa01e24711ee 100644 --- a/third_party/rust/tokio/src/fs/mocks.rs +++ b/third_party/rust/tokio/src/fs/mocks.rs @@ -81,7 +81,7 @@ impl Write for &'_ MockFile { } } -thread_local! { +tokio_thread_local! { static QUEUE: RefCell>> = RefCell::new(VecDeque::new()) } diff --git a/third_party/rust/tokio/src/fs/mod.rs b/third_party/rust/tokio/src/fs/mod.rs index ca0264b36782..f6d9605fe94d 100644 --- a/third_party/rust/tokio/src/fs/mod.rs +++ b/third_party/rust/tokio/src/fs/mod.rs @@ -22,6 +22,24 @@ //! `std::io::ErrorKind::WouldBlock` if a *worker* thread can not be converted //! to a *backup* thread immediately. //! +//! **Warning**: These adapters may create a large number of temporary tasks, +//! especially when reading large files. When performing a lot of operations +//! in one batch, it may be significantly faster to use [`spawn_blocking`] +//! directly: +//! +//! ``` +//! use tokio::fs::File; +//! use std::io::{BufReader, BufRead}; +//! async fn count_lines(file: File) -> Result { +//! let file = file.into_std().await; +//! tokio::task::spawn_blocking(move || { +//! let line_count = BufReader::new(file).lines().count(); +//! Ok(line_count) +//! }).await? +//! } +//! ``` +//! +//! [`spawn_blocking`]: fn@crate::task::spawn_blocking //! [`AsyncRead`]: trait@crate::io::AsyncRead mod canonicalize; @@ -84,6 +102,9 @@ pub use self::write::write; mod copy; pub use self::copy::copy; +mod try_exists; +pub use self::try_exists::try_exists; + #[cfg(test)] mod mocks; @@ -94,9 +115,7 @@ feature! { pub use self::symlink::symlink; } -feature! { - #![windows] - +cfg_windows! { mod symlink_dir; pub use self::symlink_dir::symlink_dir; diff --git a/third_party/rust/tokio/src/fs/open_options.rs b/third_party/rust/tokio/src/fs/open_options.rs index f3b4654741ec..103510250b64 100644 --- a/third_party/rust/tokio/src/fs/open_options.rs +++ b/third_party/rust/tokio/src/fs/open_options.rs @@ -10,6 +10,11 @@ use mock_open_options::MockOpenOptions as StdOpenOptions; #[cfg(not(test))] use std::fs::OpenOptions as StdOpenOptions; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +#[cfg(windows)] +use std::os::windows::fs::OpenOptionsExt; + /// Options and flags which can be used to configure how a file is opened. /// /// This builder exposes the ability to configure how a [`File`] is opened and @@ -399,8 +404,6 @@ impl OpenOptions { feature! { #![unix] - use std::os::unix::fs::OpenOptionsExt; - impl OpenOptions { /// Sets the mode bits that a new file will be created with. /// @@ -464,11 +467,7 @@ feature! { } } -feature! { - #![windows] - - use std::os::windows::fs::OpenOptionsExt; - +cfg_windows! { impl OpenOptions { /// Overrides the `dwDesiredAccess` argument to the call to [`CreateFile`] /// with the specified value. @@ -542,7 +541,7 @@ feature! { /// # Examples /// /// ```no_run - /// use winapi::um::winbase::FILE_FLAG_DELETE_ON_CLOSE; + /// use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_DELETE_ON_CLOSE; /// use tokio::fs::OpenOptions; /// /// # #[tokio::main] @@ -581,7 +580,7 @@ feature! { /// # Examples /// /// ```no_run - /// use winapi::um::winnt::FILE_ATTRIBUTE_HIDDEN; + /// use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_HIDDEN; /// use tokio::fs::OpenOptions; /// /// # #[tokio::main] @@ -624,7 +623,7 @@ feature! { /// # Examples /// /// ```no_run - /// use winapi::um::winbase::SECURITY_IDENTIFICATION; + /// use windows_sys::Win32::Storage::FileSystem::SECURITY_IDENTIFICATION; /// use tokio::fs::OpenOptions; /// /// # #[tokio::main] diff --git a/third_party/rust/tokio/src/fs/open_options/mock_open_options.rs b/third_party/rust/tokio/src/fs/open_options/mock_open_options.rs index cbbda0ec2563..17b4a48640a5 100644 --- a/third_party/rust/tokio/src/fs/open_options/mock_open_options.rs +++ b/third_party/rust/tokio/src/fs/open_options/mock_open_options.rs @@ -1,3 +1,4 @@ +#![allow(unreachable_pub)] //! Mock version of std::fs::OpenOptions; use mockall::mock; diff --git a/third_party/rust/tokio/src/fs/read_dir.rs b/third_party/rust/tokio/src/fs/read_dir.rs index 281ea4cd756c..def735b3c023 100644 --- a/third_party/rust/tokio/src/fs/read_dir.rs +++ b/third_party/rust/tokio/src/fs/read_dir.rs @@ -1,5 +1,6 @@ use crate::fs::asyncify; +use std::collections::VecDeque; use std::ffi::OsString; use std::fs::{FileType, Metadata}; use std::future::Future; @@ -19,6 +20,8 @@ use crate::blocking::spawn_blocking; #[cfg(not(test))] use crate::blocking::JoinHandle; +const CHUNK_SIZE: usize = 32; + /// Returns a stream over the entries within a directory. /// /// This is an async version of [`std::fs::read_dir`](std::fs::read_dir) @@ -29,12 +32,17 @@ use crate::blocking::JoinHandle; /// [`spawn_blocking`]: crate::task::spawn_blocking pub async fn read_dir(path: impl AsRef) -> io::Result { let path = path.as_ref().to_owned(); - let std = asyncify(|| std::fs::read_dir(path)).await?; + asyncify(|| -> io::Result { + let mut std = std::fs::read_dir(path)?; + let mut buf = VecDeque::with_capacity(CHUNK_SIZE); + let remain = ReadDir::next_chunk(&mut buf, &mut std); - Ok(ReadDir(State::Idle(Some(std)))) + Ok(ReadDir(State::Idle(Some((buf, std, remain))))) + }) + .await } -/// Reads the the entries in a directory. +/// Reads the entries in a directory. /// /// This struct is returned from the [`read_dir`] function of this module and /// will yield instances of [`DirEntry`]. Through a [`DirEntry`] information @@ -58,8 +66,8 @@ pub struct ReadDir(State); #[derive(Debug)] enum State { - Idle(Option), - Pending(JoinHandle<(Option>, std::fs::ReadDir)>), + Idle(Option<(VecDeque>, std::fs::ReadDir, bool)>), + Pending(JoinHandle<(VecDeque>, std::fs::ReadDir, bool)>), } impl ReadDir { @@ -94,29 +102,58 @@ impl ReadDir { pub fn poll_next_entry(&mut self, cx: &mut Context<'_>) -> Poll>> { loop { match self.0 { - State::Idle(ref mut std) => { - let mut std = std.take().unwrap(); + State::Idle(ref mut data) => { + let (buf, _, ref remain) = data.as_mut().unwrap(); + + if let Some(ent) = buf.pop_front() { + return Poll::Ready(ent.map(Some)); + } else if !remain { + return Poll::Ready(Ok(None)); + } + + let (mut buf, mut std, _) = data.take().unwrap(); self.0 = State::Pending(spawn_blocking(move || { - let ret = std.next(); - (ret, std) + let remain = ReadDir::next_chunk(&mut buf, &mut std); + (buf, std, remain) })); } State::Pending(ref mut rx) => { - let (ret, std) = ready!(Pin::new(rx).poll(cx))?; - self.0 = State::Idle(Some(std)); - - let ret = match ret { - Some(Ok(std)) => Ok(Some(DirEntry(Arc::new(std)))), - Some(Err(e)) => Err(e), - None => Ok(None), - }; - - return Poll::Ready(ret); + self.0 = State::Idle(Some(ready!(Pin::new(rx).poll(cx))?)); } } } } + + fn next_chunk(buf: &mut VecDeque>, std: &mut std::fs::ReadDir) -> bool { + for _ in 0..CHUNK_SIZE { + let ret = match std.next() { + Some(ret) => ret, + None => return false, + }; + + let success = ret.is_ok(); + + buf.push_back(ret.map(|std| DirEntry { + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks", + target_os = "nto", + target_os = "vita", + )))] + file_type: std.file_type().ok(), + std: Arc::new(std), + })); + + if !success { + break; + } + } + + true + } } feature! { @@ -160,7 +197,18 @@ feature! { /// filesystem. Each entry can be inspected via methods to learn about the full /// path or possibly other metadata through per-platform extension traits. #[derive(Debug)] -pub struct DirEntry(Arc); +pub struct DirEntry { + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks", + target_os = "nto", + target_os = "vita", + )))] + file_type: Option, + std: Arc, +} impl DirEntry { /// Returns the full path to the file that this entry represents. @@ -193,7 +241,7 @@ impl DirEntry { /// /// The exact text, of course, depends on what files you have in `.`. pub fn path(&self) -> PathBuf { - self.0.path() + self.std.path() } /// Returns the bare file name of this directory entry without any other @@ -214,7 +262,7 @@ impl DirEntry { /// # } /// ``` pub fn file_name(&self) -> OsString { - self.0.file_name() + self.std.file_name() } /// Returns the metadata for the file that this entry points at. @@ -248,7 +296,7 @@ impl DirEntry { /// # } /// ``` pub async fn metadata(&self) -> io::Result { - let std = self.0.clone(); + let std = self.std.clone(); asyncify(move || std.metadata()).await } @@ -283,13 +331,25 @@ impl DirEntry { /// # } /// ``` pub async fn file_type(&self) -> io::Result { - let std = self.0.clone(); + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks", + target_os = "nto", + target_os = "vita", + )))] + if let Some(file_type) = self.file_type { + return Ok(file_type); + } + + let std = self.std.clone(); asyncify(move || std.file_type()).await } /// Returns a reference to the underlying `std::fs::DirEntry`. #[cfg(unix)] pub(super) fn as_inner(&self) -> &std::fs::DirEntry { - &self.0 + &self.std } } diff --git a/third_party/rust/tokio/src/fs/symlink_dir.rs b/third_party/rust/tokio/src/fs/symlink_dir.rs index 736e762b48d0..6753c25eb7bf 100644 --- a/third_party/rust/tokio/src/fs/symlink_dir.rs +++ b/third_party/rust/tokio/src/fs/symlink_dir.rs @@ -10,7 +10,7 @@ use std::path::Path; /// /// This is an async version of [`std::os::windows::fs::symlink_dir`][std] /// -/// [std]: std::os::windows::fs::symlink_dir +/// [std]: https://doc.rust-lang.org/std/os/windows/fs/fn.symlink_dir.html pub async fn symlink_dir(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { let src = src.as_ref().to_owned(); let dst = dst.as_ref().to_owned(); diff --git a/third_party/rust/tokio/src/fs/symlink_file.rs b/third_party/rust/tokio/src/fs/symlink_file.rs index 07d8e604192f..623352a1bda7 100644 --- a/third_party/rust/tokio/src/fs/symlink_file.rs +++ b/third_party/rust/tokio/src/fs/symlink_file.rs @@ -10,7 +10,7 @@ use std::path::Path; /// /// This is an async version of [`std::os::windows::fs::symlink_file`][std] /// -/// [std]: std::os::windows::fs::symlink_file +/// [std]: https://doc.rust-lang.org/std/os/windows/fs/fn.symlink_file.html pub async fn symlink_file(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { let src = src.as_ref().to_owned(); let dst = dst.as_ref().to_owned(); diff --git a/third_party/rust/tokio/src/fs/try_exists.rs b/third_party/rust/tokio/src/fs/try_exists.rs new file mode 100644 index 000000000000..069518bf9463 --- /dev/null +++ b/third_party/rust/tokio/src/fs/try_exists.rs @@ -0,0 +1,34 @@ +use crate::fs::asyncify; + +use std::io; +use std::path::Path; + +/// Returns `Ok(true)` if the path points at an existing entity. +/// +/// This function will traverse symbolic links to query information about the +/// destination file. In case of broken symbolic links this will return `Ok(false)`. +/// +/// This is the async equivalent of [`std::path::Path::try_exists`][std]. +/// +/// [std]: fn@std::path::Path::try_exists +/// +/// # Examples +/// +/// ```no_run +/// use tokio::fs; +/// +/// # async fn dox() -> std::io::Result<()> { +/// fs::try_exists("foo.txt").await?; +/// # Ok(()) +/// # } +/// ``` +pub async fn try_exists(path: impl AsRef) -> io::Result { + let path = path.as_ref().to_owned(); + // std's Path::try_exists is not available for current Rust min supported version. + // Current implementation is based on its internal implementation instead. + match asyncify(move || std::fs::metadata(path)).await { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} diff --git a/third_party/rust/tokio/src/future/block_on.rs b/third_party/rust/tokio/src/future/block_on.rs index 91f9cc00550c..2c2ab3736176 100644 --- a/third_party/rust/tokio/src/future/block_on.rs +++ b/third_party/rust/tokio/src/future/block_on.rs @@ -1,15 +1,22 @@ use std::future::Future; cfg_rt! { + #[track_caller] pub(crate) fn block_on(f: F) -> F::Output { - let mut e = crate::runtime::enter::enter(false); + let mut e = crate::runtime::context::try_enter_blocking_region().expect( + "Cannot block the current thread from within a runtime. This \ + happens because a function attempted to block the current \ + thread while the thread is being used to drive asynchronous \ + tasks." + ); e.block_on(f).unwrap() } } cfg_not_rt! { + #[track_caller] pub(crate) fn block_on(f: F) -> F::Output { - let mut park = crate::park::thread::CachedParkThread::new(); + let mut park = crate::runtime::park::CachedParkThread::new(); park.block_on(f).unwrap() } } diff --git a/third_party/rust/tokio/src/future/poll_fn.rs b/third_party/rust/tokio/src/future/poll_fn.rs index d82ce8961d1d..074d9438eb47 100644 --- a/third_party/rust/tokio/src/future/poll_fn.rs +++ b/third_party/rust/tokio/src/future/poll_fn.rs @@ -7,13 +7,23 @@ use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; +// This struct is intentionally `!Unpin` when `F` is `!Unpin`. This is to +// mitigate the issue where rust puts noalias on mutable references to the +// `PollFn` type if it is `Unpin`. If the closure has ownership of a future, +// then this "leaks" and the future is affected by noalias too, which we don't +// want. +// +// See this thread for more information: +// +// +// The fact that `PollFn` is not `Unpin` when it shouldn't be is tested in +// `tests/async_send_sync.rs`. + /// Future for the [`poll_fn`] function. pub struct PollFn { f: F, } -impl Unpin for PollFn {} - /// Creates a new future wrapping around a function returning [`Poll`]. pub fn poll_fn(f: F) -> PollFn where @@ -34,7 +44,17 @@ where { type Output = T; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - (&mut self.f)(cx) + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // Safety: We never construct a `Pin<&mut F>` anywhere, so accessing `f` + // mutably in an unpinned way is sound. + // + // This use of unsafe cannot be replaced with the pin-project macro + // because: + // * If we put `#[pin]` on the field, then it gives us a `Pin<&mut F>`, + // which we can't use to call the closure. + // * If we don't put `#[pin]` on the field, then it makes `PollFn` be + // unconditionally `Unpin`, which we also don't want. + let me = unsafe { Pin::into_inner_unchecked(self) }; + (me.f)(cx) } } diff --git a/third_party/rust/tokio/src/fuzz.rs b/third_party/rust/tokio/src/fuzz.rs new file mode 100644 index 000000000000..d89718f0e0ed --- /dev/null +++ b/third_party/rust/tokio/src/fuzz.rs @@ -0,0 +1 @@ +pub use crate::util::linked_list::tests::fuzz_linked_list; diff --git a/third_party/rust/tokio/src/io/async_fd.rs b/third_party/rust/tokio/src/io/async_fd.rs index 93f9cb458a7b..0cde5a407613 100644 --- a/third_party/rust/tokio/src/io/async_fd.rs +++ b/third_party/rust/tokio/src/io/async_fd.rs @@ -1,4 +1,6 @@ -use crate::io::driver::{Handle, Interest, ReadyEvent, Registration}; +use crate::io::{Interest, Ready}; +use crate::runtime::io::{ReadyEvent, Registration}; +use crate::runtime::scheduler; use mio::unix::SourceFd; use std::io; @@ -63,8 +65,8 @@ use std::{task::Context, task::Poll}; /// # Examples /// /// This example shows how to turn [`std::net::TcpStream`] asynchronous using -/// `AsyncFd`. It implements `read` as an async fn, and `AsyncWrite` as a trait -/// to show how to implement both approaches. +/// `AsyncFd`. It implements the read/write operations both as an `async fn` +/// and using the IO traits [`AsyncRead`] and [`AsyncWrite`]. /// /// ```no_run /// use futures::ready; @@ -72,7 +74,7 @@ use std::{task::Context, task::Poll}; /// use std::net::TcpStream; /// use std::pin::Pin; /// use std::task::{Context, Poll}; -/// use tokio::io::AsyncWrite; +/// use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; /// use tokio::io::unix::AsyncFd; /// /// pub struct AsyncTcpStream { @@ -97,6 +99,39 @@ use std::{task::Context, task::Poll}; /// } /// } /// } +/// +/// pub async fn write(&self, buf: &[u8]) -> io::Result { +/// loop { +/// let mut guard = self.inner.writable().await?; +/// +/// match guard.try_io(|inner| inner.get_ref().write(buf)) { +/// Ok(result) => return result, +/// Err(_would_block) => continue, +/// } +/// } +/// } +/// } +/// +/// impl AsyncRead for AsyncTcpStream { +/// fn poll_read( +/// self: Pin<&mut Self>, +/// cx: &mut Context<'_>, +/// buf: &mut ReadBuf<'_> +/// ) -> Poll> { +/// loop { +/// let mut guard = ready!(self.inner.poll_read_ready(cx))?; +/// +/// let unfilled = buf.initialize_unfilled(); +/// match guard.try_io(|inner| inner.get_ref().read(unfilled)) { +/// Ok(Ok(len)) => { +/// buf.advance(len); +/// return Poll::Ready(Ok(())); +/// }, +/// Ok(Err(err)) => return Poll::Ready(Err(err)), +/// Err(_would_block) => continue, +/// } +/// } +/// } /// } /// /// impl AsyncWrite for AsyncTcpStream { @@ -137,6 +172,8 @@ use std::{task::Context, task::Poll}; /// [`writable`]: method@Self::writable /// [`AsyncFdReadyGuard`]: struct@self::AsyncFdReadyGuard /// [`TcpStream::poll_read_ready`]: struct@crate::net::TcpStream +/// [`AsyncRead`]: trait@crate::io::AsyncRead +/// [`AsyncWrite`]: trait@crate::io::AsyncWrite pub struct AsyncFd { registration: Registration, inner: Option, @@ -164,35 +201,50 @@ pub struct AsyncFdReadyMutGuard<'a, T: AsRawFd> { event: Option, } -const ALL_INTEREST: Interest = Interest::READABLE.add(Interest::WRITABLE); - impl AsyncFd { - #[inline] /// Creates an AsyncFd backed by (and taking ownership of) an object /// implementing [`AsRawFd`]. The backing file descriptor is cached at the /// time of creation. /// + /// Only configures the [`Interest::READABLE`] and [`Interest::WRITABLE`] interests. For more + /// control, use [`AsyncFd::with_interest`]. + /// /// This method must be called in the context of a tokio runtime. + /// + /// # Panics + /// + /// This function panics if there is no current reactor set, or if the `rt` + /// feature flag is not enabled. + #[inline] + #[track_caller] pub fn new(inner: T) -> io::Result where T: AsRawFd, { - Self::with_interest(inner, ALL_INTEREST) + Self::with_interest(inner, Interest::READABLE | Interest::WRITABLE) } + /// Creates an AsyncFd backed by (and taking ownership of) an object + /// implementing [`AsRawFd`], with a specific [`Interest`]. The backing + /// file descriptor is cached at the time of creation. + /// + /// # Panics + /// + /// This function panics if there is no current reactor set, or if the `rt` + /// feature flag is not enabled. #[inline] - /// Creates new instance as `new` with additional ability to customize interest, - /// allowing to specify whether file descriptor will be polled for read, write or both. + #[track_caller] pub fn with_interest(inner: T, interest: Interest) -> io::Result where T: AsRawFd, { - Self::new_with_handle_and_interest(inner, Handle::current(), interest) + Self::new_with_handle_and_interest(inner, scheduler::Handle::current(), interest) } + #[track_caller] pub(crate) fn new_with_handle_and_interest( inner: T, - handle: Handle, + handle: scheduler::Handle, interest: Interest, ) -> io::Result { let fd = inner.as_raw_fd(); @@ -390,7 +442,96 @@ impl AsyncFd { .into() } - async fn readiness(&self, interest: Interest) -> io::Result> { + /// Waits for any of the requested ready states, returning a + /// [`AsyncFdReadyGuard`] that must be dropped to resume + /// polling for the requested ready states. + /// + /// The function may complete without the file descriptor being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// When an IO operation does return `io::ErrorKind::WouldBlock`, the readiness must be cleared. + /// When a combined interest is used, it is important to clear only the readiness + /// that is actually observed to block. For instance when the combined + /// interest `Interest::READABLE | Interest::WRITABLE` is used, and a read blocks, only + /// read readiness should be cleared using the [`AsyncFdReadyGuard::clear_ready_matching`] method: + /// `guard.clear_ready_matching(Ready::READABLE)`. + /// Also clearing the write readiness in this case would be incorrect. The [`AsyncFdReadyGuard::clear_ready`] + /// method clears all readiness flags. + /// + /// This method takes `&self`, so it is possible to call this method + /// concurrently with other methods on this struct. This method only + /// provides shared access to the inner IO resource when handling the + /// [`AsyncFdReadyGuard`]. + /// + /// # Examples + /// + /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without + /// splitting. + /// + /// ```no_run + /// use std::error::Error; + /// use std::io; + /// use std::io::{Read, Write}; + /// use std::net::TcpStream; + /// use tokio::io::unix::AsyncFd; + /// use tokio::io::{Interest, Ready}; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let stream = TcpStream::connect("127.0.0.1:8080")?; + /// stream.set_nonblocking(true)?; + /// let stream = AsyncFd::new(stream)?; + /// + /// loop { + /// let mut guard = stream + /// .ready(Interest::READABLE | Interest::WRITABLE) + /// .await?; + /// + /// if guard.ready().is_readable() { + /// let mut data = vec![0; 1024]; + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match stream.get_ref().read(&mut data) { + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a read has blocked, but a write might still succeed. + /// // clear only the read readiness. + /// guard.clear_ready_matching(Ready::READABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// if guard.ready().is_writable() { + /// // Try to write data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match stream.get_ref().write(b"hello world") { + /// Ok(n) => { + /// println!("write {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a write has blocked, but a read might still succeed. + /// // clear only the write readiness. + /// guard.clear_ready_matching(Ready::WRITABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// } + /// } + /// ``` + pub async fn ready(&self, interest: Interest) -> io::Result> { let event = self.registration.readiness(interest).await?; Ok(AsyncFdReadyGuard { @@ -399,7 +540,94 @@ impl AsyncFd { }) } - async fn readiness_mut( + /// Waits for any of the requested ready states, returning a + /// [`AsyncFdReadyMutGuard`] that must be dropped to resume + /// polling for the requested ready states. + /// + /// The function may complete without the file descriptor being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// When an IO operation does return `io::ErrorKind::WouldBlock`, the readiness must be cleared. + /// When a combined interest is used, it is important to clear only the readiness + /// that is actually observed to block. For instance when the combined + /// interest `Interest::READABLE | Interest::WRITABLE` is used, and a read blocks, only + /// read readiness should be cleared using the [`AsyncFdReadyMutGuard::clear_ready_matching`] method: + /// `guard.clear_ready_matching(Ready::READABLE)`. + /// Also clearing the write readiness in this case would be incorrect. + /// The [`AsyncFdReadyMutGuard::clear_ready`] method clears all readiness flags. + /// + /// This method takes `&mut self`, so it is possible to access the inner IO + /// resource mutably when handling the [`AsyncFdReadyMutGuard`]. + /// + /// # Examples + /// + /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without + /// splitting. + /// + /// ```no_run + /// use std::error::Error; + /// use std::io; + /// use std::io::{Read, Write}; + /// use std::net::TcpStream; + /// use tokio::io::unix::AsyncFd; + /// use tokio::io::{Interest, Ready}; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let stream = TcpStream::connect("127.0.0.1:8080")?; + /// stream.set_nonblocking(true)?; + /// let mut stream = AsyncFd::new(stream)?; + /// + /// loop { + /// let mut guard = stream + /// .ready_mut(Interest::READABLE | Interest::WRITABLE) + /// .await?; + /// + /// if guard.ready().is_readable() { + /// let mut data = vec![0; 1024]; + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match guard.get_inner_mut().read(&mut data) { + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a read has blocked, but a write might still succeed. + /// // clear only the read readiness. + /// guard.clear_ready_matching(Ready::READABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// if guard.ready().is_writable() { + /// // Try to write data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match guard.get_inner_mut().write(b"hello world") { + /// Ok(n) => { + /// println!("write {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a write has blocked, but a read might still succeed. + /// // clear only the write readiness. + /// guard.clear_ready_matching(Ready::WRITABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// } + /// } + /// ``` + pub async fn ready_mut( &mut self, interest: Interest, ) -> io::Result> { @@ -421,7 +649,7 @@ impl AsyncFd { /// [`AsyncFdReadyGuard`]. #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering. pub async fn readable<'a>(&'a self) -> io::Result> { - self.readiness(Interest::READABLE).await + self.ready(Interest::READABLE).await } /// Waits for the file descriptor to become readable, returning a @@ -432,7 +660,7 @@ impl AsyncFd { /// resource mutably when handling the [`AsyncFdReadyMutGuard`]. #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering. pub async fn readable_mut<'a>(&'a mut self) -> io::Result> { - self.readiness_mut(Interest::READABLE).await + self.ready_mut(Interest::READABLE).await } /// Waits for the file descriptor to become writable, returning a @@ -445,7 +673,7 @@ impl AsyncFd { /// [`AsyncFdReadyGuard`]. #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering. pub async fn writable<'a>(&'a self) -> io::Result> { - self.readiness(Interest::WRITABLE).await + self.ready(Interest::WRITABLE).await } /// Waits for the file descriptor to become writable, returning a @@ -456,7 +684,110 @@ impl AsyncFd { /// resource mutably when handling the [`AsyncFdReadyMutGuard`]. #[allow(clippy::needless_lifetimes)] // The lifetime improves rustdoc rendering. pub async fn writable_mut<'a>(&'a mut self) -> io::Result> { - self.readiness_mut(Interest::WRITABLE).await + self.ready_mut(Interest::WRITABLE).await + } + + /// Reads or writes from the file descriptor using a user-provided IO operation. + /// + /// The `async_io` method is a convenience utility that waits for the file + /// descriptor to become ready, and then executes the provided IO operation. + /// Since file descriptors may be marked ready spuriously, the closure will + /// be called repeatedly until it returns something other than a + /// [`WouldBlock`] error. This is done using the following loop: + /// + /// ```no_run + /// # use std::io::{self, Result}; + /// # struct Dox { inner: T } + /// # impl Dox { + /// # async fn writable(&self) -> Result<&Self> { + /// # Ok(self) + /// # } + /// # fn try_io(&self, _: impl FnMut(&T) -> Result) -> Result> { + /// # panic!() + /// # } + /// async fn async_io(&self, mut f: impl FnMut(&T) -> io::Result) -> io::Result { + /// loop { + /// // or `readable` if called with the read interest. + /// let guard = self.writable().await?; + /// + /// match guard.try_io(&mut f) { + /// Ok(result) => return result, + /// Err(_would_block) => continue, + /// } + /// } + /// } + /// # } + /// ``` + /// + /// The closure should only return a [`WouldBlock`] error if it has performed + /// an IO operation on the file descriptor that failed due to the file descriptor not being + /// ready. Returning a [`WouldBlock`] error in any other situation will + /// incorrectly clear the readiness flag, which can cause the file descriptor to + /// behave incorrectly. + /// + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio [`AsyncFd`] type, as this will mess with the + /// readiness flag and can cause the file descriptor to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// + /// # Examples + /// + /// This example sends some bytes on the inner [`std::net::UdpSocket`]. The `async_io` + /// method waits for readiness, and retries if the send operation does block. This example + /// is equivalent to the one given for [`try_io`]. + /// + /// ```no_run + /// use tokio::io::{Interest, unix::AsyncFd}; + /// + /// use std::io; + /// use std::net::UdpSocket; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// let socket = UdpSocket::bind("0.0.0.0:8080")?; + /// socket.set_nonblocking(true)?; + /// let async_fd = AsyncFd::new(socket)?; + /// + /// let written = async_fd + /// .async_io(Interest::WRITABLE, |inner| inner.send(&[1, 2])) + /// .await?; + /// + /// println!("wrote {written} bytes"); + /// + /// Ok(()) + /// } + /// ``` + /// + /// [`try_io`]: AsyncFdReadyGuard::try_io + /// [`WouldBlock`]: std::io::ErrorKind::WouldBlock + pub async fn async_io( + &self, + interest: Interest, + mut f: impl FnMut(&T) -> io::Result, + ) -> io::Result { + self.registration + .async_io(interest, || f(self.get_ref())) + .await + } + + /// Reads or writes from the file descriptor using a user-provided IO operation. + /// + /// The behavior is the same as [`async_io`], except that the closure can mutate the inner + /// value of the [`AsyncFd`]. + /// + /// [`async_io`]: AsyncFd::async_io + pub async fn async_io_mut( + &mut self, + interest: Interest, + mut f: impl FnMut(&mut T) -> io::Result, + ) -> io::Result { + self.registration + .async_io(interest, || f(self.inner.as_mut().unwrap())) + .await } } @@ -466,6 +797,13 @@ impl AsRawFd for AsyncFd { } } +#[cfg(not(tokio_no_as_fd))] +impl std::os::unix::io::AsFd for AsyncFd { + fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> { + unsafe { std::os::unix::io::BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} + impl std::fmt::Debug for AsyncFd { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("AsyncFd") @@ -481,22 +819,117 @@ impl Drop for AsyncFd { } impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> { - /// Indicates to tokio that the file descriptor is no longer ready. The - /// internal readiness flag will be cleared, and tokio will wait for the + /// Indicates to tokio that the file descriptor is no longer ready. All + /// internal readiness flags will be cleared, and tokio will wait for the /// next edge-triggered readiness notification from the OS. /// + /// This function is commonly used with guards returned by [`AsyncFd::readable`] and + /// [`AsyncFd::writable`]. + /// /// It is critical that this function not be called unless your code /// _actually observes_ that the file descriptor is _not_ ready. Do not call /// it simply because, for example, a read succeeded; it should be called /// when a read is observed to block. - /// - /// [`drop`]: method@std::mem::drop pub fn clear_ready(&mut self) { if let Some(event) = self.event.take() { self.async_fd.registration.clear_readiness(event); } } + /// Indicates to tokio that the file descriptor no longer has a specific readiness. + /// The internal readiness flag will be cleared, and tokio will wait for the + /// next edge-triggered readiness notification from the OS. + /// + /// This function is useful in combination with the [`AsyncFd::ready`] method when a + /// combined interest like `Interest::READABLE | Interest::WRITABLE` is used. + /// + /// It is critical that this function not be called unless your code + /// _actually observes_ that the file descriptor is _not_ ready for the provided `Ready`. + /// Do not call it simply because, for example, a read succeeded; it should be called + /// when a read is observed to block. Only clear the specific readiness that is observed to + /// block. For example when a read blocks when using a combined interest, + /// only clear `Ready::READABLE`. + /// + /// # Examples + /// + /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without + /// splitting. + /// + /// ```no_run + /// use std::error::Error; + /// use std::io; + /// use std::io::{Read, Write}; + /// use std::net::TcpStream; + /// use tokio::io::unix::AsyncFd; + /// use tokio::io::{Interest, Ready}; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let stream = TcpStream::connect("127.0.0.1:8080")?; + /// stream.set_nonblocking(true)?; + /// let stream = AsyncFd::new(stream)?; + /// + /// loop { + /// let mut guard = stream + /// .ready(Interest::READABLE | Interest::WRITABLE) + /// .await?; + /// + /// if guard.ready().is_readable() { + /// let mut data = vec![0; 1024]; + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match stream.get_ref().read(&mut data) { + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a read has blocked, but a write might still succeed. + /// // clear only the read readiness. + /// guard.clear_ready_matching(Ready::READABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// if guard.ready().is_writable() { + /// // Try to write data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match stream.get_ref().write(b"hello world") { + /// Ok(n) => { + /// println!("write {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a write has blocked, but a read might still succeed. + /// // clear only the write readiness. + /// guard.clear_ready_matching(Ready::WRITABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// } + /// } + /// ``` + pub fn clear_ready_matching(&mut self, ready: Ready) { + if let Some(mut event) = self.event.take() { + self.async_fd + .registration + .clear_readiness(event.with_ready(ready)); + + // the event is no longer ready for the readiness that was just cleared + event.ready = event.ready - ready; + + if !event.ready.is_empty() { + self.event = Some(event); + } + } + } + /// This method should be invoked when you intentionally want to keep the /// ready flag asserted. /// @@ -506,6 +939,20 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> { // no-op } + /// Get the [`Ready`] value associated with this guard. + /// + /// This method will return the empty readiness state if + /// [`AsyncFdReadyGuard::clear_ready`] has been called on + /// the guard. + /// + /// [`Ready`]: crate::io::Ready + pub fn ready(&self) -> Ready { + match &self.event { + Some(event) => event.ready, + None => Ready::EMPTY, + } + } + /// Performs the provided IO operation. /// /// If `f` returns a [`WouldBlock`] error, the readiness state associated @@ -521,6 +968,43 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> { /// `AsyncFdReadyGuard` no longer expresses the readiness state that was queried to /// create this `AsyncFdReadyGuard`. /// + /// # Examples + /// + /// This example sends some bytes to the inner [`std::net::UdpSocket`]. Waiting + /// for write-readiness and retrying when the send operation does block are explicit. + /// This example can be written more succinctly using [`AsyncFd::async_io`]. + /// + /// ```no_run + /// use tokio::io::unix::AsyncFd; + /// + /// use std::io; + /// use std::net::UdpSocket; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// let socket = UdpSocket::bind("0.0.0.0:8080")?; + /// socket.set_nonblocking(true)?; + /// let async_fd = AsyncFd::new(socket)?; + /// + /// let written = loop { + /// let mut guard = async_fd.writable().await?; + /// match guard.try_io(|inner| inner.get_ref().send(&[1, 2])) { + /// Ok(result) => { + /// break result?; + /// } + /// Err(_would_block) => { + /// // try_io already cleared the file descriptor's readiness state + /// continue; + /// } + /// } + /// }; + /// + /// println!("wrote {written} bytes"); + /// + /// Ok(()) + /// } + /// ``` + /// /// [`WouldBlock`]: std::io::ErrorKind::WouldBlock // Alias for old name in 0.x #[cfg_attr(docsrs, doc(alias = "with_io"))] @@ -554,22 +1038,117 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyGuard<'a, Inner> { } impl<'a, Inner: AsRawFd> AsyncFdReadyMutGuard<'a, Inner> { - /// Indicates to tokio that the file descriptor is no longer ready. The - /// internal readiness flag will be cleared, and tokio will wait for the + /// Indicates to tokio that the file descriptor is no longer ready. All + /// internal readiness flags will be cleared, and tokio will wait for the /// next edge-triggered readiness notification from the OS. /// + /// This function is commonly used with guards returned by [`AsyncFd::readable_mut`] and + /// [`AsyncFd::writable_mut`]. + /// /// It is critical that this function not be called unless your code /// _actually observes_ that the file descriptor is _not_ ready. Do not call /// it simply because, for example, a read succeeded; it should be called /// when a read is observed to block. - /// - /// [`drop`]: method@std::mem::drop pub fn clear_ready(&mut self) { if let Some(event) = self.event.take() { self.async_fd.registration.clear_readiness(event); } } + /// Indicates to tokio that the file descriptor no longer has a specific readiness. + /// The internal readiness flag will be cleared, and tokio will wait for the + /// next edge-triggered readiness notification from the OS. + /// + /// This function is useful in combination with the [`AsyncFd::ready_mut`] method when a + /// combined interest like `Interest::READABLE | Interest::WRITABLE` is used. + /// + /// It is critical that this function not be called unless your code + /// _actually observes_ that the file descriptor is _not_ ready for the provided `Ready`. + /// Do not call it simply because, for example, a read succeeded; it should be called + /// when a read is observed to block. Only clear the specific readiness that is observed to + /// block. For example when a read blocks when using a combined interest, + /// only clear `Ready::READABLE`. + /// + /// # Examples + /// + /// Concurrently read and write to a [`std::net::TcpStream`] on the same task without + /// splitting. + /// + /// ```no_run + /// use std::error::Error; + /// use std::io; + /// use std::io::{Read, Write}; + /// use std::net::TcpStream; + /// use tokio::io::unix::AsyncFd; + /// use tokio::io::{Interest, Ready}; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let stream = TcpStream::connect("127.0.0.1:8080")?; + /// stream.set_nonblocking(true)?; + /// let mut stream = AsyncFd::new(stream)?; + /// + /// loop { + /// let mut guard = stream + /// .ready_mut(Interest::READABLE | Interest::WRITABLE) + /// .await?; + /// + /// if guard.ready().is_readable() { + /// let mut data = vec![0; 1024]; + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match guard.get_inner_mut().read(&mut data) { + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a read has blocked, but a write might still succeed. + /// // clear only the read readiness. + /// guard.clear_ready_matching(Ready::READABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// if guard.ready().is_writable() { + /// // Try to write data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match guard.get_inner_mut().write(b"hello world") { + /// Ok(n) => { + /// println!("write {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// // a write has blocked, but a read might still succeed. + /// // clear only the write readiness. + /// guard.clear_ready_matching(Ready::WRITABLE); + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// } + /// } + /// ``` + pub fn clear_ready_matching(&mut self, ready: Ready) { + if let Some(mut event) = self.event.take() { + self.async_fd + .registration + .clear_readiness(event.with_ready(ready)); + + // the event is no longer ready for the readiness that was just cleared + event.ready = event.ready - ready; + + if !event.ready.is_empty() { + self.event = Some(event); + } + } + } + /// This method should be invoked when you intentionally want to keep the /// ready flag asserted. /// @@ -579,6 +1158,20 @@ impl<'a, Inner: AsRawFd> AsyncFdReadyMutGuard<'a, Inner> { // no-op } + /// Get the [`Ready`] value associated with this guard. + /// + /// This method will return the empty readiness state if + /// [`AsyncFdReadyGuard::clear_ready`] has been called on + /// the guard. + /// + /// [`Ready`]: super::Ready + pub fn ready(&self) -> Ready { + match &self.event { + Some(event) => event.ready, + None => Ready::EMPTY, + } + } + /// Performs the provided IO operation. /// /// If `f` returns a [`WouldBlock`] error, the readiness state associated diff --git a/third_party/rust/tokio/src/io/blocking.rs b/third_party/rust/tokio/src/io/blocking.rs index 1d79ee7a2798..416573e9732e 100644 --- a/third_party/rust/tokio/src/io/blocking.rs +++ b/third_party/rust/tokio/src/io/blocking.rs @@ -26,7 +26,7 @@ pub(crate) struct Buf { pos: usize, } -pub(crate) const MAX_BUF: usize = 16 * 1024; +pub(crate) const MAX_BUF: usize = 2 * 1024 * 1024; #[derive(Debug)] enum State { @@ -34,8 +34,9 @@ enum State { Busy(sys::Blocking<(io::Result, Buf, T)>), } -cfg_io_std! { +cfg_io_blocking! { impl Blocking { + #[cfg_attr(feature = "fs", allow(dead_code))] pub(crate) fn new(inner: T) -> Blocking { Blocking { inner: Some(inner), diff --git a/third_party/rust/tokio/src/io/bsd/poll_aio.rs b/third_party/rust/tokio/src/io/bsd/poll_aio.rs index f1ac4b2d77a4..6ac9e2880e1b 100644 --- a/third_party/rust/tokio/src/io/bsd/poll_aio.rs +++ b/third_party/rust/tokio/src/io/bsd/poll_aio.rs @@ -1,6 +1,8 @@ //! Use POSIX AIO futures with Tokio. -use crate::io::driver::{Handle, Interest, ReadyEvent, Registration}; +use crate::io::interest::Interest; +use crate::runtime::io::{ReadyEvent, Registration}; +use crate::runtime::scheduler; use mio::event::Source; use mio::Registry; use mio::Token; @@ -117,7 +119,7 @@ impl Aio { fn new_with_interest(io: E, interest: Interest) -> io::Result { let mut io = MioSource(io); - let handle = Handle::current(); + let handle = scheduler::Handle::current(); let registration = Registration::new_with_interest_and_handle(&mut io, interest, handle)?; Ok(Self { io, registration }) } diff --git a/third_party/rust/tokio/src/io/driver/platform.rs b/third_party/rust/tokio/src/io/driver/platform.rs deleted file mode 100644 index 6b27988ce61a..000000000000 --- a/third_party/rust/tokio/src/io/driver/platform.rs +++ /dev/null @@ -1,44 +0,0 @@ -pub(crate) use self::sys::*; - -#[cfg(unix)] -mod sys { - use mio::unix::UnixReady; - use mio::Ready; - - pub(crate) fn hup() -> Ready { - UnixReady::hup().into() - } - - pub(crate) fn is_hup(ready: Ready) -> bool { - UnixReady::from(ready).is_hup() - } - - pub(crate) fn error() -> Ready { - UnixReady::error().into() - } - - pub(crate) fn is_error(ready: Ready) -> bool { - UnixReady::from(ready).is_error() - } -} - -#[cfg(windows)] -mod sys { - use mio::Ready; - - pub(crate) fn hup() -> Ready { - Ready::empty() - } - - pub(crate) fn is_hup(_: Ready) -> bool { - false - } - - pub(crate) fn error() -> Ready { - Ready::empty() - } - - pub(crate) fn is_error(_: Ready) -> bool { - false - } -} diff --git a/third_party/rust/tokio/src/io/driver/interest.rs b/third_party/rust/tokio/src/io/interest.rs similarity index 76% rename from third_party/rust/tokio/src/io/driver/interest.rs rename to third_party/rust/tokio/src/io/interest.rs index d6b46dfb7c6e..3a39cf761b74 100644 --- a/third_party/rust/tokio/src/io/driver/interest.rs +++ b/third_party/rust/tokio/src/io/interest.rs @@ -1,6 +1,6 @@ #![cfg_attr(not(feature = "net"), allow(dead_code, unreachable_pub))] -use crate::io::driver::Ready; +use crate::io::ready::Ready; use std::fmt; use std::ops; @@ -44,6 +44,11 @@ impl Interest { /// Writable interest includes write-closed events. pub const WRITABLE: Interest = Interest(mio::Interest::WRITABLE); + /// Returns a `Interest` set representing priority completion interests. + #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))] + pub const PRIORITY: Interest = Interest(mio::Interest::PRIORITY); + /// Returns true if the value includes readable interest. /// /// # Examples @@ -78,6 +83,25 @@ impl Interest { self.0.is_writable() } + /// Returns true if the value includes priority interest. + /// + /// # Examples + /// + /// ``` + /// use tokio::io::Interest; + /// + /// assert!(!Interest::READABLE.is_priority()); + /// assert!(Interest::PRIORITY.is_priority()); + /// + /// let both = Interest::READABLE | Interest::PRIORITY; + /// assert!(both.is_priority()); + /// ``` + #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))] + pub const fn is_priority(self) -> bool { + self.0.is_priority() + } + /// Add together two `Interest` values. /// /// This function works from a `const` context. @@ -100,10 +124,12 @@ impl Interest { self.0 } - pub(super) fn mask(self) -> Ready { + pub(crate) fn mask(self) -> Ready { match self { Interest::READABLE => Ready::READABLE | Ready::READ_CLOSED, Interest::WRITABLE => Ready::WRITABLE | Ready::WRITE_CLOSED, + #[cfg(any(target_os = "linux", target_os = "android"))] + Interest::PRIORITY => Ready::PRIORITY | Ready::READ_CLOSED, _ => Ready::EMPTY, } } diff --git a/third_party/rust/tokio/src/io/mod.rs b/third_party/rust/tokio/src/io/mod.rs index cfdda61f694d..666d69cb4385 100644 --- a/third_party/rust/tokio/src/io/mod.rs +++ b/third_party/rust/tokio/src/io/mod.rs @@ -1,5 +1,3 @@ -#![cfg_attr(loom, allow(dead_code, unreachable_pub))] - //! Traits, helpers, and type definitions for asynchronous I/O functionality. //! //! This module is the asynchronous version of `std::io`. Primarily, it @@ -132,19 +130,23 @@ //! other words, these types must never block the thread, and instead the //! current task is notified when the I/O resource is ready. //! -//! ## Conversion to and from Sink/Stream +//! ## Conversion to and from Stream/Sink //! -//! It is often convenient to encapsulate the reading and writing of -//! bytes and instead work with a [`Sink`] or [`Stream`] of some data -//! type that is encoded as bytes and/or decoded from bytes. Tokio -//! provides some utility traits in the [tokio-util] crate that -//! abstract the asynchronous buffering that is required and allows -//! you to write [`Encoder`] and [`Decoder`] functions working with a -//! buffer of bytes, and then use that ["codec"] to transform anything -//! that implements [`AsyncRead`] and [`AsyncWrite`] into a `Sink`/`Stream` of -//! your structured data. +//! It is often convenient to encapsulate the reading and writing of bytes in a +//! [`Stream`] or [`Sink`] of data. //! -//! [tokio-util]: https://docs.rs/tokio-util/0.6/tokio_util/codec/index.html +//! Tokio provides simple wrappers for converting [`AsyncRead`] to [`Stream`] +//! and vice-versa in the [tokio-util] crate, see [`ReaderStream`] and +//! [`StreamReader`]. +//! +//! There are also utility traits that abstract the asynchronous buffering +//! necessary to write your own adaptors for encoding and decoding bytes to/from +//! your structured data, allowing to transform something that implements +//! [`AsyncRead`]/[`AsyncWrite`] into a [`Stream`]/[`Sink`], see [`Decoder`] and +//! [`Encoder`] in the [tokio-util::codec] module. +//! +//! [tokio-util]: https://docs.rs/tokio-util +//! [tokio-util::codec]: https://docs.rs/tokio-util/latest/tokio_util/codec/index.html //! //! # Standard input and output //! @@ -169,9 +171,11 @@ //! [`AsyncWrite`]: trait@AsyncWrite //! [`AsyncReadExt`]: trait@AsyncReadExt //! [`AsyncWriteExt`]: trait@AsyncWriteExt -//! ["codec"]: https://docs.rs/tokio-util/0.6/tokio_util/codec/index.html -//! [`Encoder`]: https://docs.rs/tokio-util/0.6/tokio_util/codec/trait.Encoder.html -//! [`Decoder`]: https://docs.rs/tokio-util/0.6/tokio_util/codec/trait.Decoder.html +//! ["codec"]: https://docs.rs/tokio-util/latest/tokio_util/codec/index.html +//! [`Encoder`]: https://docs.rs/tokio-util/latest/tokio_util/codec/trait.Encoder.html +//! [`Decoder`]: https://docs.rs/tokio-util/latest/tokio_util/codec/trait.Decoder.html +//! [`ReaderStream`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.ReaderStream.html +//! [`StreamReader`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.StreamReader.html //! [`Error`]: struct@Error //! [`ErrorKind`]: enum@ErrorKind //! [`Result`]: type@Result @@ -180,6 +184,12 @@ //! [`Sink`]: https://docs.rs/futures/0.3/futures/sink/trait.Sink.html //! [`Stream`]: https://docs.rs/futures/0.3/futures/stream/trait.Stream.html //! [`Write`]: std::io::Write + +#![cfg_attr( + not(all(feature = "rt", feature = "net")), + allow(dead_code, unused_imports) +)] + cfg_io_blocking! { pub(crate) mod blocking; } @@ -205,15 +215,19 @@ pub use self::read_buf::ReadBuf; pub use std::io::{Error, ErrorKind, Result, SeekFrom}; cfg_io_driver_impl! { - pub(crate) mod driver; + pub(crate) mod interest; + pub(crate) mod ready; cfg_net! { - pub use driver::{Interest, Ready}; + pub use interest::Interest; + pub use ready::Ready; } + #[cfg_attr(tokio_wasi, allow(unused_imports))] mod poll_evented; #[cfg(not(loom))] + #[cfg_attr(tokio_wasi, allow(unused_imports))] pub(crate) use poll_evented::PollEvented; } diff --git a/third_party/rust/tokio/src/io/poll_evented.rs b/third_party/rust/tokio/src/io/poll_evented.rs index ce4c1426acc2..875ad7e0d0a1 100644 --- a/third_party/rust/tokio/src/io/poll_evented.rs +++ b/third_party/rust/tokio/src/io/poll_evented.rs @@ -1,4 +1,6 @@ -use crate::io::driver::{Handle, Interest, Registration}; +use crate::io::interest::Interest; +use crate::runtime::io::Registration; +use crate::runtime::scheduler; use mio::event::Source; use std::fmt; @@ -11,7 +13,7 @@ cfg_io_driver! { /// [`std::io::Write`] traits with the reactor that drives it. /// /// `PollEvented` uses [`Registration`] internally to take a type that - /// implements [`mio::event::Source`] as well as [`std::io::Read`] and or + /// implements [`mio::event::Source`] as well as [`std::io::Read`] and/or /// [`std::io::Write`] and associate it with a reactor that will drive it. /// /// Once the [`mio::event::Source`] type is wrapped by `PollEvented`, it can be @@ -41,12 +43,12 @@ cfg_io_driver! { /// [`poll_read_ready`] again will also indicate read readiness. /// /// When the operation is attempted and is unable to succeed due to the I/O - /// resource not being ready, the caller must call `clear_readiness`. + /// resource not being ready, the caller must call [`clear_readiness`]. /// This clears the readiness state until a new readiness event is received. /// /// This allows the caller to implement additional functions. For example, /// [`TcpListener`] implements poll_accept by using [`poll_read_ready`] and - /// `clear_read_ready`. + /// [`clear_readiness`]. /// /// ## Platform-specific events /// @@ -57,6 +59,7 @@ cfg_io_driver! { /// [`AsyncRead`]: crate::io::AsyncRead /// [`AsyncWrite`]: crate::io::AsyncWrite /// [`TcpListener`]: crate::net::TcpListener + /// [`clear_readiness`]: Registration::clear_readiness /// [`poll_read_ready`]: Registration::poll_read_ready /// [`poll_write_ready`]: Registration::poll_write_ready pub(crate) struct PollEvented { @@ -70,6 +73,9 @@ cfg_io_driver! { impl PollEvented { /// Creates a new `PollEvented` associated with the default reactor. /// + /// The returned `PollEvented` has readable and writable interests. For more control, use + /// [`Self::new_with_interest`]. + /// /// # Panics /// /// This function panics if thread-local runtime is not set. @@ -77,6 +83,7 @@ impl PollEvented { /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] #[cfg_attr(feature = "signal", allow(unused))] pub(crate) fn new(io: E) -> io::Result { PollEvented::new_with_interest(io, Interest::READABLE | Interest::WRITABLE) @@ -97,15 +104,17 @@ impl PollEvented { /// a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) /// function. + #[track_caller] #[cfg_attr(feature = "signal", allow(unused))] pub(crate) fn new_with_interest(io: E, interest: Interest) -> io::Result { - Self::new_with_interest_and_handle(io, interest, Handle::current()) + Self::new_with_interest_and_handle(io, interest, scheduler::Handle::current()) } + #[track_caller] pub(crate) fn new_with_interest_and_handle( mut io: E, interest: Interest, - handle: Handle, + handle: scheduler::Handle, ) -> io::Result { let registration = Registration::new_with_interest_and_handle(&mut io, interest, handle)?; Ok(Self { @@ -115,11 +124,7 @@ impl PollEvented { } /// Returns a reference to the registration. - #[cfg(any( - feature = "net", - all(unix, feature = "process"), - all(unix, feature = "signal"), - ))] + #[cfg(any(feature = "net"))] pub(crate) fn registration(&self) -> &Registration { &self.registration } @@ -134,7 +139,7 @@ impl PollEvented { } feature! { - #![any(feature = "net", feature = "process")] + #![any(feature = "net", all(unix, feature = "process"))] use crate::io::ReadBuf; use std::task::{Context, Poll}; @@ -151,16 +156,32 @@ feature! { { use std::io::Read; - let n = ready!(self.registration.poll_read_io(cx, || { - let b = &mut *(buf.unfilled_mut() as *mut [std::mem::MaybeUninit] as *mut [u8]); - self.io.as_ref().unwrap().read(b) - }))?; + loop { + let evt = ready!(self.registration.poll_read_ready(cx))?; - // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the - // buffer. - buf.assume_init(n); - buf.advance(n); - Poll::Ready(Ok(())) + let b = &mut *(buf.unfilled_mut() as *mut [std::mem::MaybeUninit] as *mut [u8]); + let len = b.len(); + + match self.io.as_ref().unwrap().read(b) { + Ok(n) => { + // if we read a partially full buffer, this is sufficient on unix to show + // that the socket buffer has been drained + if n > 0 && (!cfg!(windows) && n < len) { + self.registration.clear_readiness(evt); + } + + // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the + // buffer. + buf.assume_init(n); + buf.advance(n); + return Poll::Ready(Ok(())); + }, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.registration.clear_readiness(evt); + } + Err(e) => return Poll::Ready(Err(e)), + } + } } pub(crate) fn poll_write<'a>(&'a self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> @@ -168,10 +189,29 @@ feature! { &'a E: io::Write + 'a, { use std::io::Write; - self.registration.poll_write_io(cx, || self.io.as_ref().unwrap().write(buf)) + + loop { + let evt = ready!(self.registration.poll_write_ready(cx))?; + + match self.io.as_ref().unwrap().write(buf) { + Ok(n) => { + // if we write only part of our buffer, this is sufficient on unix to show + // that the socket buffer is full + if n > 0 && (!cfg!(windows) && n < buf.len()) { + self.registration.clear_readiness(evt); + } + + return Poll::Ready(Ok(n)); + }, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.registration.clear_readiness(evt); + } + Err(e) => return Poll::Ready(Err(e)), + } + } } - #[cfg(feature = "net")] + #[cfg(any(feature = "net", feature = "process"))] pub(crate) fn poll_write_vectored<'a>( &'a self, cx: &mut Context<'_>, diff --git a/third_party/rust/tokio/src/io/read_buf.rs b/third_party/rust/tokio/src/io/read_buf.rs index 8c34ae6c817c..283d96e30950 100644 --- a/third_party/rust/tokio/src/io/read_buf.rs +++ b/third_party/rust/tokio/src/io/read_buf.rs @@ -152,6 +152,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if `self.remaining()` is less than `n`. #[inline] + #[track_caller] pub fn initialize_unfilled_to(&mut self, n: usize) -> &mut [u8] { assert!(self.remaining() >= n, "n overflows remaining"); @@ -195,6 +196,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if the filled region of the buffer would become larger than the initialized region. #[inline] + #[track_caller] pub fn advance(&mut self, n: usize) { let new = self.filled.checked_add(n).expect("filled overflow"); self.set_filled(new); @@ -211,6 +213,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if the filled region of the buffer would become larger than the initialized region. #[inline] + #[track_caller] pub fn set_filled(&mut self, n: usize) { assert!( n <= self.initialized, @@ -241,6 +244,7 @@ impl<'a> ReadBuf<'a> { /// /// Panics if `self.remaining()` is less than `buf.len()`. #[inline] + #[track_caller] pub fn put_slice(&mut self, buf: &[u8]) { assert!( self.remaining() >= buf.len(), @@ -266,6 +270,33 @@ impl<'a> ReadBuf<'a> { } } +#[cfg(feature = "io-util")] +#[cfg_attr(docsrs, doc(cfg(feature = "io-util")))] +unsafe impl<'a> bytes::BufMut for ReadBuf<'a> { + fn remaining_mut(&self) -> usize { + self.remaining() + } + + // SAFETY: The caller guarantees that at least `cnt` unfilled bytes have been initialized. + unsafe fn advance_mut(&mut self, cnt: usize) { + self.assume_init(cnt); + self.advance(cnt); + } + + fn chunk_mut(&mut self) -> &mut bytes::buf::UninitSlice { + // SAFETY: No region of `unfilled` will be deinitialized because it is + // exposed as an `UninitSlice`, whose API guarantees that the memory is + // never deinitialized. + let unfilled = unsafe { self.unfilled_mut() }; + let len = unfilled.len(); + let ptr = unfilled.as_mut_ptr() as *mut u8; + + // SAFETY: The pointer is valid for `len` bytes because it comes from a + // slice of that length. + unsafe { bytes::buf::UninitSlice::from_raw_parts_mut(ptr, len) } + } +} + impl fmt::Debug for ReadBuf<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ReadBuf") diff --git a/third_party/rust/tokio/src/io/driver/ready.rs b/third_party/rust/tokio/src/io/ready.rs similarity index 77% rename from third_party/rust/tokio/src/io/driver/ready.rs rename to third_party/rust/tokio/src/io/ready.rs index 2430d3022f1e..33e2ef0b63fe 100644 --- a/third_party/rust/tokio/src/io/driver/ready.rs +++ b/third_party/rust/tokio/src/io/ready.rs @@ -7,12 +7,14 @@ const READABLE: usize = 0b0_01; const WRITABLE: usize = 0b0_10; const READ_CLOSED: usize = 0b0_0100; const WRITE_CLOSED: usize = 0b0_1000; +#[cfg(any(target_os = "linux", target_os = "android"))] +const PRIORITY: usize = 0b1_0000; /// Describes the readiness state of an I/O resources. /// /// `Ready` tracks which operation an I/O resource is ready to perform. #[cfg_attr(docsrs, doc(cfg(feature = "net")))] -#[derive(Clone, Copy, PartialEq, PartialOrd)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct Ready(usize); impl Ready { @@ -31,7 +33,17 @@ impl Ready { /// Returns a `Ready` representing write closed readiness. pub const WRITE_CLOSED: Ready = Ready(WRITE_CLOSED); + /// Returns a `Ready` representing priority readiness. + #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))] + pub const PRIORITY: Ready = Ready(PRIORITY); + /// Returns a `Ready` representing readiness for all operations. + #[cfg(any(target_os = "linux", target_os = "android"))] + pub const ALL: Ready = Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED | PRIORITY); + + /// Returns a `Ready` representing readiness for all operations. + #[cfg(not(any(target_os = "linux", target_os = "android")))] pub const ALL: Ready = Ready(READABLE | WRITABLE | READ_CLOSED | WRITE_CLOSED); // Must remain crate-private to avoid adding a public dependency on Mio. @@ -65,6 +77,13 @@ impl Ready { ready |= Ready::WRITE_CLOSED; } + #[cfg(any(target_os = "linux", target_os = "android"))] + { + if event.is_priority() { + ready |= Ready::PRIORITY; + } + } + ready } @@ -144,6 +163,23 @@ impl Ready { self.contains(Ready::WRITE_CLOSED) } + /// Returns `true` if the value includes priority `readiness`. + /// + /// # Examples + /// + /// ``` + /// use tokio::io::Ready; + /// + /// assert!(!Ready::EMPTY.is_priority()); + /// assert!(!Ready::WRITABLE.is_priority()); + /// assert!(Ready::PRIORITY.is_priority()); + /// ``` + #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg_attr(docsrs, doc(cfg(any(target_os = "linux", target_os = "android"))))] + pub fn is_priority(self) -> bool { + self.contains(Ready::PRIORITY) + } + /// Returns true if `self` is a superset of `other`. /// /// `other` may represent more than one readiness operations, in which case @@ -191,6 +227,12 @@ cfg_io_readiness! { ready |= Ready::WRITE_CLOSED; } + #[cfg(any(target_os = "linux", target_os = "android"))] + if interest.is_priority() { + ready |= Ready::PRIORITY; + ready |= Ready::READ_CLOSED; + } + ready } @@ -240,11 +282,16 @@ impl ops::Sub for Ready { impl fmt::Debug for Ready { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Ready") - .field("is_readable", &self.is_readable()) + let mut fmt = fmt.debug_struct("Ready"); + + fmt.field("is_readable", &self.is_readable()) .field("is_writable", &self.is_writable()) .field("is_read_closed", &self.is_read_closed()) - .field("is_write_closed", &self.is_write_closed()) - .finish() + .field("is_write_closed", &self.is_write_closed()); + + #[cfg(any(target_os = "linux", target_os = "android"))] + fmt.field("is_priority", &self.is_priority()); + + fmt.finish() } } diff --git a/third_party/rust/tokio/src/io/split.rs b/third_party/rust/tokio/src/io/split.rs index 8258a0f7a081..f067b65a8265 100644 --- a/third_party/rust/tokio/src/io/split.rs +++ b/third_party/rust/tokio/src/io/split.rs @@ -74,7 +74,11 @@ impl ReadHalf { /// same `split` operation this method will panic. /// This can be checked ahead of time by comparing the stream ID /// of the two halves. - pub fn unsplit(self, wr: WriteHalf) -> T { + #[track_caller] + pub fn unsplit(self, wr: WriteHalf) -> T + where + T: Unpin, + { if self.is_pair_of(&wr) { drop(wr); diff --git a/third_party/rust/tokio/src/io/stderr.rs b/third_party/rust/tokio/src/io/stderr.rs index 2f624fba9d60..2119c6d6ce98 100644 --- a/third_party/rust/tokio/src/io/stderr.rs +++ b/third_party/rust/tokio/src/io/stderr.rs @@ -74,16 +74,43 @@ cfg_io_std! { } #[cfg(unix)] -impl std::os::unix::io::AsRawFd for Stderr { - fn as_raw_fd(&self) -> std::os::unix::io::RawFd { - std::io::stderr().as_raw_fd() +mod sys { + #[cfg(not(tokio_no_as_fd))] + use std::os::unix::io::{AsFd, BorrowedFd}; + use std::os::unix::io::{AsRawFd, RawFd}; + + use super::Stderr; + + impl AsRawFd for Stderr { + fn as_raw_fd(&self) -> RawFd { + std::io::stderr().as_raw_fd() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for Stderr { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } } } -#[cfg(windows)] -impl std::os::windows::io::AsRawHandle for Stderr { - fn as_raw_handle(&self) -> std::os::windows::io::RawHandle { - std::io::stderr().as_raw_handle() +cfg_windows! { + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsHandle, BorrowedHandle}; + use crate::os::windows::io::{AsRawHandle, RawHandle}; + + impl AsRawHandle for Stderr { + fn as_raw_handle(&self) -> RawHandle { + std::io::stderr().as_raw_handle() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsHandle for Stderr { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } } } diff --git a/third_party/rust/tokio/src/io/stdin.rs b/third_party/rust/tokio/src/io/stdin.rs index c9578f17b64d..04cced425ffd 100644 --- a/third_party/rust/tokio/src/io/stdin.rs +++ b/third_party/rust/tokio/src/io/stdin.rs @@ -49,16 +49,43 @@ cfg_io_std! { } #[cfg(unix)] -impl std::os::unix::io::AsRawFd for Stdin { - fn as_raw_fd(&self) -> std::os::unix::io::RawFd { - std::io::stdin().as_raw_fd() +mod sys { + #[cfg(not(tokio_no_as_fd))] + use std::os::unix::io::{AsFd, BorrowedFd}; + use std::os::unix::io::{AsRawFd, RawFd}; + + use super::Stdin; + + impl AsRawFd for Stdin { + fn as_raw_fd(&self) -> RawFd { + std::io::stdin().as_raw_fd() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for Stdin { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } } } -#[cfg(windows)] -impl std::os::windows::io::AsRawHandle for Stdin { - fn as_raw_handle(&self) -> std::os::windows::io::RawHandle { - std::io::stdin().as_raw_handle() +cfg_windows! { + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsHandle, BorrowedHandle}; + use crate::os::windows::io::{AsRawHandle, RawHandle}; + + impl AsRawHandle for Stdin { + fn as_raw_handle(&self) -> RawHandle { + std::io::stdin().as_raw_handle() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsHandle for Stdin { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } } } diff --git a/third_party/rust/tokio/src/io/stdio_common.rs b/third_party/rust/tokio/src/io/stdio_common.rs index 7e4a198a82bb..06da761b8585 100644 --- a/third_party/rust/tokio/src/io/stdio_common.rs +++ b/third_party/rust/tokio/src/io/stdio_common.rs @@ -42,7 +42,7 @@ where // for further code. Since `AsyncWrite` can always shrink // buffer at its discretion, excessive (i.e. in tests) shrinking // does not break correctness. - // 2. If buffer is small, it will not be shrinked. + // 2. If buffer is small, it will not be shrunk. // That's why, it's "textness" will not change, so we don't have // to fixup it. if cfg!(not(any(target_os = "windows", test))) || buf.len() <= crate::io::blocking::MAX_BUF @@ -108,14 +108,13 @@ where #[cfg(test)] #[cfg(not(loom))] mod tests { + use crate::io::blocking::MAX_BUF; use crate::io::AsyncWriteExt; use std::io; use std::pin::Pin; use std::task::Context; use std::task::Poll; - const MAX_BUF: usize = 16 * 1024; - struct TextMockWriter; impl crate::io::AsyncWrite for TextMockWriter { @@ -177,6 +176,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn test_splitter() { let data = str::repeat("â–ˆ", MAX_BUF); let mut wr = super::SplitByUtf8BoundaryIfWindows::new(TextMockWriter); @@ -190,10 +190,11 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn test_pseudo_text() { // In this test we write a piece of binary data, whose beginning is // text though. We then validate that even in this corner case buffer - // was not shrinked too much. + // was not shrunk too much. let checked_count = super::MAGIC_CONST * super::MAX_BYTES_PER_CHAR; let mut data: Vec = str::repeat("a", checked_count).into(); data.extend(std::iter::repeat(0b1010_1010).take(MAX_BUF - checked_count + 1)); @@ -212,7 +213,7 @@ mod tests { writer.write_history.iter().copied().sum::(), data.len() ); - // Check that at most MAX_BYTES_PER_CHAR + 1 (i.e. 5) bytes were shrinked + // Check that at most MAX_BYTES_PER_CHAR + 1 (i.e. 5) bytes were shrunk // from the buffer: one because it was outside of MAX_BUF boundary, and // up to one "utf8 code point". assert!(data.len() - writer.write_history[0] <= super::MAX_BYTES_PER_CHAR + 1); diff --git a/third_party/rust/tokio/src/io/stdout.rs b/third_party/rust/tokio/src/io/stdout.rs index a08ed01eeded..f52bf5a4bcf3 100644 --- a/third_party/rust/tokio/src/io/stdout.rs +++ b/third_party/rust/tokio/src/io/stdout.rs @@ -73,16 +73,43 @@ cfg_io_std! { } #[cfg(unix)] -impl std::os::unix::io::AsRawFd for Stdout { - fn as_raw_fd(&self) -> std::os::unix::io::RawFd { - std::io::stdout().as_raw_fd() +mod sys { + #[cfg(not(tokio_no_as_fd))] + use std::os::unix::io::{AsFd, BorrowedFd}; + use std::os::unix::io::{AsRawFd, RawFd}; + + use super::Stdout; + + impl AsRawFd for Stdout { + fn as_raw_fd(&self) -> RawFd { + std::io::stdout().as_raw_fd() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for Stdout { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } } } -#[cfg(windows)] -impl std::os::windows::io::AsRawHandle for Stdout { - fn as_raw_handle(&self) -> std::os::windows::io::RawHandle { - std::io::stdout().as_raw_handle() +cfg_windows! { + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsHandle, BorrowedHandle}; + use crate::os::windows::io::{AsRawHandle, RawHandle}; + + impl AsRawHandle for Stdout { + fn as_raw_handle(&self) -> RawHandle { + std::io::stdout().as_raw_handle() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsHandle for Stdout { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } } } diff --git a/third_party/rust/tokio/src/io/util/async_buf_read_ext.rs b/third_party/rust/tokio/src/io/util/async_buf_read_ext.rs index b241e354ba83..50ea2d91a7cd 100644 --- a/third_party/rust/tokio/src/io/util/async_buf_read_ext.rs +++ b/third_party/rust/tokio/src/io/util/async_buf_read_ext.rs @@ -146,7 +146,7 @@ cfg_io_util! { /// [`next_line`] method. /// * Use [`tokio_util::codec::LinesCodec`][LinesCodec]. /// - /// [LinesCodec]: https://docs.rs/tokio-util/0.6/tokio_util/codec/struct.LinesCodec.html + /// [LinesCodec]: https://docs.rs/tokio-util/latest/tokio_util/codec/struct.LinesCodec.html /// [`read_until`]: Self::read_until /// [`lines`]: Self::lines /// [`next_line`]: crate::io::Lines::next_line diff --git a/third_party/rust/tokio/src/io/util/async_seek_ext.rs b/third_party/rust/tokio/src/io/util/async_seek_ext.rs index 46b3e6c0d398..aadf3a76a96f 100644 --- a/third_party/rust/tokio/src/io/util/async_seek_ext.rs +++ b/third_party/rust/tokio/src/io/util/async_seek_ext.rs @@ -69,7 +69,7 @@ cfg_io_util! { /// Creates a future which will rewind to the beginning of the stream. /// - /// This is convenience method, equivalent to to `self.seek(SeekFrom::Start(0))`. + /// This is convenience method, equivalent to `self.seek(SeekFrom::Start(0))`. fn rewind(&mut self) -> Seek<'_, Self> where Self: Unpin, diff --git a/third_party/rust/tokio/src/io/util/async_write_ext.rs b/third_party/rust/tokio/src/io/util/async_write_ext.rs index 93a318315eea..dfdde82f3225 100644 --- a/third_party/rust/tokio/src/io/util/async_write_ext.rs +++ b/third_party/rust/tokio/src/io/util/async_write_ext.rs @@ -406,7 +406,7 @@ cfg_io_util! { /// ``` fn write_u8(&mut self, n: u8) -> WriteU8; - /// Writes an unsigned 8-bit integer to the underlying writer. + /// Writes a signed 8-bit integer to the underlying writer. /// /// Equivalent to: /// @@ -425,7 +425,7 @@ cfg_io_util! { /// /// # Examples /// - /// Write unsigned 8 bit integers to a `AsyncWrite`: + /// Write signed 8 bit integers to a `AsyncWrite`: /// /// ```rust /// use tokio::io::{self, AsyncWriteExt}; @@ -434,10 +434,10 @@ cfg_io_util! { /// async fn main() -> io::Result<()> { /// let mut writer = Vec::new(); /// - /// writer.write_u8(2).await?; - /// writer.write_u8(5).await?; + /// writer.write_i8(-2).await?; + /// writer.write_i8(126).await?; /// - /// assert_eq!(writer, b"\x02\x05"); + /// assert_eq!(writer, b"\xFE\x7E"); /// Ok(()) /// } /// ``` diff --git a/third_party/rust/tokio/src/io/util/copy.rs b/third_party/rust/tokio/src/io/util/copy.rs index d0ab7cb14098..55861244fab0 100644 --- a/third_party/rust/tokio/src/io/util/copy.rs +++ b/third_party/rust/tokio/src/io/util/copy.rs @@ -27,6 +27,51 @@ impl CopyBuffer { } } + fn poll_fill_buf( + &mut self, + cx: &mut Context<'_>, + reader: Pin<&mut R>, + ) -> Poll> + where + R: AsyncRead + ?Sized, + { + let me = &mut *self; + let mut buf = ReadBuf::new(&mut me.buf); + buf.set_filled(me.cap); + + let res = reader.poll_read(cx, &mut buf); + if let Poll::Ready(Ok(_)) = res { + let filled_len = buf.filled().len(); + me.read_done = me.cap == filled_len; + me.cap = filled_len; + } + res + } + + fn poll_write_buf( + &mut self, + cx: &mut Context<'_>, + mut reader: Pin<&mut R>, + mut writer: Pin<&mut W>, + ) -> Poll> + where + R: AsyncRead + ?Sized, + W: AsyncWrite + ?Sized, + { + let me = &mut *self; + match writer.as_mut().poll_write(cx, &me.buf[me.pos..me.cap]) { + Poll::Pending => { + // Top up the buffer towards full if we can read a bit more + // data - this should improve the chances of a large write + if !me.read_done && me.cap < me.buf.len() { + ready!(me.poll_fill_buf(cx, reader.as_mut()))?; + } + Poll::Pending + } + res => res, + } + } + pub(super) fn poll_copy( &mut self, cx: &mut Context<'_>, @@ -41,10 +86,10 @@ impl CopyBuffer { // If our buffer is empty, then we need to read some data to // continue. if self.pos == self.cap && !self.read_done { - let me = &mut *self; - let mut buf = ReadBuf::new(&mut me.buf); + self.pos = 0; + self.cap = 0; - match reader.as_mut().poll_read(cx, &mut buf) { + match self.poll_fill_buf(cx, reader.as_mut()) { Poll::Ready(Ok(_)) => (), Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), Poll::Pending => { @@ -58,20 +103,11 @@ impl CopyBuffer { return Poll::Pending; } } - - let n = buf.filled().len(); - if n == 0 { - self.read_done = true; - } else { - self.pos = 0; - self.cap = n; - } } // If our buffer has some data, let's write it out! while self.pos < self.cap { - let me = &mut *self; - let i = ready!(writer.as_mut().poll_write(cx, &me.buf[me.pos..me.cap]))?; + let i = ready!(self.poll_write_buf(cx, reader.as_mut(), writer.as_mut()))?; if i == 0 { return Poll::Ready(Err(io::Error::new( io::ErrorKind::WriteZero, @@ -117,14 +153,22 @@ cfg_io_util! { /// /// This function returns a future that will continuously read data from /// `reader` and then write it into `writer` in a streaming fashion until - /// `reader` returns EOF. + /// `reader` returns EOF or fails. /// /// On success, the total number of bytes that were copied from `reader` to /// `writer` is returned. /// /// This is an asynchronous version of [`std::io::copy`][std]. /// + /// A heap-allocated copy buffer with 8 KB is created to take data from the + /// reader to the writer, check [`copy_buf`] if you want an alternative for + /// [`AsyncBufRead`]. You can use `copy_buf` with [`BufReader`] to change the + /// buffer capacity. + /// /// [std]: std::io::copy + /// [`copy_buf`]: crate::io::copy_buf + /// [`AsyncBufRead`]: crate::io::AsyncBufRead + /// [`BufReader`]: crate::io::BufReader /// /// # Errors /// diff --git a/third_party/rust/tokio/src/io/util/copy_bidirectional.rs b/third_party/rust/tokio/src/io/util/copy_bidirectional.rs index c93060b361ac..e1a7db127a7f 100644 --- a/third_party/rust/tokio/src/io/util/copy_bidirectional.rs +++ b/third_party/rust/tokio/src/io/util/copy_bidirectional.rs @@ -1,8 +1,8 @@ use super::copy::CopyBuffer; +use crate::future::poll_fn; use crate::io::{AsyncRead, AsyncWrite}; -use std::future::Future; use std::io; use std::pin::Pin; use std::task::{Context, Poll}; @@ -13,13 +13,6 @@ enum TransferState { Done(u64), } -struct CopyBidirectional<'a, A: ?Sized, B: ?Sized> { - a: &'a mut A, - b: &'a mut B, - a_to_b: TransferState, - b_to_a: TransferState, -} - fn transfer_one_direction( cx: &mut Context<'_>, state: &mut TransferState, @@ -48,35 +41,6 @@ where } } } - -impl<'a, A, B> Future for CopyBidirectional<'a, A, B> -where - A: AsyncRead + AsyncWrite + Unpin + ?Sized, - B: AsyncRead + AsyncWrite + Unpin + ?Sized, -{ - type Output = io::Result<(u64, u64)>; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // Unpack self into mut refs to each field to avoid borrow check issues. - let CopyBidirectional { - a, - b, - a_to_b, - b_to_a, - } = &mut *self; - - let a_to_b = transfer_one_direction(cx, a_to_b, &mut *a, &mut *b)?; - let b_to_a = transfer_one_direction(cx, b_to_a, &mut *b, &mut *a)?; - - // It is not a problem if ready! returns early because transfer_one_direction for the - // other direction will keep returning TransferState::Done(count) in future calls to poll - let a_to_b = ready!(a_to_b); - let b_to_a = ready!(b_to_a); - - Poll::Ready(Ok((a_to_b, b_to_a))) - } -} - /// Copies data in both directions between `a` and `b`. /// /// This function returns a future that will read from both streams, @@ -110,11 +74,18 @@ where A: AsyncRead + AsyncWrite + Unpin + ?Sized, B: AsyncRead + AsyncWrite + Unpin + ?Sized, { - CopyBidirectional { - a, - b, - a_to_b: TransferState::Running(CopyBuffer::new()), - b_to_a: TransferState::Running(CopyBuffer::new()), - } + let mut a_to_b = TransferState::Running(CopyBuffer::new()); + let mut b_to_a = TransferState::Running(CopyBuffer::new()); + poll_fn(|cx| { + let a_to_b = transfer_one_direction(cx, &mut a_to_b, a, b)?; + let b_to_a = transfer_one_direction(cx, &mut b_to_a, b, a)?; + + // It is not a problem if ready! returns early because transfer_one_direction for the + // other direction will keep returning TransferState::Done(count) in future calls to poll + let a_to_b = ready!(a_to_b); + let b_to_a = ready!(b_to_a); + + Poll::Ready(Ok((a_to_b, b_to_a))) + }) .await } diff --git a/third_party/rust/tokio/src/io/util/copy_buf.rs b/third_party/rust/tokio/src/io/util/copy_buf.rs index 6831580b407d..c23fc9a2b4c2 100644 --- a/third_party/rust/tokio/src/io/util/copy_buf.rs +++ b/third_party/rust/tokio/src/io/util/copy_buf.rs @@ -24,11 +24,17 @@ cfg_io_util! { /// /// This function returns a future that will continuously read data from /// `reader` and then write it into `writer` in a streaming fashion until - /// `reader` returns EOF. + /// `reader` returns EOF or fails. /// /// On success, the total number of bytes that were copied from `reader` to /// `writer` is returned. /// + /// This is a [`tokio::io::copy`] alternative for [`AsyncBufRead`] readers + /// with no extra buffer allocation, since [`AsyncBufRead`] allow access + /// to the reader's inner buffer. + /// + /// [`tokio::io::copy`]: crate::io::copy + /// [`AsyncBufRead`]: crate::io::AsyncBufRead /// /// # Errors /// diff --git a/third_party/rust/tokio/src/io/util/empty.rs b/third_party/rust/tokio/src/io/util/empty.rs index 77db60e40b4c..b96fabbaabe9 100644 --- a/third_party/rust/tokio/src/io/util/empty.rs +++ b/third_party/rust/tokio/src/io/util/empty.rs @@ -53,6 +53,7 @@ impl AsyncRead for Empty { cx: &mut Context<'_>, _: &mut ReadBuf<'_>, ) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); ready!(poll_proceed_and_make_progress(cx)); Poll::Ready(Ok(())) } @@ -61,6 +62,7 @@ impl AsyncRead for Empty { impl AsyncBufRead for Empty { #[inline] fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); ready!(poll_proceed_and_make_progress(cx)); Poll::Ready(Ok(&[])) } @@ -77,7 +79,7 @@ impl fmt::Debug for Empty { cfg_coop! { fn poll_proceed_and_make_progress(cx: &mut Context<'_>) -> Poll<()> { - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); coop.made_progress(); Poll::Ready(()) } diff --git a/third_party/rust/tokio/src/io/util/mem.rs b/third_party/rust/tokio/src/io/util/mem.rs index 4019db56ff4c..5b404c21940a 100644 --- a/third_party/rust/tokio/src/io/util/mem.rs +++ b/third_party/rust/tokio/src/io/util/mem.rs @@ -233,7 +233,8 @@ impl AsyncRead for Pipe { cx: &mut task::Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll> { - let coop = ready!(crate::coop::poll_proceed(cx)); + ready!(crate::trace::trace_leaf(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let ret = self.poll_read_internal(cx, buf); if ret.is_ready() { @@ -249,6 +250,7 @@ impl AsyncRead for Pipe { cx: &mut task::Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); self.poll_read_internal(cx, buf) } } @@ -261,7 +263,8 @@ impl AsyncWrite for Pipe { cx: &mut task::Context<'_>, buf: &[u8], ) -> Poll> { - let coop = ready!(crate::coop::poll_proceed(cx)); + ready!(crate::trace::trace_leaf(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let ret = self.poll_write_internal(cx, buf); if ret.is_ready() { @@ -277,6 +280,7 @@ impl AsyncWrite for Pipe { cx: &mut task::Context<'_>, buf: &[u8], ) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); self.poll_write_internal(cx, buf) } } diff --git a/third_party/rust/tokio/src/io/util/read.rs b/third_party/rust/tokio/src/io/util/read.rs index edc9d5a9e6e5..a1f9c8a0505a 100644 --- a/third_party/rust/tokio/src/io/util/read.rs +++ b/third_party/rust/tokio/src/io/util/read.rs @@ -48,7 +48,7 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let me = self.project(); - let mut buf = ReadBuf::new(*me.buf); + let mut buf = ReadBuf::new(me.buf); ready!(Pin::new(me.reader).poll_read(cx, &mut buf))?; Poll::Ready(Ok(buf.filled().len())) } diff --git a/third_party/rust/tokio/src/io/util/read_to_end.rs b/third_party/rust/tokio/src/io/util/read_to_end.rs index f4a564d7dd49..8edba2a17110 100644 --- a/third_party/rust/tokio/src/io/util/read_to_end.rs +++ b/third_party/rust/tokio/src/io/util/read_to_end.rs @@ -1,11 +1,11 @@ use crate::io::util::vec_with_initialized::{into_read_buf_parts, VecU8, VecWithInitialized}; -use crate::io::AsyncRead; +use crate::io::{AsyncRead, ReadBuf}; use pin_project_lite::pin_project; use std::future::Future; use std::io; use std::marker::PhantomPinned; -use std::mem; +use std::mem::{self, MaybeUninit}; use std::pin::Pin; use std::task::{Context, Poll}; @@ -67,16 +67,47 @@ fn poll_read_to_end( // has 4 bytes while still making large reads if the reader does have a ton // of data to return. Simply tacking on an extra DEFAULT_BUF_SIZE space every // time is 4,500 times (!) slower than this if the reader has a very small - // amount of data to return. - buf.reserve(32); + // amount of data to return. When the vector is full with its starting + // capacity, we first try to read into a small buffer to see if we reached + // an EOF. This only happens when the starting capacity is >= NUM_BYTES, since + // we allocate at least NUM_BYTES each time. This avoids the unnecessary + // allocation that we attempt before reading into the vector. + + const NUM_BYTES: usize = 32; + let try_small_read = buf.try_small_read_first(NUM_BYTES); // Get a ReadBuf into the vector. - let mut read_buf = buf.get_read_buf(); + let mut read_buf; + let poll_result; - let filled_before = read_buf.filled().len(); - let poll_result = read.poll_read(cx, &mut read_buf); - let filled_after = read_buf.filled().len(); - let n = filled_after - filled_before; + let n = if try_small_read { + // Read some bytes using a small read. + let mut small_buf: [MaybeUninit; NUM_BYTES] = [MaybeUninit::uninit(); NUM_BYTES]; + let mut small_read_buf = ReadBuf::uninit(&mut small_buf); + poll_result = read.poll_read(cx, &mut small_read_buf); + let to_write = small_read_buf.filled(); + + // Ensure we have enough space to fill our vector with what we read. + read_buf = buf.get_read_buf(); + if to_write.len() > read_buf.remaining() { + buf.reserve(NUM_BYTES); + read_buf = buf.get_read_buf(); + } + read_buf.put_slice(to_write); + + to_write.len() + } else { + // Ensure we have enough space for reading. + buf.reserve(NUM_BYTES); + read_buf = buf.get_read_buf(); + + // Read data directly into vector. + let filled_before = read_buf.filled().len(); + poll_result = read.poll_read(cx, &mut read_buf); + + // Compute the number of bytes read. + read_buf.filled().len() - filled_before + }; // Update the length of the vector using the result of poll_read. let read_buf_parts = into_read_buf_parts(read_buf); @@ -87,11 +118,11 @@ fn poll_read_to_end( // In this case, nothing should have been read. However we still // update the vector in case the poll_read call initialized parts of // the vector's unused capacity. - debug_assert_eq!(filled_before, filled_after); + debug_assert_eq!(n, 0); Poll::Pending } Poll::Ready(Err(err)) => { - debug_assert_eq!(filled_before, filled_after); + debug_assert_eq!(n, 0); Poll::Ready(Err(err)) } Poll::Ready(Ok(())) => Poll::Ready(Ok(n)), diff --git a/third_party/rust/tokio/src/io/util/read_until.rs b/third_party/rust/tokio/src/io/util/read_until.rs index 90a0e8a18d57..fb6fb22d9f15 100644 --- a/third_party/rust/tokio/src/io/util/read_until.rs +++ b/third_party/rust/tokio/src/io/util/read_until.rs @@ -1,4 +1,5 @@ use crate::io::AsyncBufRead; +use crate::util::memchr; use pin_project_lite::pin_project; use std::future::Future; diff --git a/third_party/rust/tokio/src/io/util/vec_with_initialized.rs b/third_party/rust/tokio/src/io/util/vec_with_initialized.rs index 208cc939c1a4..f527492dcd96 100644 --- a/third_party/rust/tokio/src/io/util/vec_with_initialized.rs +++ b/third_party/rust/tokio/src/io/util/vec_with_initialized.rs @@ -1,19 +1,18 @@ use crate::io::ReadBuf; use std::mem::MaybeUninit; -mod private { - pub trait Sealed {} +/// Something that looks like a `Vec`. +/// +/// # Safety +/// +/// The implementor must guarantee that the vector returned by the +/// `as_mut` and `as_mut` methods do not change from one call to +/// another. +pub(crate) unsafe trait VecU8: AsRef> + AsMut> {} - impl Sealed for Vec {} - impl Sealed for &mut Vec {} -} +unsafe impl VecU8 for Vec {} +unsafe impl VecU8 for &mut Vec {} -/// A sealed trait that constrains the generic type parameter in `VecWithInitialized`. That struct's safety relies -/// on certain invariants upheld by `Vec`. -pub(crate) trait VecU8: AsMut> + private::Sealed {} - -impl VecU8 for Vec {} -impl VecU8 for &mut Vec {} /// This struct wraps a `Vec` or `&mut Vec`, combining it with a /// `num_initialized`, which keeps track of the number of initialized bytes /// in the unused capacity. @@ -29,6 +28,7 @@ pub(crate) struct VecWithInitialized { // The number of initialized bytes in the vector. // Always between `vec.len()` and `vec.capacity()`. num_initialized: usize, + starting_capacity: usize, } impl VecWithInitialized> { @@ -48,6 +48,7 @@ where // to its length are initialized. Self { num_initialized: vec.as_mut().len(), + starting_capacity: vec.as_ref().capacity(), vec, } } @@ -64,8 +65,8 @@ where } #[cfg(feature = "io-util")] - pub(crate) fn is_empty(&mut self) -> bool { - self.vec.as_mut().is_empty() + pub(crate) fn is_empty(&self) -> bool { + self.vec.as_ref().is_empty() } pub(crate) fn get_read_buf<'a>(&'a mut self) -> ReadBuf<'a> { @@ -112,6 +113,15 @@ where vec.set_len(parts.len); } } + + // Returns a boolean telling the caller to try reading into a small local buffer first if true. + // Doing so would avoid overallocating when vec is filled to capacity and we reached EOF. + pub(crate) fn try_small_read_first(&self, num_bytes: usize) -> bool { + let vec = self.vec.as_ref(); + vec.capacity() - vec.len() < num_bytes + && self.starting_capacity == vec.capacity() + && self.starting_capacity >= num_bytes + } } pub(crate) struct ReadBufParts { diff --git a/third_party/rust/tokio/src/lib.rs b/third_party/rust/tokio/src/lib.rs index 35295d837a66..c712945a4800 100644 --- a/third_party/rust/tokio/src/lib.rs +++ b/third_party/rust/tokio/src/lib.rs @@ -1,7 +1,9 @@ #![allow( clippy::cognitive_complexity, clippy::large_enum_variant, - clippy::needless_doctest_main + clippy::module_inception, + clippy::needless_doctest_main, + clippy::declare_interior_mutable_const )] #![warn( missing_debug_implementations, @@ -16,6 +18,7 @@ ))] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(docsrs, allow(unused_attributes))] +#![cfg_attr(loom, allow(dead_code, unreachable_pub))] //! A runtime for writing reliable network applications without compromising speed. //! @@ -114,7 +117,7 @@ //! The [`tokio::sync`] module contains synchronization primitives to use when //! needing to communicate or share data. These include: //! -//! * channels ([`oneshot`], [`mpsc`], and [`watch`]), for sending values +//! * channels ([`oneshot`], [`mpsc`], [`watch`], and [`broadcast`]), for sending values //! between tasks, //! * a non-blocking [`Mutex`], for controlling access to a shared, mutable //! value, @@ -130,6 +133,7 @@ //! [`oneshot`]: crate::sync::oneshot //! [`mpsc`]: crate::sync::mpsc //! [`watch`]: crate::sync::watch +//! [`broadcast`]: crate::sync::broadcast //! //! The [`tokio::time`] module provides utilities for tracking time and //! scheduling work. This includes functions for setting [timeouts][timeout] for @@ -151,7 +155,7 @@ //! provide the functionality you need. //! //! Using the runtime requires the "rt" or "rt-multi-thread" feature flags, to -//! enable the basic [single-threaded scheduler][rt] and the [thread-pool +//! enable the current-thread [single-threaded scheduler][rt] and the [multi-thread //! scheduler][rt-multi-thread], respectively. See the [`runtime` module //! documentation][rt-features] for details. In addition, the "macros" feature //! flag enables the `#[tokio::main]` and `#[tokio::test]` attributes. @@ -170,12 +174,15 @@ //! swapping the currently running task on each thread. However, this kind of //! swapping can only happen at `.await` points, so code that spends a long time //! without reaching an `.await` will prevent other tasks from running. To -//! combat this, Tokio provides two kinds of threads: Core threads and blocking -//! threads. The core threads are where all asynchronous code runs, and Tokio -//! will by default spawn one for each CPU core. The blocking threads are -//! spawned on demand, can be used to run blocking code that would otherwise -//! block other tasks from running and are kept alive when not used for a certain -//! amount of time which can be configured with [`thread_keep_alive`]. +//! combat this, Tokio provides two kinds of threads: Core threads and blocking threads. +//! +//! The core threads are where all asynchronous code runs, and Tokio will by default +//! spawn one for each CPU core. You can use the environment variable `TOKIO_WORKER_THREADS` +//! to override the default value. +//! +//! The blocking threads are spawned on demand, can be used to run blocking code +//! that would otherwise block other tasks from running and are kept alive when +//! not used for a certain amount of time which can be configured with [`thread_keep_alive`]. //! Since it is not possible for Tokio to swap out blocking tasks, like it //! can do with asynchronous code, the upper limit on the number of blocking //! threads is very large. These limits can be configured on the [`Builder`]. @@ -308,7 +315,7 @@ //! need. //! //! - `full`: Enables all features listed below except `test-util` and `tracing`. -//! - `rt`: Enables `tokio::spawn`, the basic (current thread) scheduler, +//! - `rt`: Enables `tokio::spawn`, the current-thread scheduler, //! and non-scheduler utilities. //! - `rt-multi-thread`: Enables the heavier, multi-threaded, work-stealing scheduler. //! - `io-util`: Enables the IO based `Ext` traits. @@ -324,30 +331,117 @@ //! - `signal`: Enables all `tokio::signal` types. //! - `fs`: Enables `tokio::fs` types. //! - `test-util`: Enables testing based infrastructure for the Tokio runtime. +//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's +//! synchronization primitives internally. Also, this +//! dependency is necessary to construct some of our primitives +//! in a const context. MSRV may increase according to the +//! _parking_lot_ release in use. //! //! _Note: `AsyncRead` and `AsyncWrite` traits do not require any features and are //! always available._ //! -//! ### Internal features -//! -//! These features do not expose any new API, but influence internal -//! implementation aspects of Tokio, and can pull in additional -//! dependencies. -//! -//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's -//! synchronization primitives internally. MSRV may increase according to the -//! _parking_lot_ release in use. -//! //! ### Unstable features //! -//! These feature flags enable **unstable** features. The public API may break in 1.x -//! releases. To enable these features, the `--cfg tokio_unstable` must be passed to -//! `rustc` when compiling. This is easiest done using the `RUSTFLAGS` env variable: -//! `RUSTFLAGS="--cfg tokio_unstable"`. +//! Some feature flags are only available when specifying the `tokio_unstable` flag: //! //! - `tracing`: Enables tracing events. //! +//! Likewise, some parts of the API are only available with the same flag: +//! +//! - [`task::Builder`] +//! - Some methods on [`task::JoinSet`] +//! - [`runtime::RuntimeMetrics`] +//! - [`runtime::Builder::unhandled_panic`] +//! - [`task::Id`] +//! +//! This flag enables **unstable** features. The public API of these features +//! may break in 1.x releases. To enable these features, the `--cfg +//! tokio_unstable` argument must be passed to `rustc` when compiling. This +//! serves to explicitly opt-in to features which may break semver conventions, +//! since Cargo [does not yet directly support such opt-ins][unstable features]. +//! +//! You can specify it in your project's `.cargo/config.toml` file: +//! +//! ```toml +//! [build] +//! rustflags = ["--cfg", "tokio_unstable"] +//! ``` +//! +//! Alternatively, you can specify it with an environment variable: +//! +//! ```sh +//! ## Many *nix shells: +//! export RUSTFLAGS="--cfg tokio_unstable" +//! cargo build +//! ``` +//! +//! ```powershell +//! ## Windows PowerShell: +//! $Env:RUSTFLAGS="--cfg tokio_unstable" +//! cargo build +//! ``` +//! +//! [unstable features]: https://internals.rust-lang.org/t/feature-request-unstable-opt-in-non-transitive-crate-features/16193#why-not-a-crate-feature-2 //! [feature flags]: https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section +//! +//! ## Supported platforms +//! +//! Tokio currently guarantees support for the following platforms: +//! +//! * Linux +//! * Windows +//! * Android (API level 21) +//! * macOS +//! * iOS +//! * FreeBSD +//! +//! Tokio will continue to support these platforms in the future. However, +//! future releases may change requirements such as the minimum required libc +//! version on Linux, the API level on Android, or the supported FreeBSD +//! release. +//! +//! Beyond the above platforms, Tokio is intended to work on all platforms +//! supported by the mio crate. You can find a longer list [in mio's +//! documentation][mio-supported]. However, these additional platforms may +//! become unsupported in the future. +//! +//! Note that Wine is considered to be a different platform from Windows. See +//! mio's documentation for more information on Wine support. +//! +//! [mio-supported]: https://crates.io/crates/mio#platforms +//! +//! ### WASM support +//! +//! Tokio has some limited support for the WASM platform. Without the +//! `tokio_unstable` flag, the following features are supported: +//! +//! * `sync` +//! * `macros` +//! * `io-util` +//! * `rt` +//! * `time` +//! +//! Enabling any other feature (including `full`) will cause a compilation +//! failure. +//! +//! The `time` module will only work on WASM platforms that have support for +//! timers (e.g. wasm32-wasi). The timing functions will panic if used on a WASM +//! platform that does not support timers. +//! +//! Note also that if the runtime becomes indefinitely idle, it will panic +//! immediately instead of blocking forever. On platforms that don't support +//! time, this means that the runtime can never be idle in any way. +//! +//! ### Unstable WASM support +//! +//! Tokio also has unstable support for some additional WASM features. This +//! requires the use of the `tokio_unstable` flag. +//! +//! Using this flag enables the use of `tokio::net` on the wasm32-wasi target. +//! However, not all methods are available on the networking types as WASI +//! currently does not support the creation of new sockets from within WASM. +//! Because of this, sockets must currently be created via the `FromRawFd` +//! trait. // Test that pointer width is compatible. This asserts that e.g. usize is at // least 32 bits, which a lot of components in Tokio currently assumes. @@ -362,6 +456,52 @@ compile_error! { "Tokio requires the platform pointer width to be 32, 64, or 128 bits" } +// Ensure that our build script has correctly set cfg flags for wasm. +// +// Each condition is written all(a, not(b)). This should be read as +// "if a, then we must also have b". +#[cfg(any( + all(target_arch = "wasm32", not(tokio_wasm)), + all(target_arch = "wasm64", not(tokio_wasm)), + all(target_family = "wasm", not(tokio_wasm)), + all(target_os = "wasi", not(tokio_wasm)), + all(target_os = "wasi", not(tokio_wasi)), + all(target_os = "wasi", tokio_wasm_not_wasi), + all(tokio_wasm, not(any(target_arch = "wasm32", target_arch = "wasm64"))), + all(tokio_wasm_not_wasi, not(tokio_wasm)), + all(tokio_wasi, not(tokio_wasm)) +))] +compile_error!("Tokio's build script has incorrectly detected wasm."); + +#[cfg(all( + not(tokio_unstable), + tokio_wasm, + any( + feature = "fs", + feature = "io-std", + feature = "net", + feature = "process", + feature = "rt-multi-thread", + feature = "signal" + ) +))] +compile_error!("Only features sync,macros,io-util,rt,time are supported on wasm."); + +#[cfg(all(not(tokio_unstable), tokio_taskdump))] +compile_error!("The `tokio_taskdump` feature requires `--cfg tokio_unstable`."); + +#[cfg(all( + tokio_taskdump, + not(all( + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + )) +))] +compile_error!( + "The `tokio_taskdump` feature is only currently supported on \ +linux, on `aarch64`, `x86` and `x86_64`." +); + // Includes re-exports used by macros. // // This module is not intended to be part of the public API. In general, any @@ -380,20 +520,25 @@ pub mod io; pub mod net; mod loom; -mod park; cfg_process! { pub mod process; } -#[cfg(any(feature = "net", feature = "fs", feature = "io-std"))] +#[cfg(any( + feature = "fs", + feature = "io-std", + feature = "net", + all(windows, feature = "process"), +))] mod blocking; cfg_rt! { pub mod runtime; } - -pub(crate) mod coop; +cfg_not_rt! { + pub(crate) mod runtime; +} cfg_signal! { pub mod signal; @@ -422,6 +567,40 @@ cfg_time! { pub mod time; } +mod trace { + use std::future::Future; + use std::pin::Pin; + use std::task::{Context, Poll}; + + cfg_taskdump! { + pub(crate) use crate::runtime::task::trace::trace_leaf; + } + + cfg_not_taskdump! { + #[inline(always)] + #[allow(dead_code)] + pub(crate) fn trace_leaf(_: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Ready(()) + } + } + + #[cfg_attr(not(feature = "sync"), allow(dead_code))] + pub(crate) fn async_trace_leaf() -> impl Future { + struct Trace; + + impl Future for Trace { + type Output = (); + + #[inline(always)] + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + trace_leaf(cx) + } + } + + Trace + } +} + mod util; /// Due to the `Stream` trait's inclusion in `std` landing later than Tokio's 1.0 @@ -477,14 +656,6 @@ pub(crate) use self::doc::os; #[allow(unused)] pub(crate) use std::os; -#[cfg(docsrs)] -#[allow(unused)] -pub(crate) use self::doc::winapi; - -#[cfg(all(not(docsrs), windows, feature = "net"))] -#[allow(unused)] -pub(crate) use ::winapi; - cfg_macros! { /// Implementation detail of the `select!` macro. This macro is **not** /// intended to be used as part of the public API and is permitted to @@ -535,3 +706,7 @@ cfg_macros! { #[cfg(feature = "io-util")] #[cfg(test)] fn is_unpin() {} + +/// fuzz test (fuzz_linked_list) +#[cfg(fuzzing)] +pub mod fuzz; diff --git a/third_party/rust/tokio/src/loom/mocked.rs b/third_party/rust/tokio/src/loom/mocked.rs index 367d59b43a42..56dc1a06344d 100644 --- a/third_party/rust/tokio/src/loom/mocked.rs +++ b/third_party/rust/tokio/src/loom/mocked.rs @@ -25,6 +25,13 @@ pub(crate) mod sync { } } pub(crate) use loom::sync::*; + + pub(crate) mod atomic { + pub(crate) use loom::sync::atomic::*; + + // TODO: implement a loom version + pub(crate) type StaticAtomicU64 = std::sync::atomic::AtomicU64; + } } pub(crate) mod rand { @@ -38,3 +45,8 @@ pub(crate) mod sys { 2 } } + +pub(crate) mod thread { + pub use loom::lazy_static::AccessError; + pub use loom::thread::*; +} diff --git a/third_party/rust/tokio/src/loom/std/atomic_ptr.rs b/third_party/rust/tokio/src/loom/std/atomic_ptr.rs deleted file mode 100644 index 236645f037b0..000000000000 --- a/third_party/rust/tokio/src/loom/std/atomic_ptr.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::fmt; -use std::ops::{Deref, DerefMut}; - -/// `AtomicPtr` providing an additional `load_unsync` function. -pub(crate) struct AtomicPtr { - inner: std::sync::atomic::AtomicPtr, -} - -impl AtomicPtr { - pub(crate) fn new(ptr: *mut T) -> AtomicPtr { - let inner = std::sync::atomic::AtomicPtr::new(ptr); - AtomicPtr { inner } - } -} - -impl Deref for AtomicPtr { - type Target = std::sync::atomic::AtomicPtr; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl DerefMut for AtomicPtr { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} - -impl fmt::Debug for AtomicPtr { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - self.deref().fmt(fmt) - } -} diff --git a/third_party/rust/tokio/src/loom/std/atomic_u16.rs b/third_party/rust/tokio/src/loom/std/atomic_u16.rs index c1c531208c2c..c9e105c19343 100644 --- a/third_party/rust/tokio/src/loom/std/atomic_u16.rs +++ b/third_party/rust/tokio/src/loom/std/atomic_u16.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt; use std::ops::Deref; -/// `AtomicU16` providing an additional `load_unsync` function. +/// `AtomicU16` providing an additional `unsync_load` function. pub(crate) struct AtomicU16 { inner: UnsafeCell, } @@ -23,7 +23,7 @@ impl AtomicU16 { /// All mutations must have happened before the unsynchronized load. /// Additionally, there must be no concurrent mutations. pub(crate) unsafe fn unsync_load(&self) -> u16 { - *(*self.inner.get()).get_mut() + core::ptr::read(self.inner.get() as *const u16) } } diff --git a/third_party/rust/tokio/src/loom/std/atomic_u32.rs b/third_party/rust/tokio/src/loom/std/atomic_u32.rs index 61f95fb30ce2..ee0d2d38050c 100644 --- a/third_party/rust/tokio/src/loom/std/atomic_u32.rs +++ b/third_party/rust/tokio/src/loom/std/atomic_u32.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt; use std::ops::Deref; -/// `AtomicU32` providing an additional `load_unsync` function. +/// `AtomicU32` providing an additional `unsync_load` function. pub(crate) struct AtomicU32 { inner: UnsafeCell, } @@ -15,6 +15,16 @@ impl AtomicU32 { let inner = UnsafeCell::new(std::sync::atomic::AtomicU32::new(val)); AtomicU32 { inner } } + + /// Performs an unsynchronized load. + /// + /// # Safety + /// + /// All mutations must have happened before the unsynchronized load. + /// Additionally, there must be no concurrent mutations. + pub(crate) unsafe fn unsync_load(&self) -> u32 { + core::ptr::read(self.inner.get() as *const u32) + } } impl Deref for AtomicU32 { diff --git a/third_party/rust/tokio/src/loom/std/atomic_u64.rs b/third_party/rust/tokio/src/loom/std/atomic_u64.rs index 113992d97754..ce391be3e11c 100644 --- a/third_party/rust/tokio/src/loom/std/atomic_u64.rs +++ b/third_party/rust/tokio/src/loom/std/atomic_u64.rs @@ -7,72 +7,13 @@ // `#[cfg(target_has_atomic = "64")]`. // Refs: https://github.com/rust-lang/rust/tree/master/src/librustc_target cfg_has_atomic_u64! { - pub(crate) use std::sync::atomic::AtomicU64; + #[path = "atomic_u64_native.rs"] + mod imp; } cfg_not_has_atomic_u64! { - use crate::loom::sync::Mutex; - use std::sync::atomic::Ordering; - - #[derive(Debug)] - pub(crate) struct AtomicU64 { - inner: Mutex, - } - - impl AtomicU64 { - pub(crate) fn new(val: u64) -> Self { - Self { - inner: Mutex::new(val), - } - } - - pub(crate) fn load(&self, _: Ordering) -> u64 { - *self.inner.lock() - } - - pub(crate) fn store(&self, val: u64, _: Ordering) { - *self.inner.lock() = val; - } - - pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 { - let mut lock = self.inner.lock(); - let prev = *lock; - *lock = prev + val; - prev - } - - pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 { - let mut lock = self.inner.lock(); - let prev = *lock; - *lock = prev | val; - prev - } - - pub(crate) fn compare_exchange( - &self, - current: u64, - new: u64, - _success: Ordering, - _failure: Ordering, - ) -> Result { - let mut lock = self.inner.lock(); - - if *lock == current { - *lock = new; - Ok(current) - } else { - Err(*lock) - } - } - - pub(crate) fn compare_exchange_weak( - &self, - current: u64, - new: u64, - success: Ordering, - failure: Ordering, - ) -> Result { - self.compare_exchange(current, new, success, failure) - } - } + #[path = "atomic_u64_as_mutex.rs"] + mod imp; } + +pub(crate) use imp::{AtomicU64, StaticAtomicU64}; diff --git a/third_party/rust/tokio/src/loom/std/atomic_u64_as_mutex.rs b/third_party/rust/tokio/src/loom/std/atomic_u64_as_mutex.rs new file mode 100644 index 000000000000..9b3b6fac68cc --- /dev/null +++ b/third_party/rust/tokio/src/loom/std/atomic_u64_as_mutex.rs @@ -0,0 +1,76 @@ +use crate::loom::sync::Mutex; +use std::sync::atomic::Ordering; + +cfg_has_const_mutex_new! { + #[path = "atomic_u64_static_const_new.rs"] + mod static_macro; +} + +cfg_not_has_const_mutex_new! { + #[path = "atomic_u64_static_once_cell.rs"] + mod static_macro; +} + +pub(crate) use static_macro::StaticAtomicU64; + +#[derive(Debug)] +pub(crate) struct AtomicU64 { + inner: Mutex, +} + +impl AtomicU64 { + pub(crate) fn load(&self, _: Ordering) -> u64 { + *self.inner.lock() + } + + pub(crate) fn store(&self, val: u64, _: Ordering) { + *self.inner.lock() = val; + } + + pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 { + let mut lock = self.inner.lock(); + let prev = *lock; + *lock = prev + val; + prev + } + + pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 { + let mut lock = self.inner.lock(); + let prev = *lock; + *lock = prev | val; + prev + } + + pub(crate) fn compare_exchange( + &self, + current: u64, + new: u64, + _success: Ordering, + _failure: Ordering, + ) -> Result { + let mut lock = self.inner.lock(); + + if *lock == current { + *lock = new; + Ok(current) + } else { + Err(*lock) + } + } + + pub(crate) fn compare_exchange_weak( + &self, + current: u64, + new: u64, + success: Ordering, + failure: Ordering, + ) -> Result { + self.compare_exchange(current, new, success, failure) + } +} + +impl Default for AtomicU64 { + fn default() -> AtomicU64 { + AtomicU64::new(u64::default()) + } +} diff --git a/third_party/rust/tokio/src/loom/std/atomic_u64_native.rs b/third_party/rust/tokio/src/loom/std/atomic_u64_native.rs new file mode 100644 index 000000000000..08adb28629ad --- /dev/null +++ b/third_party/rust/tokio/src/loom/std/atomic_u64_native.rs @@ -0,0 +1,4 @@ +pub(crate) use std::sync::atomic::{AtomicU64, Ordering}; + +/// Alias `AtomicU64` to `StaticAtomicU64` +pub(crate) type StaticAtomicU64 = AtomicU64; diff --git a/third_party/rust/tokio/src/loom/std/atomic_u64_static_const_new.rs b/third_party/rust/tokio/src/loom/std/atomic_u64_static_const_new.rs new file mode 100644 index 000000000000..a4215342b68b --- /dev/null +++ b/third_party/rust/tokio/src/loom/std/atomic_u64_static_const_new.rs @@ -0,0 +1,12 @@ +use super::AtomicU64; +use crate::loom::sync::Mutex; + +pub(crate) type StaticAtomicU64 = AtomicU64; + +impl AtomicU64 { + pub(crate) const fn new(val: u64) -> Self { + Self { + inner: Mutex::const_new(val), + } + } +} diff --git a/third_party/rust/tokio/src/loom/std/atomic_u64_static_once_cell.rs b/third_party/rust/tokio/src/loom/std/atomic_u64_static_once_cell.rs new file mode 100644 index 000000000000..40c6172a524f --- /dev/null +++ b/third_party/rust/tokio/src/loom/std/atomic_u64_static_once_cell.rs @@ -0,0 +1,57 @@ +use super::AtomicU64; +use crate::loom::sync::{atomic::Ordering, Mutex}; +use crate::util::once_cell::OnceCell; + +pub(crate) struct StaticAtomicU64 { + init: u64, + cell: OnceCell>, +} + +impl AtomicU64 { + pub(crate) fn new(val: u64) -> Self { + Self { + inner: Mutex::new(val), + } + } +} + +impl StaticAtomicU64 { + pub(crate) const fn new(val: u64) -> StaticAtomicU64 { + StaticAtomicU64 { + init: val, + cell: OnceCell::new(), + } + } + + pub(crate) fn load(&self, order: Ordering) -> u64 { + *self.inner().lock() + } + + pub(crate) fn fetch_add(&self, val: u64, order: Ordering) -> u64 { + let mut lock = self.inner().lock(); + let prev = *lock; + *lock = prev + val; + prev + } + + pub(crate) fn compare_exchange_weak( + &self, + current: u64, + new: u64, + _success: Ordering, + _failure: Ordering, + ) -> Result { + let mut lock = self.inner().lock(); + + if *lock == current { + *lock = new; + Ok(current) + } else { + Err(*lock) + } + } + + fn inner(&self) -> &Mutex { + self.cell.get(|| Mutex::new(self.init)) + } +} diff --git a/third_party/rust/tokio/src/loom/std/atomic_u8.rs b/third_party/rust/tokio/src/loom/std/atomic_u8.rs deleted file mode 100644 index 408aea338c6a..000000000000 --- a/third_party/rust/tokio/src/loom/std/atomic_u8.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::cell::UnsafeCell; -use std::fmt; -use std::ops::Deref; - -/// `AtomicU8` providing an additional `load_unsync` function. -pub(crate) struct AtomicU8 { - inner: UnsafeCell, -} - -unsafe impl Send for AtomicU8 {} -unsafe impl Sync for AtomicU8 {} - -impl AtomicU8 { - pub(crate) const fn new(val: u8) -> AtomicU8 { - let inner = UnsafeCell::new(std::sync::atomic::AtomicU8::new(val)); - AtomicU8 { inner } - } -} - -impl Deref for AtomicU8 { - type Target = std::sync::atomic::AtomicU8; - - fn deref(&self) -> &Self::Target { - // safety: it is always safe to access `&self` fns on the inner value as - // we never perform unsafe mutations. - unsafe { &*self.inner.get() } - } -} - -impl fmt::Debug for AtomicU8 { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - self.deref().fmt(fmt) - } -} diff --git a/third_party/rust/tokio/src/loom/std/atomic_usize.rs b/third_party/rust/tokio/src/loom/std/atomic_usize.rs index 0d5f36e43103..c5503a2c1229 100644 --- a/third_party/rust/tokio/src/loom/std/atomic_usize.rs +++ b/third_party/rust/tokio/src/loom/std/atomic_usize.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt; use std::ops; -/// `AtomicUsize` providing an additional `load_unsync` function. +/// `AtomicUsize` providing an additional `unsync_load` function. pub(crate) struct AtomicUsize { inner: UnsafeCell, } @@ -23,7 +23,7 @@ impl AtomicUsize { /// All mutations must have happened before the unsynchronized load. /// Additionally, there must be no concurrent mutations. pub(crate) unsafe fn unsync_load(&self) -> usize { - *(*self.inner.get()).get_mut() + core::ptr::read(self.inner.get() as *const usize) } pub(crate) fn with_mut(&mut self, f: impl FnOnce(&mut usize) -> R) -> R { diff --git a/third_party/rust/tokio/src/loom/std/barrier.rs b/third_party/rust/tokio/src/loom/std/barrier.rs new file mode 100644 index 000000000000..a3f0ca0ab6d4 --- /dev/null +++ b/third_party/rust/tokio/src/loom/std/barrier.rs @@ -0,0 +1,217 @@ +//! A `Barrier` that provides `wait_timeout`. +//! +//! This implementation mirrors that of the Rust standard library. + +use crate::loom::sync::{Condvar, Mutex}; +use std::fmt; +use std::time::{Duration, Instant}; + +/// A barrier enables multiple threads to synchronize the beginning +/// of some computation. +/// +/// # Examples +/// +/// ``` +/// use std::sync::{Arc, Barrier}; +/// use std::thread; +/// +/// let mut handles = Vec::with_capacity(10); +/// let barrier = Arc::new(Barrier::new(10)); +/// for _ in 0..10 { +/// let c = Arc::clone(&barrier); +/// // The same messages will be printed together. +/// // You will NOT see any interleaving. +/// handles.push(thread::spawn(move|| { +/// println!("before wait"); +/// c.wait(); +/// println!("after wait"); +/// })); +/// } +/// // Wait for other threads to finish. +/// for handle in handles { +/// handle.join().unwrap(); +/// } +/// ``` +pub(crate) struct Barrier { + lock: Mutex, + cvar: Condvar, + num_threads: usize, +} + +// The inner state of a double barrier +struct BarrierState { + count: usize, + generation_id: usize, +} + +/// A `BarrierWaitResult` is returned by [`Barrier::wait()`] when all threads +/// in the [`Barrier`] have rendezvoused. +/// +/// # Examples +/// +/// ``` +/// use std::sync::Barrier; +/// +/// let barrier = Barrier::new(1); +/// let barrier_wait_result = barrier.wait(); +/// ``` +pub(crate) struct BarrierWaitResult(bool); + +impl fmt::Debug for Barrier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Barrier").finish_non_exhaustive() + } +} + +impl Barrier { + /// Creates a new barrier that can block a given number of threads. + /// + /// A barrier will block `n`-1 threads which call [`wait()`] and then wake + /// up all threads at once when the `n`th thread calls [`wait()`]. + /// + /// [`wait()`]: Barrier::wait + /// + /// # Examples + /// + /// ``` + /// use std::sync::Barrier; + /// + /// let barrier = Barrier::new(10); + /// ``` + #[must_use] + pub(crate) fn new(n: usize) -> Barrier { + Barrier { + lock: Mutex::new(BarrierState { + count: 0, + generation_id: 0, + }), + cvar: Condvar::new(), + num_threads: n, + } + } + + /// Blocks the current thread until all threads have rendezvoused here. + /// + /// Barriers are re-usable after all threads have rendezvoused once, and can + /// be used continuously. + /// + /// A single (arbitrary) thread will receive a [`BarrierWaitResult`] that + /// returns `true` from [`BarrierWaitResult::is_leader()`] when returning + /// from this function, and all other threads will receive a result that + /// will return `false` from [`BarrierWaitResult::is_leader()`]. + /// + /// # Examples + /// + /// ``` + /// use std::sync::{Arc, Barrier}; + /// use std::thread; + /// + /// let mut handles = Vec::with_capacity(10); + /// let barrier = Arc::new(Barrier::new(10)); + /// for _ in 0..10 { + /// let c = Arc::clone(&barrier); + /// // The same messages will be printed together. + /// // You will NOT see any interleaving. + /// handles.push(thread::spawn(move|| { + /// println!("before wait"); + /// c.wait(); + /// println!("after wait"); + /// })); + /// } + /// // Wait for other threads to finish. + /// for handle in handles { + /// handle.join().unwrap(); + /// } + /// ``` + pub(crate) fn wait(&self) -> BarrierWaitResult { + let mut lock = self.lock.lock(); + let local_gen = lock.generation_id; + lock.count += 1; + if lock.count < self.num_threads { + // We need a while loop to guard against spurious wakeups. + // https://en.wikipedia.org/wiki/Spurious_wakeup + while local_gen == lock.generation_id { + lock = self.cvar.wait(lock).unwrap(); + } + BarrierWaitResult(false) + } else { + lock.count = 0; + lock.generation_id = lock.generation_id.wrapping_add(1); + self.cvar.notify_all(); + BarrierWaitResult(true) + } + } + + /// Blocks the current thread until all threads have rendezvoused here for + /// at most `timeout` duration. + pub(crate) fn wait_timeout(&self, timeout: Duration) -> Option { + // This implementation mirrors `wait`, but with each blocking operation + // replaced by a timeout-amenable alternative. + + let deadline = Instant::now() + timeout; + + // Acquire `self.lock` with at most `timeout` duration. + let mut lock = loop { + if let Some(guard) = self.lock.try_lock() { + break guard; + } else if Instant::now() > deadline { + return None; + } else { + std::thread::yield_now(); + } + }; + + // Shrink the `timeout` to account for the time taken to acquire `lock`. + let timeout = deadline.saturating_duration_since(Instant::now()); + + let local_gen = lock.generation_id; + lock.count += 1; + if lock.count < self.num_threads { + // We need a while loop to guard against spurious wakeups. + // https://en.wikipedia.org/wiki/Spurious_wakeup + while local_gen == lock.generation_id { + let (guard, timeout_result) = self.cvar.wait_timeout(lock, timeout).unwrap(); + lock = guard; + if timeout_result.timed_out() { + return None; + } + } + Some(BarrierWaitResult(false)) + } else { + lock.count = 0; + lock.generation_id = lock.generation_id.wrapping_add(1); + self.cvar.notify_all(); + Some(BarrierWaitResult(true)) + } + } +} + +impl fmt::Debug for BarrierWaitResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BarrierWaitResult") + .field("is_leader", &self.is_leader()) + .finish() + } +} + +impl BarrierWaitResult { + /// Returns `true` if this thread is the "leader thread" for the call to + /// [`Barrier::wait()`]. + /// + /// Only one thread will have `true` returned from their result, all other + /// threads will have `false` returned. + /// + /// # Examples + /// + /// ``` + /// use std::sync::Barrier; + /// + /// let barrier = Barrier::new(1); + /// let barrier_wait_result = barrier.wait(); + /// println!("{:?}", barrier_wait_result.is_leader()); + /// ``` + #[must_use] + pub(crate) fn is_leader(&self) -> bool { + self.0 + } +} diff --git a/third_party/rust/tokio/src/loom/std/mod.rs b/third_party/rust/tokio/src/loom/std/mod.rs index 0c70bee74eb4..0a732791f767 100644 --- a/third_party/rust/tokio/src/loom/std/mod.rs +++ b/third_party/rust/tokio/src/loom/std/mod.rs @@ -1,11 +1,10 @@ #![cfg_attr(any(not(feature = "full"), loom), allow(unused_imports, dead_code))] -mod atomic_ptr; mod atomic_u16; mod atomic_u32; mod atomic_u64; -mod atomic_u8; mod atomic_usize; +mod barrier; mod mutex; #[cfg(feature = "parking_lot")] mod parking_lot; @@ -71,21 +70,41 @@ pub(crate) mod sync { pub(crate) use crate::loom::std::mutex::Mutex; pub(crate) mod atomic { - pub(crate) use crate::loom::std::atomic_ptr::AtomicPtr; pub(crate) use crate::loom::std::atomic_u16::AtomicU16; pub(crate) use crate::loom::std::atomic_u32::AtomicU32; - pub(crate) use crate::loom::std::atomic_u64::AtomicU64; - pub(crate) use crate::loom::std::atomic_u8::AtomicU8; + pub(crate) use crate::loom::std::atomic_u64::{AtomicU64, StaticAtomicU64}; pub(crate) use crate::loom::std::atomic_usize::AtomicUsize; - pub(crate) use std::sync::atomic::{fence, AtomicBool, Ordering}; + pub(crate) use std::sync::atomic::{fence, AtomicBool, AtomicPtr, AtomicU8, Ordering}; } + + pub(crate) use super::barrier::Barrier; } pub(crate) mod sys { #[cfg(feature = "rt-multi-thread")] pub(crate) fn num_cpus() -> usize { - usize::max(1, num_cpus::get()) + const ENV_WORKER_THREADS: &str = "TOKIO_WORKER_THREADS"; + + match std::env::var(ENV_WORKER_THREADS) { + Ok(s) => { + let n = s.parse().unwrap_or_else(|e| { + panic!( + "\"{}\" must be usize, error: {}, value: {}", + ENV_WORKER_THREADS, e, s + ) + }); + assert!(n > 0, "\"{}\" cannot be set to 0", ENV_WORKER_THREADS); + n + } + Err(std::env::VarError::NotPresent) => usize::max(1, num_cpus::get()), + Err(std::env::VarError::NotUnicode(e)) => { + panic!( + "\"{}\" must be valid unicode, error: {:?}", + ENV_WORKER_THREADS, e + ) + } + } } #[cfg(not(feature = "rt-multi-thread"))] @@ -102,7 +121,7 @@ pub(crate) mod thread { #[allow(unused_imports)] pub(crate) use std::thread::{ - current, panicking, park, park_timeout, sleep, spawn, Builder, JoinHandle, LocalKey, - Result, Thread, ThreadId, + current, panicking, park, park_timeout, sleep, spawn, AccessError, Builder, JoinHandle, + LocalKey, Result, Thread, ThreadId, }; } diff --git a/third_party/rust/tokio/src/loom/std/mutex.rs b/third_party/rust/tokio/src/loom/std/mutex.rs index 3f686e0a78eb..076f78611044 100644 --- a/third_party/rust/tokio/src/loom/std/mutex.rs +++ b/third_party/rust/tokio/src/loom/std/mutex.rs @@ -12,6 +12,12 @@ impl Mutex { Mutex(sync::Mutex::new(t)) } + #[inline] + #[cfg(not(tokio_no_const_mutex_new))] + pub(crate) const fn const_new(t: T) -> Mutex { + Mutex(sync::Mutex::new(t)) + } + #[inline] pub(crate) fn lock(&self) -> MutexGuard<'_, T> { match self.0.lock() { diff --git a/third_party/rust/tokio/src/loom/std/parking_lot.rs b/third_party/rust/tokio/src/loom/std/parking_lot.rs index 034a0ce69a5f..e3af258d1160 100644 --- a/third_party/rust/tokio/src/loom/std/parking_lot.rs +++ b/third_party/rust/tokio/src/loom/std/parking_lot.rs @@ -52,7 +52,7 @@ impl Mutex { } #[inline] - #[cfg(all(feature = "parking_lot", not(all(loom, test)),))] + #[cfg(all(feature = "parking_lot", not(all(loom, test))))] #[cfg_attr(docsrs, doc(cfg(all(feature = "parking_lot",))))] pub(crate) const fn const_new(t: T) -> Mutex { Mutex(PhantomData, parking_lot::const_mutex(t)) diff --git a/third_party/rust/tokio/src/macros/addr_of.rs b/third_party/rust/tokio/src/macros/addr_of.rs new file mode 100644 index 000000000000..9d28158f2004 --- /dev/null +++ b/third_party/rust/tokio/src/macros/addr_of.rs @@ -0,0 +1,22 @@ +//! This module defines a macro that lets you go from a raw pointer to a struct +//! to a raw pointer to a field of the struct. + +macro_rules! generate_addr_of_methods { + ( + impl<$($gen:ident)*> $struct_name:ty {$( + $(#[$attrs:meta])* + $vis:vis unsafe fn $fn_name:ident(self: NonNull) -> NonNull<$field_type:ty> { + &self$(.$field_name:tt)+ + } + )*} + ) => { + impl<$($gen)*> $struct_name {$( + $(#[$attrs])* + $vis unsafe fn $fn_name(me: ::core::ptr::NonNull) -> ::core::ptr::NonNull<$field_type> { + let me = me.as_ptr(); + let field = ::std::ptr::addr_of_mut!((*me) $(.$field_name)+ ); + ::core::ptr::NonNull::new_unchecked(field) + } + )*} + }; +} diff --git a/third_party/rust/tokio/src/macros/cfg.rs b/third_party/rust/tokio/src/macros/cfg.rs index b6beb3d6952b..52ffc102bf3f 100644 --- a/third_party/rust/tokio/src/macros/cfg.rs +++ b/third_party/rust/tokio/src/macros/cfg.rs @@ -13,6 +13,18 @@ macro_rules! feature { } } +/// Enables Windows-specific code. +/// Use this macro instead of `cfg(windows)` to generate docs properly. +macro_rules! cfg_windows { + ($($item:item)*) => { + $( + #[cfg(any(all(doc, docsrs), windows))] + #[cfg_attr(docsrs, doc(cfg(windows)))] + $item + )* + } +} + /// Enables enter::block_on. macro_rules! cfg_block_on { ($($item:item)*) => { @@ -61,6 +73,7 @@ macro_rules! cfg_fs { ($($item:item)*) => { $( #[cfg(feature = "fs")] + #[cfg(not(tokio_wasi))] #[cfg_attr(docsrs, doc(cfg(feature = "fs")))] $item )* @@ -69,7 +82,11 @@ macro_rules! cfg_fs { macro_rules! cfg_io_blocking { ($($item:item)*) => { - $( #[cfg(any(feature = "io-std", feature = "fs"))] $item )* + $( #[cfg(any( + feature = "io-std", + feature = "fs", + all(windows, feature = "process"), + ))] $item )* } } @@ -78,12 +95,12 @@ macro_rules! cfg_io_driver { $( #[cfg(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), ))] #[cfg_attr(docsrs, doc(cfg(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), ))))] $item @@ -96,7 +113,7 @@ macro_rules! cfg_io_driver_impl { $( #[cfg(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), ))] $item @@ -109,7 +126,7 @@ macro_rules! cfg_not_io_driver { $( #[cfg(not(any( feature = "net", - feature = "process", + all(unix, feature = "process"), all(unix, feature = "signal"), )))] $item @@ -195,6 +212,22 @@ macro_rules! cfg_not_metrics { } } +macro_rules! cfg_not_rt_and_metrics_and_net { + ($($item:item)*) => { + $( #[cfg(not(all(feature = "net", feature = "rt", all(tokio_unstable, not(loom)))))]$item )* + } +} + +macro_rules! cfg_net_or_process { + ($($item:item)*) => { + $( + #[cfg(any(feature = "net", feature = "process"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "net", feature = "process"))))] + $item + )* + } +} + macro_rules! cfg_net { ($($item:item)*) => { $( @@ -231,6 +264,7 @@ macro_rules! cfg_process { #[cfg(feature = "process")] #[cfg_attr(docsrs, doc(cfg(feature = "process")))] #[cfg(not(loom))] + #[cfg(not(tokio_wasi))] $item )* } @@ -259,6 +293,7 @@ macro_rules! cfg_signal { #[cfg(feature = "signal")] #[cfg_attr(docsrs, doc(cfg(feature = "signal")))] #[cfg(not(loom))] + #[cfg(not(tokio_wasi))] $item )* } @@ -274,6 +309,13 @@ macro_rules! cfg_signal_internal { } } +macro_rules! cfg_signal_internal_and_unix { + ($($item:item)*) => { + #[cfg(unix)] + cfg_signal_internal! { $($item)* } + } +} + macro_rules! cfg_not_signal_internal { ($($item:item)*) => { $( @@ -318,7 +360,7 @@ macro_rules! cfg_not_rt { macro_rules! cfg_rt_multi_thread { ($($item:item)*) => { $( - #[cfg(feature = "rt-multi-thread")] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] $item )* @@ -331,6 +373,44 @@ macro_rules! cfg_not_rt_multi_thread { } } +macro_rules! cfg_taskdump { + ($($item:item)*) => { + $( + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64" + ) + ))] + $item + )* + }; +} + +macro_rules! cfg_not_taskdump { + ($($item:item)*) => { + $( + #[cfg(not(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64" + ) + )))] + $item + )* + }; +} + macro_rules! cfg_test_util { ($($item:item)*) => { $( @@ -431,12 +511,14 @@ macro_rules! cfg_not_coop { macro_rules! cfg_has_atomic_u64 { ($($item:item)*) => { $( - #[cfg(not(any( - target_arch = "arm", - target_arch = "mips", - target_arch = "powerpc", - target_arch = "riscv32" - )))] + #[cfg_attr( + not(tokio_no_target_has_atomic), + cfg(all(target_has_atomic = "64", not(tokio_no_atomic_u64)) + ))] + #[cfg_attr( + tokio_no_target_has_atomic, + cfg(not(tokio_no_atomic_u64)) + )] $item )* } @@ -445,12 +527,62 @@ macro_rules! cfg_has_atomic_u64 { macro_rules! cfg_not_has_atomic_u64 { ($($item:item)*) => { $( - #[cfg(any( - target_arch = "arm", - target_arch = "mips", - target_arch = "powerpc", - target_arch = "riscv32" - ))] + #[cfg_attr( + not(tokio_no_target_has_atomic), + cfg(any(not(target_has_atomic = "64"), tokio_no_atomic_u64) + ))] + #[cfg_attr( + tokio_no_target_has_atomic, + cfg(tokio_no_atomic_u64) + )] + $item + )* + } +} + +macro_rules! cfg_has_const_mutex_new { + ($($item:item)*) => { + $( + #[cfg(all( + not(all(loom, test)), + any( + feature = "parking_lot", + not(tokio_no_const_mutex_new) + ) + ))] + $item + )* + } +} + +macro_rules! cfg_not_has_const_mutex_new { + ($($item:item)*) => { + $( + #[cfg(not(all( + not(all(loom, test)), + any( + feature = "parking_lot", + not(tokio_no_const_mutex_new) + ) + )))] + $item + )* + } +} + +macro_rules! cfg_not_wasi { + ($($item:item)*) => { + $( + #[cfg(not(tokio_wasi))] + $item + )* + } +} + +macro_rules! cfg_is_wasm_not_wasi { + ($($item:item)*) => { + $( + #[cfg(tokio_wasm_not_wasi)] $item )* } diff --git a/third_party/rust/tokio/src/macros/join.rs b/third_party/rust/tokio/src/macros/join.rs index f91b5f191454..8a0198600b2a 100644 --- a/third_party/rust/tokio/src/macros/join.rs +++ b/third_party/rust/tokio/src/macros/join.rs @@ -12,7 +12,7 @@ /// for **all** branches complete regardless if any complete with `Err`. Use /// [`try_join!`] to return early when `Err` is encountered. /// -/// [`try_join!`]: macro@try_join +/// [`try_join!`]: crate::try_join /// /// # Notes /// @@ -60,6 +60,9 @@ macro_rules! join { // normalization is complete. ( $($count:tt)* ) + // The expression `0+1+1+ ... +1` equal to the number of branches. + ( $($total:tt)* ) + // Normalized join! branches $( ( $($skip:tt)* ) $e:expr, )* @@ -69,24 +72,66 @@ macro_rules! join { // Safety: nothing must be moved out of `futures`. This is to satisfy // the requirement of `Pin::new_unchecked` called below. + // + // We can't use the `pin!` macro for this because `futures` is a tuple + // and the standard library provides no way to pin-project to the fields + // of a tuple. let mut futures = ( $( maybe_done($e), )* ); + // This assignment makes sure that the `poll_fn` closure only has a + // reference to the futures, instead of taking ownership of them. This + // mitigates the issue described in + // + let mut futures = &mut futures; + + // Each time the future created by poll_fn is polled, a different future will be polled first + // to ensure every future passed to join! gets a chance to make progress even if + // one of the futures consumes the whole budget. + // + // This is number of futures that will be skipped in the first loop + // iteration the next time. + let mut skip_next_time: u32 = 0; + poll_fn(move |cx| { + const COUNT: u32 = $($total)*; + let mut is_pending = false; + let mut to_run = COUNT; + + // The number of futures that will be skipped in the first loop iteration. + let mut skip = skip_next_time; + + skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 }; + + // This loop runs twice and the first `skip` futures + // are not polled in the first iteration. + loop { $( - // Extract the future for this branch from the tuple. - let ( $($skip,)* fut, .. ) = &mut futures; + if skip == 0 { + if to_run == 0 { + // Every future has been polled + break; + } + to_run -= 1; - // Safety: future is stored on the stack above - // and never moved. - let mut fut = unsafe { Pin::new_unchecked(fut) }; + // Extract the future for this branch from the tuple. + let ( $($skip,)* fut, .. ) = &mut *futures; - // Try polling - if fut.poll(cx).is_pending() { - is_pending = true; + // Safety: future is stored on the stack above + // and never moved. + let mut fut = unsafe { Pin::new_unchecked(fut) }; + + // Try polling + if fut.poll(cx).is_pending() { + is_pending = true; + } + } else { + // Future skipped, one less future to skip in the next iteration + skip -= 1; } )* + } if is_pending { Pending @@ -107,13 +152,15 @@ macro_rules! join { // ===== Normalize ===== - (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { - $crate::join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*) + (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { + $crate::join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*) }; // ===== Entry point ===== - ( $($e:expr),* $(,)?) => { - $crate::join!(@{ () } $($e,)*) + ( $($e:expr),+ $(,)?) => { + $crate::join!(@{ () (0) } $($e,)*) }; + + () => { async {}.await } } diff --git a/third_party/rust/tokio/src/macros/loom.rs b/third_party/rust/tokio/src/macros/loom.rs index d57d9fb0f7b2..fa2653ce7d61 100644 --- a/third_party/rust/tokio/src/macros/loom.rs +++ b/third_party/rust/tokio/src/macros/loom.rs @@ -1,11 +1,7 @@ macro_rules! if_loom { ($($t:tt)*) => {{ #[cfg(loom)] - const LOOM: bool = true; - #[cfg(not(loom))] - const LOOM: bool = false; - - if LOOM { + { $($t)* } }} diff --git a/third_party/rust/tokio/src/macros/mod.rs b/third_party/rust/tokio/src/macros/mod.rs index a1839c830528..82f42dbff35e 100644 --- a/third_party/rust/tokio/src/macros/mod.rs +++ b/third_party/rust/tokio/src/macros/mod.rs @@ -15,15 +15,14 @@ mod ready; #[macro_use] mod thread_local; +#[macro_use] +mod addr_of; + cfg_trace! { #[macro_use] mod trace; } -#[macro_use] -#[cfg(feature = "rt")] -pub(crate) mod scoped_tls; - cfg_macros! { #[macro_use] mod select; diff --git a/third_party/rust/tokio/src/macros/scoped_tls.rs b/third_party/rust/tokio/src/macros/scoped_tls.rs deleted file mode 100644 index f2504cbadf7f..000000000000 --- a/third_party/rust/tokio/src/macros/scoped_tls.rs +++ /dev/null @@ -1,77 +0,0 @@ -use crate::loom::thread::LocalKey; - -use std::cell::Cell; -use std::marker; - -/// Sets a reference as a thread-local. -macro_rules! scoped_thread_local { - ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty) => ( - $(#[$attrs])* - $vis static $name: $crate::macros::scoped_tls::ScopedKey<$ty> - = $crate::macros::scoped_tls::ScopedKey { - inner: { - thread_local!(static FOO: ::std::cell::Cell<*const ()> = { - std::cell::Cell::new(::std::ptr::null()) - }); - &FOO - }, - _marker: ::std::marker::PhantomData, - }; - ) -} - -/// Type representing a thread local storage key corresponding to a reference -/// to the type parameter `T`. -pub(crate) struct ScopedKey { - pub(crate) inner: &'static LocalKey>, - pub(crate) _marker: marker::PhantomData, -} - -unsafe impl Sync for ScopedKey {} - -impl ScopedKey { - /// Inserts a value into this scoped thread local storage slot for a - /// duration of a closure. - pub(crate) fn set(&'static self, t: &T, f: F) -> R - where - F: FnOnce() -> R, - { - struct Reset { - key: &'static LocalKey>, - val: *const (), - } - - impl Drop for Reset { - fn drop(&mut self) { - self.key.with(|c| c.set(self.val)); - } - } - - let prev = self.inner.with(|c| { - let prev = c.get(); - c.set(t as *const _ as *const ()); - prev - }); - - let _reset = Reset { - key: self.inner, - val: prev, - }; - - f() - } - - /// Gets a value out of this scoped variable. - pub(crate) fn with(&'static self, f: F) -> R - where - F: FnOnce(Option<&T>) -> R, - { - let val = self.inner.with(|c| c.get()); - - if val.is_null() { - f(None) - } else { - unsafe { f(Some(&*(val as *const T))) } - } - } -} diff --git a/third_party/rust/tokio/src/macros/select.rs b/third_party/rust/tokio/src/macros/select.rs index 051f8cb72a8c..31c9b3ac2e5c 100644 --- a/third_party/rust/tokio/src/macros/select.rs +++ b/third_party/rust/tokio/src/macros/select.rs @@ -101,6 +101,7 @@ /// * [`tokio::sync::watch::Receiver::changed`](crate::sync::watch::Receiver::changed) /// * [`tokio::net::TcpListener::accept`](crate::net::TcpListener::accept) /// * [`tokio::net::UnixListener::accept`](crate::net::UnixListener::accept) +/// * [`tokio::signal::unix::Signal::recv`](crate::signal::unix::Signal::recv) /// * [`tokio::io::AsyncReadExt::read`](crate::io::AsyncReadExt::read) on any `AsyncRead` /// * [`tokio::io::AsyncReadExt::read_buf`](crate::io::AsyncReadExt::read_buf) on any `AsyncRead` /// * [`tokio::io::AsyncWriteExt::write`](crate::io::AsyncWriteExt::write) on any `AsyncWrite` @@ -130,6 +131,13 @@ /// correctly even if it is restarted while waiting at an `.await`, then it is /// cancellation safe. /// +/// Cancellation safety can be defined in the following way: If you have a +/// future that has not yet completed, then it must be a no-op to drop that +/// future and recreate it. This definition is motivated by the situation where +/// a `select!` is used in a loop. Without this guarantee, you would lose your +/// progress when another branch completes and you restart the `select!` by +/// going around the loop. +/// /// Be aware that cancelling something that is not cancellation safe is not /// necessarily wrong. For example, if you are cancelling a task because the /// application is shutting down, then you probably don't care that partially @@ -429,7 +437,8 @@ macro_rules! select { // // This module is defined within a scope and should not leak out of this // macro. - mod util { + #[doc(hidden)] + mod __tokio_select_util { // Generate an enum with one variant per select branch $crate::select_priv_declare_output_enum!( ( $($count)* ) ); } @@ -442,13 +451,13 @@ macro_rules! select { const BRANCHES: u32 = $crate::count!( $($count)* ); - let mut disabled: util::Mask = Default::default(); + let mut disabled: __tokio_select_util::Mask = Default::default(); // First, invoke all the pre-conditions. For any that return true, // set the appropriate bit in `disabled`. $( if !$c { - let mask: util::Mask = 1 << $crate::count!( $($skip)* ); + let mask: __tokio_select_util::Mask = 1 << $crate::count!( $($skip)* ); disabled |= mask; } )* @@ -458,8 +467,18 @@ macro_rules! select { let mut output = { // Safety: Nothing must be moved out of `futures`. This is to // satisfy the requirement of `Pin::new_unchecked` called below. + // + // We can't use the `pin!` macro for this because `futures` is a + // tuple and the standard library provides no way to pin-project to + // the fields of a tuple. let mut futures = ( $( $fut , )+ ); + // This assignment makes sure that the `poll_fn` closure only has a + // reference to the futures, instead of taking ownership of them. + // This mitigates the issue described in + // + let mut futures = &mut futures; + $crate::macros::support::poll_fn(|cx| { // Track if any branch returns pending. If no branch completes // **or** returns pending, this implies that all branches are @@ -495,7 +514,7 @@ macro_rules! select { // Extract the future for this branch from the // tuple - let ( $($skip,)* fut, .. ) = &mut futures; + let ( $($skip,)* fut, .. ) = &mut *futures; // Safety: future is stored on the stack above // and never moved. @@ -525,7 +544,7 @@ macro_rules! select { } // The select is complete, return the value - return Ready($crate::select_variant!(util::Out, ($($skip)*))(out)); + return Ready($crate::select_variant!(__tokio_select_util::Out, ($($skip)*))(out)); } )* _ => unreachable!("reaching this means there probably is an off by one bug"), @@ -536,16 +555,16 @@ macro_rules! select { Pending } else { // All branches have been disabled. - Ready(util::Out::Disabled) + Ready(__tokio_select_util::Out::Disabled) } }).await }; match output { $( - $crate::select_variant!(util::Out, ($($skip)*) ($bind)) => $handle, + $crate::select_variant!(__tokio_select_util::Out, ($($skip)*) ($bind)) => $handle, )* - util::Out::Disabled => $else, + __tokio_select_util::Out::Disabled => $else, _ => unreachable!("failed to match bind"), } }}; @@ -801,6 +820,9 @@ macro_rules! count { (_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _) => { 63 }; + (_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _) => { + 64 + }; } #[macro_export] diff --git a/third_party/rust/tokio/src/macros/support.rs b/third_party/rust/tokio/src/macros/support.rs index 7f11bc68001a..10526bcbca7f 100644 --- a/third_party/rust/tokio/src/macros/support.rs +++ b/third_party/rust/tokio/src/macros/support.rs @@ -1,7 +1,11 @@ cfg_macros! { pub use crate::future::poll_fn; pub use crate::future::maybe_done::maybe_done; - pub use crate::util::thread_rng_n; + + #[doc(hidden)] + pub fn thread_rng_n(n: u32) -> u32 { + crate::runtime::context::thread_rng_n(n) + } } pub use std::future::Future; diff --git a/third_party/rust/tokio/src/macros/thread_local.rs b/third_party/rust/tokio/src/macros/thread_local.rs index d848947350dd..74be99abf65b 100644 --- a/third_party/rust/tokio/src/macros/thread_local.rs +++ b/third_party/rust/tokio/src/macros/thread_local.rs @@ -1,4 +1,32 @@ #[cfg(all(loom, test))] -macro_rules! thread_local { +macro_rules! tokio_thread_local { + ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => { + loom::thread_local! { + $(#[$attrs])* + $vis static $name: $ty = $expr; + } + }; + ($($tts:tt)+) => { loom::thread_local!{ $($tts)+ } } } + +#[cfg(not(tokio_no_const_thread_local))] +#[cfg(not(all(loom, test)))] +macro_rules! tokio_thread_local { + ($($tts:tt)+) => { + ::std::thread_local!{ $($tts)+ } + } +} + +#[cfg(tokio_no_const_thread_local)] +#[cfg(not(all(loom, test)))] +macro_rules! tokio_thread_local { + ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => { + ::std::thread_local! { + $(#[$attrs])* + $vis static $name: $ty = $expr; + } + }; + + ($($tts:tt)+) => { ::std::thread_local!{ $($tts)+ } } +} diff --git a/third_party/rust/tokio/src/macros/try_join.rs b/third_party/rust/tokio/src/macros/try_join.rs index 6d3a893b7e14..7b1237092310 100644 --- a/third_party/rust/tokio/src/macros/try_join.rs +++ b/third_party/rust/tokio/src/macros/try_join.rs @@ -106,6 +106,9 @@ macro_rules! try_join { // normalization is complete. ( $($count:tt)* ) + // The expression `0+1+1+ ... +1` equal to the number of branches. + ( $($total:tt)* ) + // Normalized try_join! branches $( ( $($skip:tt)* ) $e:expr, )* @@ -115,26 +118,68 @@ macro_rules! try_join { // Safety: nothing must be moved out of `futures`. This is to satisfy // the requirement of `Pin::new_unchecked` called below. + // + // We can't use the `pin!` macro for this because `futures` is a tuple + // and the standard library provides no way to pin-project to the fields + // of a tuple. let mut futures = ( $( maybe_done($e), )* ); + // This assignment makes sure that the `poll_fn` closure only has a + // reference to the futures, instead of taking ownership of them. This + // mitigates the issue described in + // + let mut futures = &mut futures; + + // Each time the future created by poll_fn is polled, a different future will be polled first + // to ensure every future passed to join! gets a chance to make progress even if + // one of the futures consumes the whole budget. + // + // This is number of futures that will be skipped in the first loop + // iteration the next time. + let mut skip_next_time: u32 = 0; + poll_fn(move |cx| { + const COUNT: u32 = $($total)*; + let mut is_pending = false; + let mut to_run = COUNT; + + // The number of futures that will be skipped in the first loop iteration + let mut skip = skip_next_time; + + skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 }; + + // This loop runs twice and the first `skip` futures + // are not polled in the first iteration. + loop { $( - // Extract the future for this branch from the tuple. - let ( $($skip,)* fut, .. ) = &mut futures; + if skip == 0 { + if to_run == 0 { + // Every future has been polled + break; + } + to_run -= 1; - // Safety: future is stored on the stack above - // and never moved. - let mut fut = unsafe { Pin::new_unchecked(fut) }; + // Extract the future for this branch from the tuple. + let ( $($skip,)* fut, .. ) = &mut *futures; - // Try polling - if fut.as_mut().poll(cx).is_pending() { - is_pending = true; - } else if fut.as_mut().output_mut().expect("expected completed future").is_err() { - return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap())) + // Safety: future is stored on the stack above + // and never moved. + let mut fut = unsafe { Pin::new_unchecked(fut) }; + + // Try polling + if fut.as_mut().poll(cx).is_pending() { + is_pending = true; + } else if fut.as_mut().output_mut().expect("expected completed future").is_err() { + return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap())) + } + } else { + // Future skipped, one less future to skip in the next iteration + skip -= 1; } )* + } if is_pending { Pending @@ -159,13 +204,15 @@ macro_rules! try_join { // ===== Normalize ===== - (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { - $crate::try_join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*) + (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => { + $crate::try_join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*) }; // ===== Entry point ===== - ( $($e:expr),* $(,)?) => { - $crate::try_join!(@{ () } $($e,)*) + ( $($e:expr),+ $(,)?) => { + $crate::try_join!(@{ () (0) } $($e,)*) }; + + () => { async { Ok(()) }.await } } diff --git a/third_party/rust/tokio/src/net/addr.rs b/third_party/rust/tokio/src/net/addr.rs index 13f743c96285..fb8248fb3ba2 100644 --- a/third_party/rust/tokio/src/net/addr.rs +++ b/third_party/rust/tokio/src/net/addr.rs @@ -136,7 +136,22 @@ impl sealed::ToSocketAddrsPriv for &[SocketAddr] { type Future = ReadyFuture; fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future { - let iter = self.to_vec().into_iter(); + #[inline] + fn slice_to_vec(addrs: &[SocketAddr]) -> Vec { + addrs.to_vec() + } + + // This uses a helper method because clippy doesn't like the `to_vec()` + // call here (it will allocate, whereas `self.iter().copied()` would + // not), but it's actually necessary in order to ensure that the + // returned iterator is valid for the `'static` lifetime, which the + // borrowed `slice::Iter` iterator would not be. + // + // Note that we can't actually add an `allow` attribute for + // `clippy::unnecessary_to_owned` here, as Tokio's CI runs clippy lints + // on Rust 1.52 to avoid breaking LTS releases of Tokio. Users of newer + // Rust versions who see this lint should just ignore it. + let iter = slice_to_vec(self).into_iter(); future::ready(Ok(iter)) } } @@ -229,7 +244,7 @@ cfg_net! { type Future = ::Future; fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future { - (&self[..]).to_socket_addrs(sealed::Internal) + self[..].to_socket_addrs(sealed::Internal) } } } diff --git a/third_party/rust/tokio/src/net/mod.rs b/third_party/rust/tokio/src/net/mod.rs index 0b8c1ecd1945..2d317a8a219c 100644 --- a/third_party/rust/tokio/src/net/mod.rs +++ b/third_party/rust/tokio/src/net/mod.rs @@ -23,8 +23,10 @@ //! [`UnixDatagram`]: UnixDatagram mod addr; -#[cfg(feature = "net")] -pub(crate) use addr::to_socket_addrs; +cfg_not_wasi! { + #[cfg(feature = "net")] + pub(crate) use addr::to_socket_addrs; +} pub use addr::ToSocketAddrs; cfg_net! { @@ -33,11 +35,13 @@ cfg_net! { pub mod tcp; pub use tcp::listener::TcpListener; - pub use tcp::socket::TcpSocket; pub use tcp::stream::TcpStream; + cfg_not_wasi! { + pub use tcp::socket::TcpSocket; - mod udp; - pub use udp::UdpSocket; + mod udp; + pub use udp::UdpSocket; + } } cfg_net_unix! { diff --git a/third_party/rust/tokio/src/net/tcp/listener.rs b/third_party/rust/tokio/src/net/tcp/listener.rs index 8aecb21aaa94..34e393c895f8 100644 --- a/third_party/rust/tokio/src/net/tcp/listener.rs +++ b/third_party/rust/tokio/src/net/tcp/listener.rs @@ -1,8 +1,10 @@ use crate::io::{Interest, PollEvented}; use crate::net::tcp::TcpStream; -use crate::net::{to_socket_addrs, ToSocketAddrs}; -use std::convert::TryFrom; +cfg_not_wasi! { + use crate::net::{to_socket_addrs, ToSocketAddrs}; +} + use std::fmt; use std::io; use std::net::{self, SocketAddr}; @@ -55,68 +57,70 @@ cfg_net! { } impl TcpListener { - /// Creates a new TcpListener, which will be bound to the specified address. - /// - /// The returned listener is ready for accepting connections. - /// - /// Binding with a port number of 0 will request that the OS assigns a port - /// to this listener. The port allocated can be queried via the `local_addr` - /// method. - /// - /// The address type can be any implementor of the [`ToSocketAddrs`] trait. - /// If `addr` yields multiple addresses, bind will be attempted with each of - /// the addresses until one succeeds and returns the listener. If none of - /// the addresses succeed in creating a listener, the error returned from - /// the last attempt (the last address) is returned. - /// - /// This function sets the `SO_REUSEADDR` option on the socket. - /// - /// To configure the socket before binding, you can use the [`TcpSocket`] - /// type. - /// - /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs - /// [`TcpSocket`]: struct@crate::net::TcpSocket - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpListener; - /// - /// use std::io; - /// - /// #[tokio::main] - /// async fn main() -> io::Result<()> { - /// let listener = TcpListener::bind("127.0.0.1:2345").await?; - /// - /// // use the listener - /// - /// # let _ = listener; - /// Ok(()) - /// } - /// ``` - pub async fn bind(addr: A) -> io::Result { - let addrs = to_socket_addrs(addr).await?; + cfg_not_wasi! { + /// Creates a new TcpListener, which will be bound to the specified address. + /// + /// The returned listener is ready for accepting connections. + /// + /// Binding with a port number of 0 will request that the OS assigns a port + /// to this listener. The port allocated can be queried via the `local_addr` + /// method. + /// + /// The address type can be any implementor of the [`ToSocketAddrs`] trait. + /// If `addr` yields multiple addresses, bind will be attempted with each of + /// the addresses until one succeeds and returns the listener. If none of + /// the addresses succeed in creating a listener, the error returned from + /// the last attempt (the last address) is returned. + /// + /// This function sets the `SO_REUSEADDR` option on the socket. + /// + /// To configure the socket before binding, you can use the [`TcpSocket`] + /// type. + /// + /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs + /// [`TcpSocket`]: struct@crate::net::TcpSocket + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpListener; + /// + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// let listener = TcpListener::bind("127.0.0.1:2345").await?; + /// + /// // use the listener + /// + /// # let _ = listener; + /// Ok(()) + /// } + /// ``` + pub async fn bind(addr: A) -> io::Result { + let addrs = to_socket_addrs(addr).await?; - let mut last_err = None; + let mut last_err = None; - for addr in addrs { - match TcpListener::bind_addr(addr) { - Ok(listener) => return Ok(listener), - Err(e) => last_err = Some(e), + for addr in addrs { + match TcpListener::bind_addr(addr) { + Ok(listener) => return Ok(listener), + Err(e) => last_err = Some(e), + } } + + Err(last_err.unwrap_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "could not resolve to any address", + ) + })) } - Err(last_err.unwrap_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "could not resolve to any address", - ) - })) - } - - fn bind_addr(addr: SocketAddr) -> io::Result { - let listener = mio::net::TcpListener::bind(addr)?; - TcpListener::new(listener) + fn bind_addr(addr: SocketAddr) -> io::Result { + let listener = mio::net::TcpListener::bind(addr)?; + TcpListener::new(listener) + } } /// Accepts a new incoming connection from this listener. @@ -190,15 +194,22 @@ impl TcpListener { /// Creates new `TcpListener` from a `std::net::TcpListener`. /// /// This function is intended to be used to wrap a TCP listener from the - /// standard library in the Tokio equivalent. The conversion assumes nothing - /// about the underlying listener; it is left up to the user to set it in - /// non-blocking mode. + /// standard library in the Tokio equivalent. /// /// This API is typically paired with the `socket2` crate and the `Socket` /// type to build up and customize a listener before it's shipped off to the /// backing event loop. This allows configuration of options like /// `SO_REUSEPORT`, binding to multiple addresses, etc. /// + /// # Notes + /// + /// The caller is responsible for ensuring that the listener is in + /// non-blocking mode. Otherwise all I/O operations on the listener + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::TcpListener::set_nonblocking + /// /// # Examples /// /// ```rust,no_run @@ -216,11 +227,13 @@ impl TcpListener { /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(listener: net::TcpListener) -> io::Result { let io = mio::net::TcpListener::from_std(listener); let io = PollEvented::new(io)?; @@ -267,11 +280,22 @@ impl TcpListener { .map(|io| io.into_raw_socket()) .map(|raw_socket| unsafe { std::net::TcpListener::from_raw_socket(raw_socket) }) } + + #[cfg(tokio_wasi)] + { + use std::os::wasi::io::{FromRawFd, IntoRawFd}; + self.io + .into_inner() + .map(|io| io.into_raw_fd()) + .map(|raw_fd| unsafe { std::net::TcpListener::from_raw_fd(raw_fd) }) + } } - pub(crate) fn new(listener: mio::net::TcpListener) -> io::Result { - let io = PollEvented::new(listener)?; - Ok(TcpListener { io }) + cfg_not_wasi! { + pub(crate) fn new(listener: mio::net::TcpListener) -> io::Result { + let io = PollEvented::new(listener)?; + Ok(TcpListener { io }) + } } /// Returns the local address that this listener is bound to. @@ -382,16 +406,51 @@ mod sys { self.io.as_raw_fd() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for TcpListener { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } } -#[cfg(windows)] -mod sys { - use super::TcpListener; - use std::os::windows::prelude::*; +cfg_unstable! { + #[cfg(tokio_wasi)] + mod sys { + use super::TcpListener; + use std::os::wasi::prelude::*; + + impl AsRawFd for TcpListener { + fn as_raw_fd(&self) -> RawFd { + self.io.as_raw_fd() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for TcpListener { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } + } +} + +cfg_windows! { + use crate::os::windows::io::{AsRawSocket, RawSocket}; + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsSocket, BorrowedSocket}; impl AsRawSocket for TcpListener { fn as_raw_socket(&self) -> RawSocket { self.io.as_raw_socket() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsSocket for TcpListener { + fn as_socket(&self) -> BorrowedSocket<'_> { + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } + } } diff --git a/third_party/rust/tokio/src/net/tcp/mod.rs b/third_party/rust/tokio/src/net/tcp/mod.rs index cb8a8b238bef..734eabe6dda7 100644 --- a/third_party/rust/tokio/src/net/tcp/mod.rs +++ b/third_party/rust/tokio/src/net/tcp/mod.rs @@ -2,7 +2,9 @@ pub(crate) mod listener; -pub(crate) mod socket; +cfg_not_wasi! { + pub(crate) mod socket; +} mod split; pub use split::{ReadHalf, WriteHalf}; diff --git a/third_party/rust/tokio/src/net/tcp/socket.rs b/third_party/rust/tokio/src/net/tcp/socket.rs index 171e24018990..df792f9a6158 100644 --- a/third_party/rust/tokio/src/net/tcp/socket.rs +++ b/third_party/rust/tokio/src/net/tcp/socket.rs @@ -4,12 +4,18 @@ use std::fmt; use std::io; use std::net::SocketAddr; +#[cfg(all(unix, not(tokio_no_as_fd)))] +use std::os::unix::io::{AsFd, BorrowedFd}; #[cfg(unix)] use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; -#[cfg(windows)] -use std::os::windows::io::{AsRawSocket, FromRawSocket, IntoRawSocket, RawSocket}; use std::time::Duration; +cfg_windows! { + use crate::os::windows::io::{AsRawSocket, FromRawSocket, IntoRawSocket, RawSocket}; + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsSocket, BorrowedSocket}; +} + cfg_net! { /// A TCP socket that has not yet been converted to a `TcpStream` or /// `TcpListener`. @@ -398,6 +404,134 @@ impl TcpSocket { self.inner.linger() } + /// Sets the value of the `TCP_NODELAY` option on this socket. + /// + /// If set, this option disables the Nagle algorithm. This means that segments are always + /// sent as soon as possible, even if there is only a small amount of data. When not set, + /// data is buffered until there is a sufficient amount to send out, thereby avoiding + /// the frequent sending of small packets. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpSocket; + /// + /// # async fn dox() -> Result<(), Box> { + /// let socket = TcpSocket::new_v4()?; + /// + /// println!("{:?}", socket.nodelay()?); + /// # Ok(()) + /// # } + /// ``` + pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { + self.inner.set_nodelay(nodelay) + } + + /// Gets the value of the `TCP_NODELAY` option on this socket. + /// + /// For more information about this option, see [`set_nodelay`]. + /// + /// [`set_nodelay`]: TcpSocket::set_nodelay + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpSocket; + /// + /// # async fn dox() -> Result<(), Box> { + /// let stream = TcpSocket::new_v4()?; + /// + /// stream.set_nodelay(true)?; + /// # Ok(()) + /// # } + /// ``` + pub fn nodelay(&self) -> io::Result { + self.inner.nodelay() + } + + /// Gets the value of the `IP_TOS` option for this socket. + /// + /// For more information about this option, see [`set_tos`]. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + /// + /// [`set_tos`]: Self::set_tos + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn tos(&self) -> io::Result { + self.inner.tos() + } + + /// Sets the value for the `IP_TOS` option on this socket. + /// + /// This value sets the type-of-service field that is used in every packet + /// sent from this socket. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn set_tos(&self, tos: u32) -> io::Result<()> { + self.inner.set_tos(tos) + } + + /// Gets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// This value gets the socket binded device's interface name. + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))] + #[cfg_attr( + docsrs, + doc(cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))) + )] + pub fn device(&self) -> io::Result>> { + self.inner.device() + } + + /// Sets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// If a socket is bound to an interface, only packets received from that + /// particular interface are processed by the socket. Note that this only + /// works for some socket types, particularly `AF_INET` sockets. + /// + /// If `interface` is `None` or an empty string it removes the binding. + #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] + #[cfg_attr( + docsrs, + doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))) + )] + pub fn bind_device(&self, interface: Option<&[u8]>) -> io::Result<()> { + self.inner.bind_device(interface) + } + /// Gets the local address of this socket. /// /// Will fail on windows if called before `bind`. @@ -424,6 +558,11 @@ impl TcpSocket { self.inner.local_addr().and_then(convert_address) } + /// Returns the value of the `SO_ERROR` option. + pub fn take_error(&self) -> io::Result> { + self.inner.take_error() + } + /// Binds the socket to the given address. /// /// This calls the `bind(2)` operating-system function. Behavior is @@ -582,6 +721,15 @@ impl TcpSocket { /// [`std::net::TcpStream`]: struct@std::net::TcpStream /// [`socket2`]: https://docs.rs/socket2/ /// + /// # Notes + /// + /// The caller is responsible for ensuring that the socket is in + /// non-blocking mode. Otherwise all I/O operations on the socket + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::TcpStream::set_nonblocking + /// /// # Examples /// /// ``` @@ -590,8 +738,8 @@ impl TcpSocket { /// /// #[tokio::main] /// async fn main() -> std::io::Result<()> { - /// /// let socket2_socket = Socket::new(Domain::IPV4, Type::STREAM, None)?; + /// socket2_socket.set_nonblocking(true)?; /// /// let socket = TcpSocket::from_std_stream(socket2_socket.into()); /// @@ -640,6 +788,13 @@ impl AsRawFd for TcpSocket { } } +#[cfg(all(unix, not(tokio_no_as_fd)))] +impl AsFd for TcpSocket { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} + #[cfg(unix)] impl FromRawFd for TcpSocket { /// Converts a `RawFd` to a `TcpSocket`. @@ -661,30 +816,36 @@ impl IntoRawFd for TcpSocket { } } -#[cfg(windows)] -impl IntoRawSocket for TcpSocket { - fn into_raw_socket(self) -> RawSocket { - self.inner.into_raw_socket() +cfg_windows! { + impl IntoRawSocket for TcpSocket { + fn into_raw_socket(self) -> RawSocket { + self.inner.into_raw_socket() + } } -} -#[cfg(windows)] -impl AsRawSocket for TcpSocket { - fn as_raw_socket(&self) -> RawSocket { - self.inner.as_raw_socket() + impl AsRawSocket for TcpSocket { + fn as_raw_socket(&self) -> RawSocket { + self.inner.as_raw_socket() + } } -} -#[cfg(windows)] -impl FromRawSocket for TcpSocket { - /// Converts a `RawSocket` to a `TcpStream`. - /// - /// # Notes - /// - /// The caller is responsible for ensuring that the socket is in - /// non-blocking mode. - unsafe fn from_raw_socket(socket: RawSocket) -> TcpSocket { - let inner = socket2::Socket::from_raw_socket(socket); - TcpSocket { inner } + #[cfg(not(tokio_no_as_fd))] + impl AsSocket for TcpSocket { + fn as_socket(&self) -> BorrowedSocket<'_> { + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } + } + + impl FromRawSocket for TcpSocket { + /// Converts a `RawSocket` to a `TcpStream`. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the socket is in + /// non-blocking mode. + unsafe fn from_raw_socket(socket: RawSocket) -> TcpSocket { + let inner = socket2::Socket::from_raw_socket(socket); + TcpSocket { inner } + } } } diff --git a/third_party/rust/tokio/src/net/tcp/split.rs b/third_party/rust/tokio/src/net/tcp/split.rs index 0e02928495f0..1a1e22253f2b 100644 --- a/third_party/rust/tokio/src/net/tcp/split.rs +++ b/third_party/rust/tokio/src/net/tcp/split.rs @@ -141,12 +141,21 @@ impl ReadHalf<'_> { /// Waits for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_read()`]. It can be used instead + /// of [`readable()`] to check the returned ready set for [`Ready::READABLE`] + /// and [`Ready::READ_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// This function is equivalent to [`TcpStream::ready`]. /// + /// [`try_read()`]: Self::try_read + /// [`readable()`]: Self::readable + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -190,8 +199,12 @@ impl ReadHalf<'_> { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result { self.0.try_read(buf) @@ -265,12 +278,21 @@ impl ReadHalf<'_> { impl WriteHalf<'_> { /// Waits for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_write()`]. It can be used instead + /// of [`writable()`] to check the returned ready set for [`Ready::WRITABLE`] + /// and [`Ready::WRITE_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// This function is equivalent to [`TcpStream::ready`]. /// + /// [`try_write()`]: Self::try_write + /// [`writable()`]: Self::writable + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method diff --git a/third_party/rust/tokio/src/net/tcp/split_owned.rs b/third_party/rust/tokio/src/net/tcp/split_owned.rs index ef4e7b53611e..6771d6497a2a 100644 --- a/third_party/rust/tokio/src/net/tcp/split_owned.rs +++ b/third_party/rust/tokio/src/net/tcp/split_owned.rs @@ -196,12 +196,21 @@ impl OwnedReadHalf { /// Waits for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_read()`]. It can be used instead + /// of [`readable()`] to check the returned ready set for [`Ready::READABLE`] + /// and [`Ready::READ_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// This function is equivalent to [`TcpStream::ready`]. /// + /// [`try_read()`]: Self::try_read + /// [`readable()`]: Self::readable + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -245,8 +254,12 @@ impl OwnedReadHalf { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result { self.inner.try_read(buf) @@ -347,12 +360,21 @@ impl OwnedWriteHalf { /// Waits for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_write()`]. It can be used instead + /// of [`writable()`] to check the returned ready set for [`Ready::WRITABLE`] + /// and [`Ready::WRITE_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// This function is equivalent to [`TcpStream::ready`]. /// + /// [`try_write()`]: Self::try_write + /// [`writable()`]: Self::writable + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -474,12 +496,12 @@ impl AsyncWrite for OwnedWriteHalf { impl AsRef for OwnedReadHalf { fn as_ref(&self) -> &TcpStream { - &*self.inner + &self.inner } } impl AsRef for OwnedWriteHalf { fn as_ref(&self) -> &TcpStream { - &*self.inner + &self.inner } } diff --git a/third_party/rust/tokio/src/net/tcp/stream.rs b/third_party/rust/tokio/src/net/tcp/stream.rs index ebb67b84d16a..b7104d78b405 100644 --- a/third_party/rust/tokio/src/net/tcp/stream.rs +++ b/third_party/rust/tokio/src/net/tcp/stream.rs @@ -1,16 +1,18 @@ -use crate::future::poll_fn; +cfg_not_wasi! { + use crate::future::poll_fn; + use crate::net::{to_socket_addrs, ToSocketAddrs}; + use std::time::Duration; +} + use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready}; use crate::net::tcp::split::{split, ReadHalf, WriteHalf}; use crate::net::tcp::split_owned::{split_owned, OwnedReadHalf, OwnedWriteHalf}; -use crate::net::{to_socket_addrs, ToSocketAddrs}; -use std::convert::TryFrom; use std::fmt; use std::io; use std::net::{Shutdown, SocketAddr}; use std::pin::Pin; use std::task::{Context, Poll}; -use std::time::Duration; cfg_io_util! { use bytes::BufMut; @@ -70,86 +72,88 @@ cfg_net! { } impl TcpStream { - /// Opens a TCP connection to a remote host. - /// - /// `addr` is an address of the remote host. Anything which implements the - /// [`ToSocketAddrs`] trait can be supplied as the address. If `addr` - /// yields multiple addresses, connect will be attempted with each of the - /// addresses until a connection is successful. If none of the addresses - /// result in a successful connection, the error returned from the last - /// connection attempt (the last address) is returned. - /// - /// To configure the socket before connecting, you can use the [`TcpSocket`] - /// type. - /// - /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs - /// [`TcpSocket`]: struct@crate::net::TcpSocket - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpStream; - /// use tokio::io::AsyncWriteExt; - /// use std::error::Error; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// // Connect to a peer - /// let mut stream = TcpStream::connect("127.0.0.1:8080").await?; - /// - /// // Write some data. - /// stream.write_all(b"hello world!").await?; - /// - /// Ok(()) - /// } - /// ``` - /// - /// The [`write_all`] method is defined on the [`AsyncWriteExt`] trait. - /// - /// [`write_all`]: fn@crate::io::AsyncWriteExt::write_all - /// [`AsyncWriteExt`]: trait@crate::io::AsyncWriteExt - pub async fn connect(addr: A) -> io::Result { - let addrs = to_socket_addrs(addr).await?; + cfg_not_wasi! { + /// Opens a TCP connection to a remote host. + /// + /// `addr` is an address of the remote host. Anything which implements the + /// [`ToSocketAddrs`] trait can be supplied as the address. If `addr` + /// yields multiple addresses, connect will be attempted with each of the + /// addresses until a connection is successful. If none of the addresses + /// result in a successful connection, the error returned from the last + /// connection attempt (the last address) is returned. + /// + /// To configure the socket before connecting, you can use the [`TcpSocket`] + /// type. + /// + /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs + /// [`TcpSocket`]: struct@crate::net::TcpSocket + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpStream; + /// use tokio::io::AsyncWriteExt; + /// use std::error::Error; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// // Connect to a peer + /// let mut stream = TcpStream::connect("127.0.0.1:8080").await?; + /// + /// // Write some data. + /// stream.write_all(b"hello world!").await?; + /// + /// Ok(()) + /// } + /// ``` + /// + /// The [`write_all`] method is defined on the [`AsyncWriteExt`] trait. + /// + /// [`write_all`]: fn@crate::io::AsyncWriteExt::write_all + /// [`AsyncWriteExt`]: trait@crate::io::AsyncWriteExt + pub async fn connect(addr: A) -> io::Result { + let addrs = to_socket_addrs(addr).await?; - let mut last_err = None; + let mut last_err = None; - for addr in addrs { - match TcpStream::connect_addr(addr).await { - Ok(stream) => return Ok(stream), - Err(e) => last_err = Some(e), + for addr in addrs { + match TcpStream::connect_addr(addr).await { + Ok(stream) => return Ok(stream), + Err(e) => last_err = Some(e), + } } + + Err(last_err.unwrap_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "could not resolve to any address", + ) + })) } - Err(last_err.unwrap_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "could not resolve to any address", - ) - })) - } - - /// Establishes a connection to the specified `addr`. - async fn connect_addr(addr: SocketAddr) -> io::Result { - let sys = mio::net::TcpStream::connect(addr)?; - TcpStream::connect_mio(sys).await - } - - pub(crate) async fn connect_mio(sys: mio::net::TcpStream) -> io::Result { - let stream = TcpStream::new(sys)?; - - // Once we've connected, wait for the stream to be writable as - // that's when the actual connection has been initiated. Once we're - // writable we check for `take_socket_error` to see if the connect - // actually hit an error or not. - // - // If all that succeeded then we ship everything on up. - poll_fn(|cx| stream.io.registration().poll_write_ready(cx)).await?; - - if let Some(e) = stream.io.take_error()? { - return Err(e); + /// Establishes a connection to the specified `addr`. + async fn connect_addr(addr: SocketAddr) -> io::Result { + let sys = mio::net::TcpStream::connect(addr)?; + TcpStream::connect_mio(sys).await } - Ok(stream) + pub(crate) async fn connect_mio(sys: mio::net::TcpStream) -> io::Result { + let stream = TcpStream::new(sys)?; + + // Once we've connected, wait for the stream to be writable as + // that's when the actual connection has been initiated. Once we're + // writable we check for `take_socket_error` to see if the connect + // actually hit an error or not. + // + // If all that succeeded then we ship everything on up. + poll_fn(|cx| stream.io.registration().poll_write_ready(cx)).await?; + + if let Some(e) = stream.io.take_error()? { + return Err(e); + } + + Ok(stream) + } } pub(crate) fn new(connected: mio::net::TcpStream) -> io::Result { @@ -160,9 +164,16 @@ impl TcpStream { /// Creates new `TcpStream` from a `std::net::TcpStream`. /// /// This function is intended to be used to wrap a TCP stream from the - /// standard library in the Tokio equivalent. The conversion assumes nothing - /// about the underlying stream; it is left up to the user to set it in - /// non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the stream is in + /// non-blocking mode. Otherwise all I/O operations on the stream + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::TcpStream::set_nonblocking /// /// # Examples /// @@ -181,11 +192,13 @@ impl TcpStream { /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(stream: std::net::TcpStream) -> io::Result { let io = mio::net::TcpStream::from_std(stream); let io = PollEvented::new(io)?; @@ -244,6 +257,15 @@ impl TcpStream { .map(|io| io.into_raw_socket()) .map(|raw_socket| unsafe { std::net::TcpStream::from_raw_socket(raw_socket) }) } + + #[cfg(tokio_wasi)] + { + use std::os::wasi::io::{FromRawFd, IntoRawFd}; + self.io + .into_inner() + .map(|io| io.into_raw_fd()) + .map(|raw_fd| unsafe { std::net::TcpStream::from_raw_fd(raw_fd) }) + } } /// Returns the local address that this stream is bound to. @@ -264,6 +286,11 @@ impl TcpStream { self.io.local_addr() } + /// Returns the value of the `SO_ERROR` option. + pub fn take_error(&self) -> io::Result> { + self.io.take_error() + } + /// Returns the remote address that this stream is connected to. /// /// # Examples @@ -356,6 +383,12 @@ impl TcpStream { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -526,8 +559,12 @@ impl TcpStream { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -939,7 +976,7 @@ impl TcpStream { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -958,6 +995,11 @@ impl TcpStream { /// defined on the Tokio `TcpStream` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: TcpStream::readable() @@ -968,7 +1010,45 @@ impl TcpStream { interest: Interest, f: impl FnOnce() -> io::Result, ) -> io::Result { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) + } + + /// Reads or writes from the socket using a user-provided IO operation. + /// + /// The readiness of the socket is awaited and when the socket is ready, + /// the provided closure is called. The closure should attempt to perform + /// IO operation on the socket by manually calling the appropriate syscall. + /// If the operation fails because the socket is not actually ready, + /// then the closure should return a `WouldBlock` error. In such case the + /// readiness flag is cleared and the socket readiness is awaited again. + /// This loop is repeated until the closure returns an `Ok` or an error + /// other than `WouldBlock`. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `TcpStream` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + pub async fn async_io( + &self, + interest: Interest, + mut f: impl FnMut() -> io::Result, + ) -> io::Result { + self.io + .registration() + .async_io(interest, || self.io.try_io(&mut f)) + .await } /// Receives data on the socket from the remote address to which it is @@ -1070,52 +1150,54 @@ impl TcpStream { self.io.set_nodelay(nodelay) } - /// Reads the linger duration for this socket by getting the `SO_LINGER` - /// option. - /// - /// For more information about this option, see [`set_linger`]. - /// - /// [`set_linger`]: TcpStream::set_linger - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpStream; - /// - /// # async fn dox() -> Result<(), Box> { - /// let stream = TcpStream::connect("127.0.0.1:8080").await?; - /// - /// println!("{:?}", stream.linger()?); - /// # Ok(()) - /// # } - /// ``` - pub fn linger(&self) -> io::Result> { - socket2::SockRef::from(self).linger() - } + cfg_not_wasi! { + /// Reads the linger duration for this socket by getting the `SO_LINGER` + /// option. + /// + /// For more information about this option, see [`set_linger`]. + /// + /// [`set_linger`]: TcpStream::set_linger + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpStream; + /// + /// # async fn dox() -> Result<(), Box> { + /// let stream = TcpStream::connect("127.0.0.1:8080").await?; + /// + /// println!("{:?}", stream.linger()?); + /// # Ok(()) + /// # } + /// ``` + pub fn linger(&self) -> io::Result> { + socket2::SockRef::from(self).linger() + } - /// Sets the linger duration of this socket by setting the SO_LINGER option. - /// - /// This option controls the action taken when a stream has unsent messages and the stream is - /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the - /// data or until the time expires. - /// - /// If SO_LINGER is not specified, and the stream is closed, the system handles the call in a - /// way that allows the process to continue as quickly as possible. - /// - /// # Examples - /// - /// ```no_run - /// use tokio::net::TcpStream; - /// - /// # async fn dox() -> Result<(), Box> { - /// let stream = TcpStream::connect("127.0.0.1:8080").await?; - /// - /// stream.set_linger(None)?; - /// # Ok(()) - /// # } - /// ``` - pub fn set_linger(&self, dur: Option) -> io::Result<()> { - socket2::SockRef::from(self).set_linger(dur) + /// Sets the linger duration of this socket by setting the SO_LINGER option. + /// + /// This option controls the action taken when a stream has unsent messages and the stream is + /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the + /// data or until the time expires. + /// + /// If SO_LINGER is not specified, and the stream is closed, the system handles the call in a + /// way that allows the process to continue as quickly as possible. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::TcpStream; + /// + /// # async fn dox() -> Result<(), Box> { + /// let stream = TcpStream::connect("127.0.0.1:8080").await?; + /// + /// stream.set_linger(None)?; + /// # Ok(()) + /// # } + /// ``` + pub fn set_linger(&self, dur: Option) -> io::Result<()> { + socket2::SockRef::from(self).set_linger(dur) + } } /// Gets the value of the `IP_TTL` option for this socket. @@ -1295,16 +1377,49 @@ mod sys { self.io.as_raw_fd() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for TcpStream { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } } -#[cfg(windows)] -mod sys { - use super::TcpStream; - use std::os::windows::prelude::*; +cfg_windows! { + use crate::os::windows::io::{AsRawSocket, RawSocket}; + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsSocket, BorrowedSocket}; impl AsRawSocket for TcpStream { fn as_raw_socket(&self) -> RawSocket { self.io.as_raw_socket() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsSocket for TcpStream { + fn as_socket(&self) -> BorrowedSocket<'_> { + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } + } +} + +#[cfg(all(tokio_unstable, tokio_wasi))] +mod sys { + use super::TcpStream; + use std::os::wasi::prelude::*; + + impl AsRawFd for TcpStream { + fn as_raw_fd(&self) -> RawFd { + self.io.as_raw_fd() + } + } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for TcpStream { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } } diff --git a/third_party/rust/tokio/src/net/udp.rs b/third_party/rust/tokio/src/net/udp.rs index 12af5152c283..8da377e16a76 100644 --- a/third_party/rust/tokio/src/net/udp.rs +++ b/third_party/rust/tokio/src/net/udp.rs @@ -1,7 +1,6 @@ use crate::io::{Interest, PollEvented, ReadBuf, Ready}; use crate::net::{to_socket_addrs, ToSocketAddrs}; -use std::convert::TryFrom; use std::fmt; use std::io; use std::net::{self, Ipv4Addr, Ipv6Addr, SocketAddr}; @@ -170,6 +169,7 @@ impl UdpSocket { UdpSocket::new(sys) } + #[track_caller] fn new(socket: mio::net::UdpSocket) -> io::Result { let io = PollEvented::new(socket)?; Ok(UdpSocket { io }) @@ -178,14 +178,21 @@ impl UdpSocket { /// Creates new `UdpSocket` from a previously bound `std::net::UdpSocket`. /// /// This function is intended to be used to wrap a UDP socket from the - /// standard library in the Tokio equivalent. The conversion assumes nothing - /// about the underlying socket; it is left up to the user to set it in - /// non-blocking mode. + /// standard library in the Tokio equivalent. /// /// This can be used in conjunction with socket2's `Socket` interface to /// configure a socket before it's handed off, such as setting options like /// `reuse_address` or binding to multiple addresses. /// + /// # Notes + /// + /// The caller is responsible for ensuring that the socket is in + /// non-blocking mode. Otherwise all I/O operations on the socket + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::net::UdpSocket::set_nonblocking + /// /// # Panics /// /// This function panics if thread-local runtime is not set. @@ -210,6 +217,7 @@ impl UdpSocket { /// # Ok(()) /// # } /// ``` + #[track_caller] pub fn from_std(socket: net::UdpSocket) -> io::Result { let io = mio::net::UdpSocket::from_std(socket); UdpSocket::new(io) @@ -257,6 +265,10 @@ impl UdpSocket { } } + fn as_socket(&self) -> socket2::SockRef<'_> { + socket2::SockRef::from(self) + } + /// Returns the local address that this socket is bound to. /// /// # Example @@ -278,6 +290,28 @@ impl UdpSocket { self.io.local_addr() } + /// Returns the socket address of the remote peer this socket was connected to. + /// + /// # Example + /// + /// ``` + /// use tokio::net::UdpSocket; + /// + /// # use std::{io, net::SocketAddr}; + /// # #[tokio::main] + /// # async fn main() -> io::Result<()> { + /// let addr = "0.0.0.0:8080".parse::().unwrap(); + /// let peer = "127.0.0.1:11100".parse::().unwrap(); + /// let sock = UdpSocket::bind(addr).await?; + /// sock.connect(peer).await?; + /// assert_eq!(peer, sock.peer_addr()?); + /// # Ok(()) + /// # } + /// ``` + pub fn peer_addr(&self) -> io::Result { + self.io.peer_addr() + } + /// Connects the UDP socket setting the default destination for send() and /// limiting packets that are read via recv from the address specified in /// `addr`. @@ -329,7 +363,9 @@ impl UdpSocket { /// /// The function may complete without the socket being ready. This is a /// false-positive and attempting an operation will return with - /// `io::ErrorKind::WouldBlock`. + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// # Cancel safety /// @@ -712,7 +748,7 @@ impl UdpSocket { /// /// # Cancel safety /// - /// This method is cancel safe. If `recv_from` is used as the event in a + /// This method is cancel safe. If `recv` is used as the event in a /// [`tokio::select!`](crate::select) statement and some other branch /// completes first, it is guaranteed that no messages were received on this /// socket. @@ -789,7 +825,7 @@ impl UdpSocket { /// address to which it is connected. On success, returns the number of /// bytes read. /// - /// The function must be called with valid byte array buf of sufficient size + /// This method must be called with valid byte array buf of sufficient size /// to hold the message bytes. If a message is too long to fit in the /// supplied buffer, excess bytes may be discarded. /// @@ -845,10 +881,12 @@ impl UdpSocket { /// Tries to receive data from the stream into the provided buffer, advancing the /// buffer's internal cursor, returning how many bytes were read. /// - /// The function must be called with valid byte array buf of sufficient size + /// This method must be called with valid byte array buf of sufficient size /// to hold the message bytes. If a message is too long to fit in the /// supplied buffer, excess bytes may be discarded. /// + /// This method can be used even if `buf` is uninitialized. + /// /// When there is no pending data, `Err(io::ErrorKind::WouldBlock)` is /// returned. This function is usually paired with `readable()`. /// @@ -895,10 +933,10 @@ impl UdpSocket { let dst = unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + let n = (*self.io).recv(dst)?; + // Safety: We trust `UdpSocket::recv` to have filled up `n` bytes in the // buffer. - let n = (&*self.io).recv(dst)?; - unsafe { buf.advance_mut(n); } @@ -907,16 +945,75 @@ impl UdpSocket { }) } - /// Tries to receive a single datagram message on the socket. On success, - /// returns the number of bytes read and the origin. + /// Receives a single datagram message on the socket from the remote address + /// to which it is connected, advancing the buffer's internal cursor, + /// returning how many bytes were read. /// - /// The function must be called with valid byte array buf of sufficient size + /// This method must be called with valid byte array buf of sufficient size /// to hold the message bytes. If a message is too long to fit in the /// supplied buffer, excess bytes may be discarded. /// + /// This method can be used even if `buf` is uninitialized. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::UdpSocket; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Connect to a peer + /// let socket = UdpSocket::bind("127.0.0.1:8080").await?; + /// socket.connect("127.0.0.1:8081").await?; + /// + /// let mut buf = Vec::with_capacity(512); + /// let len = socket.recv_buf(&mut buf).await?; + /// + /// println!("received {} bytes {:?}", len, &buf[..len]); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv_buf(&self, buf: &mut B) -> io::Result { + self.io.registration().async_io(Interest::READABLE, || { + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + + let n = (*self.io).recv(dst)?; + + // Safety: We trust `UdpSocket::recv` to have filled up `n` bytes in the + // buffer. + unsafe { + buf.advance_mut(n); + } + + Ok(n) + }).await + } + + /// Tries to receive a single datagram message on the socket. On success, + /// returns the number of bytes read and the origin. + /// + /// This method must be called with valid byte array buf of sufficient size + /// to hold the message bytes. If a message is too long to fit in the + /// supplied buffer, excess bytes may be discarded. + /// + /// This method can be used even if `buf` is uninitialized. + /// /// When there is no pending data, `Err(io::ErrorKind::WouldBlock)` is /// returned. This function is usually paired with `readable()`. /// + /// # Notes + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection + /// /// # Examples /// /// ```no_run @@ -959,10 +1056,10 @@ impl UdpSocket { let dst = unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + let (n, addr) = (*self.io).recv_from(dst)?; + // Safety: We trust `UdpSocket::recv_from` to have filled up `n` bytes in the // buffer. - let (n, addr) = (&*self.io).recv_from(dst)?; - unsafe { buf.advance_mut(n); } @@ -970,6 +1067,62 @@ impl UdpSocket { Ok((n, addr)) }) } + + /// Receives a single datagram message on the socket, advancing the + /// buffer's internal cursor, returning how many bytes were read and the origin. + /// + /// This method must be called with valid byte array buf of sufficient size + /// to hold the message bytes. If a message is too long to fit in the + /// supplied buffer, excess bytes may be discarded. + /// + /// This method can be used even if `buf` is uninitialized. + /// + /// # Notes + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::UdpSocket; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Connect to a peer + /// let socket = UdpSocket::bind("127.0.0.1:8080").await?; + /// socket.connect("127.0.0.1:8081").await?; + /// + /// let mut buf = Vec::with_capacity(512); + /// let (len, addr) = socket.recv_buf_from(&mut buf).await?; + /// + /// println!("received {:?} bytes from {:?}", len, addr); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv_buf_from(&self, buf: &mut B) -> io::Result<(usize, SocketAddr)> { + self.io.registration().async_io(Interest::READABLE, || { + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + + let (n, addr) = (*self.io).recv_from(dst)?; + + // Safety: We trust `UdpSocket::recv_from` to have filled up `n` bytes in the + // buffer. + unsafe { + buf.advance_mut(n); + } + + Ok((n,addr)) + }).await + } } /// Sends data on the socket to the given address. On success, returns the @@ -1140,6 +1293,15 @@ impl UdpSocket { /// Ok(()) /// } /// ``` + /// + /// # Notes + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection pub async fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { self.io .registration() @@ -1164,6 +1326,15 @@ impl UdpSocket { /// # Errors /// /// This function may encounter any standard I/O error except `WouldBlock`. + /// + /// # Notes + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection pub fn poll_recv_from( &self, cx: &mut Context<'_>, @@ -1189,13 +1360,23 @@ impl UdpSocket { /// Tries to receive a single datagram message on the socket. On success, /// returns the number of bytes read and the origin. /// - /// The function must be called with valid byte array buf of sufficient size + /// This method must be called with valid byte array buf of sufficient size /// to hold the message bytes. If a message is too long to fit in the /// supplied buffer, excess bytes may be discarded. /// /// When there is no pending data, `Err(io::ErrorKind::WouldBlock)` is /// returned. This function is usually paired with `readable()`. /// + /// # Notes + /// + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection + /// /// # Examples /// /// ```no_run @@ -1243,7 +1424,7 @@ impl UdpSocket { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -1262,6 +1443,11 @@ impl UdpSocket { /// defined on the Tokio `UdpSocket` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: UdpSocket::readable() @@ -1272,7 +1458,45 @@ impl UdpSocket { interest: Interest, f: impl FnOnce() -> io::Result, ) -> io::Result { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) + } + + /// Reads or writes from the socket using a user-provided IO operation. + /// + /// The readiness of the socket is awaited and when the socket is ready, + /// the provided closure is called. The closure should attempt to perform + /// IO operation on the socket by manually calling the appropriate syscall. + /// If the operation fails because the socket is not actually ready, + /// then the closure should return a `WouldBlock` error. In such case the + /// readiness flag is cleared and the socket readiness is awaited again. + /// This loop is repeated until the closure returns an `Ok` or an error + /// other than `WouldBlock`. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `UdpSocket` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + pub async fn async_io( + &self, + interest: Interest, + mut f: impl FnMut() -> io::Result, + ) -> io::Result { + self.io + .registration() + .async_io(interest, || self.io.try_io(&mut f)) + .await } /// Receives data from the socket, without removing it from the input queue. @@ -1287,6 +1511,17 @@ impl UdpSocket { /// Make sure to always use a sufficiently large buffer to hold the /// maximum UDP packet size, which can be up to 65536 bytes in size. /// + /// MacOS will return an error if you pass a zero-sized buffer. + /// + /// If you're merely interested in learning the sender of the data at the head of the queue, + /// try [`peek_sender`]. + /// + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// /// # Examples /// /// ```no_run @@ -1305,6 +1540,9 @@ impl UdpSocket { /// Ok(()) /// } /// ``` + /// + /// [`peek_sender`]: method@Self::peek_sender + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection pub async fn peek_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { self.io .registration() @@ -1313,7 +1551,7 @@ impl UdpSocket { } /// Receives data from the socket, without removing it from the input queue. - /// On success, returns the number of bytes read. + /// On success, returns the sending address of the datagram. /// /// # Notes /// @@ -1327,6 +1565,17 @@ impl UdpSocket { /// Make sure to always use a sufficiently large buffer to hold the /// maximum UDP packet size, which can be up to 65536 bytes in size. /// + /// MacOS will return an error if you pass a zero-sized buffer. + /// + /// If you're merely interested in learning the sender of the data at the head of the queue, + /// try [`poll_peek_sender`]. + /// + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// /// # Return value /// /// The function returns: @@ -1338,6 +1587,9 @@ impl UdpSocket { /// # Errors /// /// This function may encounter any standard I/O error except `WouldBlock`. + /// + /// [`poll_peek_sender`]: method@Self::poll_peek_sender + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection pub fn poll_peek_from( &self, cx: &mut Context<'_>, @@ -1360,6 +1612,117 @@ impl UdpSocket { Poll::Ready(Ok(addr)) } + /// Tries to receive data on the socket without removing it from the input queue. + /// On success, returns the number of bytes read and the sending address of the + /// datagram. + /// + /// When there is no pending data, `Err(io::ErrorKind::WouldBlock)` is + /// returned. This function is usually paired with `readable()`. + /// + /// # Notes + /// + /// On Windows, if the data is larger than the buffer specified, the buffer + /// is filled with the first part of the data, and peek returns the error + /// WSAEMSGSIZE(10040). The excess data is lost. + /// Make sure to always use a sufficiently large buffer to hold the + /// maximum UDP packet size, which can be up to 65536 bytes in size. + /// + /// MacOS will return an error if you pass a zero-sized buffer. + /// + /// If you're merely interested in learning the sender of the data at the head of the queue, + /// try [`try_peek_sender`]. + /// + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [`try_peek_sender`]: method@Self::try_peek_sender + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection + pub fn try_peek_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { + self.io + .registration() + .try_io(Interest::READABLE, || self.io.peek_from(buf)) + } + + /// Retrieve the sender of the data at the head of the input queue, waiting if empty. + /// + /// This is equivalent to calling [`peek_from`] with a zero-sized buffer, + /// but suppresses the `WSAEMSGSIZE` error on Windows and the "invalid argument" error on macOS. + /// + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [`peek_from`]: method@Self::peek_from + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection + pub async fn peek_sender(&self) -> io::Result { + self.io + .registration() + .async_io(Interest::READABLE, || self.peek_sender_inner()) + .await + } + + /// Retrieve the sender of the data at the head of the input queue, + /// scheduling a wakeup if empty. + /// + /// This is equivalent to calling [`poll_peek_from`] with a zero-sized buffer, + /// but suppresses the `WSAEMSGSIZE` error on Windows and the "invalid argument" error on macOS. + /// + /// # Notes + /// + /// Note that on multiple calls to a `poll_*` method in the recv direction, only the + /// `Waker` from the `Context` passed to the most recent call will be scheduled to + /// receive a wakeup. + /// + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [`poll_peek_from`]: method@Self::poll_peek_from + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection + pub fn poll_peek_sender(&self, cx: &mut Context<'_>) -> Poll> { + self.io + .registration() + .poll_read_io(cx, || self.peek_sender_inner()) + } + + /// Try to retrieve the sender of the data at the head of the input queue. + /// + /// When there is no pending data, `Err(io::ErrorKind::WouldBlock)` is + /// returned. This function is usually paired with `readable()`. + /// + /// Note that the socket address **cannot** be implicitly trusted, because it is relatively + /// trivial to send a UDP datagram with a spoofed origin in a [packet injection attack]. + /// Because UDP is stateless and does not validate the origin of a packet, + /// the attacker does not need to be able to intercept traffic in order to interfere. + /// It is important to be aware of this when designing your application-level protocol. + /// + /// [packet injection attack]: https://en.wikipedia.org/wiki/Packet_injection + pub fn try_peek_sender(&self) -> io::Result { + self.io + .registration() + .try_io(Interest::READABLE, || self.peek_sender_inner()) + } + + #[inline] + fn peek_sender_inner(&self) -> io::Result { + self.io.try_io(|| { + self.as_socket() + .peek_sender()? + // May be `None` if the platform doesn't populate the sender for some reason. + // In testing, that only occurred on macOS if you pass a zero-sized buffer, + // but the implementation of `Socket::peek_sender()` covers that. + .as_socket() + .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "sender not available")) + }) + } + /// Gets the value of the `SO_BROADCAST` option for this socket. /// /// For more information about this option, see [`set_broadcast`]. @@ -1484,6 +1847,89 @@ impl UdpSocket { self.io.set_ttl(ttl) } + /// Gets the value of the `IP_TOS` option for this socket. + /// + /// For more information about this option, see [`set_tos`]. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + /// + /// [`set_tos`]: Self::set_tos + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn tos(&self) -> io::Result { + self.as_socket().tos() + } + + /// Sets the value for the `IP_TOS` option on this socket. + /// + /// This value sets the type-of-service field that is used in every packet + /// sent from this socket. + /// + /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or + /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options) + // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178 + #[cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))] + #[cfg_attr( + docsrs, + doc(cfg(not(any( + target_os = "fuchsia", + target_os = "redox", + target_os = "solaris", + target_os = "illumos", + )))) + )] + pub fn set_tos(&self, tos: u32) -> io::Result<()> { + self.as_socket().set_tos(tos) + } + + /// Gets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// This value gets the socket-bound device's interface name. + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))] + #[cfg_attr( + docsrs, + doc(cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))) + )] + pub fn device(&self) -> io::Result>> { + self.as_socket().device() + } + + /// Sets the value for the `SO_BINDTODEVICE` option on this socket + /// + /// If a socket is bound to an interface, only packets received from that + /// particular interface are processed by the socket. Note that this only + /// works for some socket types, particularly `AF_INET` sockets. + /// + /// If `interface` is `None` or an empty string it removes the binding. + #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] + #[cfg_attr( + docsrs, + doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))) + )] + pub fn bind_device(&self, interface: Option<&[u8]>) -> io::Result<()> { + self.as_socket().bind_device(interface) + } + /// Executes an operation of the `IP_ADD_MEMBERSHIP` type. /// /// This function specifies a new multicast group for this socket to join. @@ -1564,7 +2010,7 @@ impl fmt::Debug for UdpSocket { } } -#[cfg(all(unix))] +#[cfg(unix)] mod sys { use super::UdpSocket; use std::os::unix::prelude::*; @@ -1574,16 +2020,30 @@ mod sys { self.io.as_raw_fd() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for UdpSocket { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } } -#[cfg(windows)] -mod sys { - use super::UdpSocket; - use std::os::windows::prelude::*; +cfg_windows! { + use crate::os::windows::io::{AsRawSocket, RawSocket}; + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsSocket, BorrowedSocket}; impl AsRawSocket for UdpSocket { fn as_raw_socket(&self) -> RawSocket { self.io.as_raw_socket() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsSocket for UdpSocket { + fn as_socket(&self) -> BorrowedSocket<'_> { + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } + } } diff --git a/third_party/rust/tokio/src/net/unix/datagram/socket.rs b/third_party/rust/tokio/src/net/unix/datagram/socket.rs index d5b618663dc6..c97d101aae90 100644 --- a/third_party/rust/tokio/src/net/unix/datagram/socket.rs +++ b/third_party/rust/tokio/src/net/unix/datagram/socket.rs @@ -1,10 +1,11 @@ use crate::io::{Interest, PollEvented, ReadBuf, Ready}; use crate::net::unix::SocketAddr; -use std::convert::TryFrom; use std::fmt; use std::io; use std::net::Shutdown; +#[cfg(not(tokio_no_as_fd))] +use std::os::unix::io::{AsFd, BorrowedFd}; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::os::unix::net; use std::path::Path; @@ -90,6 +91,7 @@ cfg_net_unix! { /// # Ok(()) /// # } /// ``` + #[cfg_attr(docsrs, doc(alias = "uds"))] pub struct UnixDatagram { io: PollEvented, } @@ -104,7 +106,9 @@ impl UnixDatagram { /// /// The function may complete without the socket being ready. This is a /// false-positive and attempting an operation will return with - /// `io::ErrorKind::WouldBlock`. + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. /// /// # Cancel safety /// @@ -424,13 +428,21 @@ impl UnixDatagram { /// Creates new `UnixDatagram` from a `std::os::unix::net::UnixDatagram`. /// /// This function is intended to be used to wrap a UnixDatagram from the - /// standard library in the Tokio equivalent. The conversion assumes - /// nothing about the underlying datagram; it is left up to the user to set - /// it in non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the socker is in + /// non-blocking mode. Otherwise all I/O operations on the socket + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::os::unix::net::UnixDatagram::set_nonblocking /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a Tokio runtime, otherwise runtime can be set @@ -457,6 +469,7 @@ impl UnixDatagram { /// # Ok(()) /// # } /// ``` + #[track_caller] pub fn from_std(datagram: net::UnixDatagram) -> io::Result { let socket = mio::net::UnixDatagram::from_std(datagram); let io = PollEvented::new(socket)?; @@ -466,21 +479,19 @@ impl UnixDatagram { /// Turns a [`tokio::net::UnixDatagram`] into a [`std::os::unix::net::UnixDatagram`]. /// /// The returned [`std::os::unix::net::UnixDatagram`] will have nonblocking - /// mode set as `true`. Use [`set_nonblocking`] to change the blocking mode + /// mode set as `true`. Use [`set_nonblocking`] to change the blocking mode /// if needed. /// /// # Examples /// /// ```rust,no_run - /// use std::error::Error; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let tokio_socket = tokio::net::UnixDatagram::bind("127.0.0.1:0")?; - /// let std_socket = tokio_socket.into_std()?; - /// std_socket.set_nonblocking(false)?; - /// Ok(()) - /// } + /// # use std::error::Error; + /// # async fn dox() -> Result<(), Box> { + /// let tokio_socket = tokio::net::UnixDatagram::bind("/path/to/the/socket")?; + /// let std_socket = tokio_socket.into_std()?; + /// std_socket.set_nonblocking(false)?; + /// # Ok(()) + /// # } /// ``` /// /// [`tokio::net::UnixDatagram`]: UnixDatagram @@ -797,6 +808,8 @@ impl UnixDatagram { cfg_io_util! { /// Tries to receive data from the socket without waiting. /// + /// This method can be used even if `buf` is uninitialized. + /// /// # Examples /// /// ```no_run @@ -844,7 +857,7 @@ impl UnixDatagram { // Safety: We trust `UnixDatagram::recv_from` to have filled up `n` bytes in the // buffer. - let (n, addr) = (&*self.io).recv_from(dst)?; + let (n, addr) = (*self.io).recv_from(dst)?; unsafe { buf.advance_mut(n); @@ -856,9 +869,64 @@ impl UnixDatagram { Ok((n, SocketAddr(addr))) } + /// Receives from the socket, advances the + /// buffer's internal cursor and returns how many bytes were read and the origin. + /// + /// This method can be used even if `buf` is uninitialized. + /// + /// # Examples + /// ``` + /// # use std::error::Error; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// use tokio::net::UnixDatagram; + /// use tempfile::tempdir; + /// + /// // We use a temporary directory so that the socket + /// // files left by the bound sockets will get cleaned up. + /// let tmp = tempdir()?; + /// + /// // Bind each socket to a filesystem path + /// let tx_path = tmp.path().join("tx"); + /// let tx = UnixDatagram::bind(&tx_path)?; + /// let rx_path = tmp.path().join("rx"); + /// let rx = UnixDatagram::bind(&rx_path)?; + /// + /// let bytes = b"hello world"; + /// tx.send_to(bytes, &rx_path).await?; + /// + /// let mut buf = Vec::with_capacity(24); + /// let (size, addr) = rx.recv_buf_from(&mut buf).await?; + /// + /// let dgram = &buf[..size]; + /// assert_eq!(dgram, bytes); + /// assert_eq!(addr.as_pathname().unwrap(), &tx_path); + /// + /// # Ok(()) + /// # } + /// ``` + pub async fn recv_buf_from(&self, buf: &mut B) -> io::Result<(usize, SocketAddr)> { + self.io.registration().async_io(Interest::READABLE, || { + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + + // Safety: We trust `UnixDatagram::recv_from` to have filled up `n` bytes in the + // buffer. + let (n, addr) = (*self.io).recv_from(dst)?; + + unsafe { + buf.advance_mut(n); + } + Ok((n,SocketAddr(addr))) + }).await + } + /// Tries to read data from the stream into the provided buffer, advancing the /// buffer's internal cursor, returning how many bytes were read. /// + /// This method can be used even if `buf` is uninitialized. + /// /// # Examples /// /// ```no_run @@ -907,7 +975,7 @@ impl UnixDatagram { // Safety: We trust `UnixDatagram::recv` to have filled up `n` bytes in the // buffer. - let n = (&*self.io).recv(dst)?; + let n = (*self.io).recv(dst)?; unsafe { buf.advance_mut(n); @@ -916,6 +984,52 @@ impl UnixDatagram { Ok(n) }) } + + /// Receives data from the socket from the address to which it is connected, + /// advancing the buffer's internal cursor, returning how many bytes were read. + /// + /// This method can be used even if `buf` is uninitialized. + /// + /// # Examples + /// ``` + /// # use std::error::Error; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// use tokio::net::UnixDatagram; + /// + /// // Create the pair of sockets + /// let (sock1, sock2) = UnixDatagram::pair()?; + /// + /// // Since the sockets are paired, the paired send/recv + /// // functions can be used + /// let bytes = b"hello world"; + /// sock1.send(bytes).await?; + /// + /// let mut buff = Vec::with_capacity(24); + /// let size = sock2.recv_buf(&mut buff).await?; + /// + /// let dgram = &buff[..size]; + /// assert_eq!(dgram, bytes); + /// + /// # Ok(()) + /// # } + /// ``` + pub async fn recv_buf(&self, buf: &mut B) -> io::Result { + self.io.registration().async_io(Interest::READABLE, || { + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + + // Safety: We trust `UnixDatagram::recv_from` to have filled up `n` bytes in the + // buffer. + let n = (*self.io).recv(dst)?; + + unsafe { + buf.advance_mut(n); + } + Ok(n) + }).await + } } /// Sends data on the socket to the specified address. @@ -1212,7 +1326,7 @@ impl UnixDatagram { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -1231,6 +1345,11 @@ impl UnixDatagram { /// defined on the Tokio `UnixDatagram` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: UnixDatagram::readable() @@ -1241,7 +1360,45 @@ impl UnixDatagram { interest: Interest, f: impl FnOnce() -> io::Result, ) -> io::Result { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) + } + + /// Reads or writes from the socket using a user-provided IO operation. + /// + /// The readiness of the socket is awaited and when the socket is ready, + /// the provided closure is called. The closure should attempt to perform + /// IO operation on the socket by manually calling the appropriate syscall. + /// If the operation fails because the socket is not actually ready, + /// then the closure should return a `WouldBlock` error. In such case the + /// readiness flag is cleared and the socket readiness is awaited again. + /// This loop is repeated until the closure returns an `Ok` or an error + /// other than `WouldBlock`. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `UnixDatagram` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + pub async fn async_io( + &self, + interest: Interest, + mut f: impl FnMut() -> io::Result, + ) -> io::Result { + self.io + .registration() + .async_io(interest, || self.io.try_io(&mut f)) + .await } /// Returns the local address that this socket is bound to. @@ -1420,3 +1577,10 @@ impl AsRawFd for UnixDatagram { self.io.as_raw_fd() } } + +#[cfg(not(tokio_no_as_fd))] +impl AsFd for UnixDatagram { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} diff --git a/third_party/rust/tokio/src/net/unix/listener.rs b/third_party/rust/tokio/src/net/unix/listener.rs index 1785f8b0f735..37a5a4ae4005 100644 --- a/third_party/rust/tokio/src/net/unix/listener.rs +++ b/third_party/rust/tokio/src/net/unix/listener.rs @@ -1,9 +1,10 @@ use crate::io::{Interest, PollEvented}; use crate::net::unix::{SocketAddr, UnixStream}; -use std::convert::TryFrom; use std::fmt; use std::io; +#[cfg(not(tokio_no_as_fd))] +use std::os::unix::io::{AsFd, BorrowedFd}; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::os::unix::net; use std::path::Path; @@ -44,6 +45,7 @@ cfg_net_unix! { /// } /// } /// ``` + #[cfg_attr(docsrs, doc(alias = "uds"))] pub struct UnixListener { io: PollEvented, } @@ -54,11 +56,13 @@ impl UnixListener { /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn bind

(path: P) -> io::Result where P: AsRef, @@ -71,17 +75,41 @@ impl UnixListener { /// Creates new `UnixListener` from a `std::os::unix::net::UnixListener `. /// /// This function is intended to be used to wrap a UnixListener from the - /// standard library in the Tokio equivalent. The conversion assumes - /// nothing about the underlying listener; it is left up to the user to set - /// it in non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the listener is in + /// non-blocking mode. Otherwise all I/O operations on the listener + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::os::unix::net::UnixListener::set_nonblocking + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::UnixListener; + /// use std::os::unix::net::UnixListener as StdUnixListener; + /// # use std::error::Error; + /// + /// # async fn dox() -> Result<(), Box> { + /// let std_listener = StdUnixListener::bind("/path/to/the/socket")?; + /// std_listener.set_nonblocking(true)?; + /// let listener = UnixListener::from_std(std_listener)?; + /// # Ok(()) + /// # } + /// ``` /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(listener: net::UnixListener) -> io::Result { let listener = mio::net::UnixListener::from_std(listener); let io = PollEvented::new(listener)?; @@ -91,20 +119,18 @@ impl UnixListener { /// Turns a [`tokio::net::UnixListener`] into a [`std::os::unix::net::UnixListener`]. /// /// The returned [`std::os::unix::net::UnixListener`] will have nonblocking mode - /// set as `true`. Use [`set_nonblocking`] to change the blocking mode if needed. + /// set as `true`. Use [`set_nonblocking`] to change the blocking mode if needed. /// /// # Examples /// /// ```rust,no_run - /// use std::error::Error; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let tokio_listener = tokio::net::UnixListener::bind("127.0.0.1:0")?; - /// let std_listener = tokio_listener.into_std()?; - /// std_listener.set_nonblocking(false)?; - /// Ok(()) - /// } + /// # use std::error::Error; + /// # async fn dox() -> Result<(), Box> { + /// let tokio_listener = tokio::net::UnixListener::bind("/path/to/the/socket")?; + /// let std_listener = tokio_listener.into_std()?; + /// std_listener.set_nonblocking(false)?; + /// # Ok(()) + /// # } /// ``` /// /// [`tokio::net::UnixListener`]: UnixListener @@ -184,3 +210,10 @@ impl AsRawFd for UnixListener { self.io.as_raw_fd() } } + +#[cfg(not(tokio_no_as_fd))] +impl AsFd for UnixListener { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} diff --git a/third_party/rust/tokio/src/net/unix/mod.rs b/third_party/rust/tokio/src/net/unix/mod.rs index 14cb456705fa..a49b70af34a0 100644 --- a/third_party/rust/tokio/src/net/unix/mod.rs +++ b/third_party/rust/tokio/src/net/unix/mod.rs @@ -1,5 +1,4 @@ -//! Unix domain socket utility types. - +//! Unix specific network types. // This module does not currently provide any public API, but it was // unintentionally defined as a public module. Hide it from the documentation // instead of changing it to a private module to avoid breakage. @@ -22,3 +21,17 @@ pub(crate) use stream::UnixStream; mod ucred; pub use ucred::UCred; + +pub mod pipe; + +/// A type representing process and process group IDs. +#[allow(non_camel_case_types)] +pub type uid_t = u32; + +/// A type representing user ID. +#[allow(non_camel_case_types)] +pub type gid_t = u32; + +/// A type representing group ID. +#[allow(non_camel_case_types)] +pub type pid_t = i32; diff --git a/third_party/rust/tokio/src/net/unix/pipe.rs b/third_party/rust/tokio/src/net/unix/pipe.rs new file mode 100644 index 000000000000..188ac68b11c1 --- /dev/null +++ b/third_party/rust/tokio/src/net/unix/pipe.rs @@ -0,0 +1,1222 @@ +//! Unix pipe types. + +use crate::io::interest::Interest; +use crate::io::{AsyncRead, AsyncWrite, PollEvented, ReadBuf, Ready}; + +use mio::unix::pipe as mio_pipe; +use std::fs::File; +use std::io::{self, Read, Write}; +use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; +#[cfg(not(tokio_no_as_fd))] +use std::os::unix::io::{AsFd, BorrowedFd}; +use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; +use std::path::Path; +use std::pin::Pin; +use std::task::{Context, Poll}; + +cfg_io_util! { + use bytes::BufMut; +} + +/// Options and flags which can be used to configure how a FIFO file is opened. +/// +/// This builder allows configuring how to create a pipe end from a FIFO file. +/// Generally speaking, when using `OpenOptions`, you'll first call [`new`], +/// then chain calls to methods to set each option, then call either +/// [`open_receiver`] or [`open_sender`], passing the path of the FIFO file you +/// are trying to open. This will give you a [`io::Result`][result] with a pipe +/// end inside that you can further operate on. +/// +/// [`new`]: OpenOptions::new +/// [`open_receiver`]: OpenOptions::open_receiver +/// [`open_sender`]: OpenOptions::open_sender +/// [result]: std::io::Result +/// +/// # Examples +/// +/// Opening a pair of pipe ends from a FIFO file: +/// +/// ```no_run +/// use tokio::net::unix::pipe; +/// # use std::error::Error; +/// +/// const FIFO_NAME: &str = "path/to/a/fifo"; +/// +/// # async fn dox() -> Result<(), Box> { +/// let rx = pipe::OpenOptions::new().open_receiver(FIFO_NAME)?; +/// let tx = pipe::OpenOptions::new().open_sender(FIFO_NAME)?; +/// # Ok(()) +/// # } +/// ``` +/// +/// Opening a [`Sender`] on Linux when you are sure the file is a FIFO: +/// +/// ```ignore +/// use tokio::net::unix::pipe; +/// use nix::{unistd::mkfifo, sys::stat::Mode}; +/// # use std::error::Error; +/// +/// // Our program has exclusive access to this path. +/// const FIFO_NAME: &str = "path/to/a/new/fifo"; +/// +/// # async fn dox() -> Result<(), Box> { +/// mkfifo(FIFO_NAME, Mode::S_IRWXU)?; +/// let tx = pipe::OpenOptions::new() +/// .read_write(true) +/// .unchecked(true) +/// .open_sender(FIFO_NAME)?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct OpenOptions { + #[cfg(target_os = "linux")] + read_write: bool, + unchecked: bool, +} + +impl OpenOptions { + /// Creates a blank new set of options ready for configuration. + /// + /// All options are initially set to `false`. + pub fn new() -> OpenOptions { + OpenOptions { + #[cfg(target_os = "linux")] + read_write: false, + unchecked: false, + } + } + + /// Sets the option for read-write access. + /// + /// This option, when true, will indicate that a FIFO file will be opened + /// in read-write access mode. This operation is not defined by the POSIX + /// standard and is only guaranteed to work on Linux. + /// + /// # Examples + /// + /// Opening a [`Sender`] even if there are no open reading ends: + /// + /// ```ignore + /// use tokio::net::unix::pipe; + /// + /// let tx = pipe::OpenOptions::new() + /// .read_write(true) + /// .open_sender("path/to/a/fifo"); + /// ``` + /// + /// Opening a resilient [`Receiver`] i.e. a reading pipe end which will not + /// fail with [`UnexpectedEof`] during reading if all writing ends of the + /// pipe close the FIFO file. + /// + /// [`UnexpectedEof`]: std::io::ErrorKind::UnexpectedEof + /// + /// ```ignore + /// use tokio::net::unix::pipe; + /// + /// let tx = pipe::OpenOptions::new() + /// .read_write(true) + /// .open_receiver("path/to/a/fifo"); + /// ``` + #[cfg(target_os = "linux")] + #[cfg_attr(docsrs, doc(cfg(target_os = "linux")))] + pub fn read_write(&mut self, value: bool) -> &mut Self { + self.read_write = value; + self + } + + /// Sets the option to skip the check for FIFO file type. + /// + /// By default, [`open_receiver`] and [`open_sender`] functions will check + /// if the opened file is a FIFO file. Set this option to `true` if you are + /// sure the file is a FIFO file. + /// + /// [`open_receiver`]: OpenOptions::open_receiver + /// [`open_sender`]: OpenOptions::open_sender + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use nix::{unistd::mkfifo, sys::stat::Mode}; + /// # use std::error::Error; + /// + /// // Our program has exclusive access to this path. + /// const FIFO_NAME: &str = "path/to/a/new/fifo"; + /// + /// # async fn dox() -> Result<(), Box> { + /// mkfifo(FIFO_NAME, Mode::S_IRWXU)?; + /// let rx = pipe::OpenOptions::new() + /// .unchecked(true) + /// .open_receiver(FIFO_NAME)?; + /// # Ok(()) + /// # } + /// ``` + pub fn unchecked(&mut self, value: bool) -> &mut Self { + self.unchecked = value; + self + } + + /// Creates a [`Receiver`] from a FIFO file with the options specified by `self`. + /// + /// This function will open the FIFO file at the specified path, possibly + /// check if it is a pipe, and associate the pipe with the default event + /// loop for reading. + /// + /// # Errors + /// + /// If the file type check fails, this function will fail with `io::ErrorKind::InvalidInput`. + /// This function may also fail with other standard OS errors. + /// + /// # Panics + /// + /// This function panics if it is not called from within a runtime with + /// IO enabled. + /// + /// The runtime is usually set implicitly when this function is called + /// from a future driven by a tokio runtime, otherwise runtime can be set + /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + pub fn open_receiver>(&self, path: P) -> io::Result { + let file = self.open(path.as_ref(), PipeEnd::Receiver)?; + Receiver::from_file_unchecked(file) + } + + /// Creates a [`Sender`] from a FIFO file with the options specified by `self`. + /// + /// This function will open the FIFO file at the specified path, possibly + /// check if it is a pipe, and associate the pipe with the default event + /// loop for writing. + /// + /// # Errors + /// + /// If the file type check fails, this function will fail with `io::ErrorKind::InvalidInput`. + /// If the file is not opened in read-write access mode and the file is not + /// currently open for reading, this function will fail with `ENXIO`. + /// This function may also fail with other standard OS errors. + /// + /// # Panics + /// + /// This function panics if it is not called from within a runtime with + /// IO enabled. + /// + /// The runtime is usually set implicitly when this function is called + /// from a future driven by a tokio runtime, otherwise runtime can be set + /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + pub fn open_sender>(&self, path: P) -> io::Result { + let file = self.open(path.as_ref(), PipeEnd::Sender)?; + Sender::from_file_unchecked(file) + } + + fn open(&self, path: &Path, pipe_end: PipeEnd) -> io::Result { + let mut options = std::fs::OpenOptions::new(); + options + .read(pipe_end == PipeEnd::Receiver) + .write(pipe_end == PipeEnd::Sender) + .custom_flags(libc::O_NONBLOCK); + + #[cfg(target_os = "linux")] + if self.read_write { + options.read(true).write(true); + } + + let file = options.open(path)?; + + if !self.unchecked && !is_fifo(&file)? { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "not a pipe")); + } + + Ok(file) + } +} + +impl Default for OpenOptions { + fn default() -> OpenOptions { + OpenOptions::new() + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum PipeEnd { + Sender, + Receiver, +} + +/// Writing end of a Unix pipe. +/// +/// It can be constructed from a FIFO file with [`OpenOptions::open_sender`]. +/// +/// Opening a named pipe for writing involves a few steps. +/// Call to [`OpenOptions::open_sender`] might fail with an error indicating +/// different things: +/// +/// * [`io::ErrorKind::NotFound`] - There is no file at the specified path. +/// * [`io::ErrorKind::InvalidInput`] - The file exists, but it is not a FIFO. +/// * [`ENXIO`] - The file is a FIFO, but no process has it open for reading. +/// Sleep for a while and try again. +/// * Other OS errors not specific to opening FIFO files. +/// +/// Opening a `Sender` from a FIFO file should look like this: +/// +/// ```no_run +/// use tokio::net::unix::pipe; +/// use tokio::time::{self, Duration}; +/// +/// const FIFO_NAME: &str = "path/to/a/fifo"; +/// +/// # async fn dox() -> Result<(), Box> { +/// // Wait for a reader to open the file. +/// let tx = loop { +/// match pipe::OpenOptions::new().open_sender(FIFO_NAME) { +/// Ok(tx) => break tx, +/// Err(e) if e.raw_os_error() == Some(libc::ENXIO) => {}, +/// Err(e) => return Err(e.into()), +/// } +/// +/// time::sleep(Duration::from_millis(50)).await; +/// }; +/// # Ok(()) +/// # } +/// ``` +/// +/// On Linux, it is possible to create a `Sender` without waiting in a sleeping +/// loop. This is done by opening a named pipe in read-write access mode with +/// `OpenOptions::read_write`. This way, a `Sender` can at the same time hold +/// both a writing end and a reading end, and the latter allows to open a FIFO +/// without [`ENXIO`] error since the pipe is open for reading as well. +/// +/// `Sender` cannot be used to read from a pipe, so in practice the read access +/// is only used when a FIFO is opened. However, using a `Sender` in read-write +/// mode **may lead to lost data**, because written data will be dropped by the +/// system as soon as all pipe ends are closed. To avoid lost data you have to +/// make sure that a reading end has been opened before dropping a `Sender`. +/// +/// Note that using read-write access mode with FIFO files is not defined by +/// the POSIX standard and it is only guaranteed to work on Linux. +/// +/// ```ignore +/// use tokio::io::AsyncWriteExt; +/// use tokio::net::unix::pipe; +/// +/// const FIFO_NAME: &str = "path/to/a/fifo"; +/// +/// # async fn dox() -> Result<(), Box> { +/// let mut tx = pipe::OpenOptions::new() +/// .read_write(true) +/// .open_sender(FIFO_NAME)?; +/// +/// // Asynchronously write to the pipe before a reader. +/// tx.write_all(b"hello world").await?; +/// # Ok(()) +/// # } +/// ``` +/// +/// [`ENXIO`]: https://docs.rs/libc/latest/libc/constant.ENXIO.html +#[derive(Debug)] +pub struct Sender { + io: PollEvented, +} + +impl Sender { + fn from_mio(mio_tx: mio_pipe::Sender) -> io::Result { + let io = PollEvented::new_with_interest(mio_tx, Interest::WRITABLE)?; + Ok(Sender { io }) + } + + /// Creates a new `Sender` from a [`File`]. + /// + /// This function is intended to construct a pipe from a [`File`] representing + /// a special FIFO file. It will check if the file is a pipe and has write access, + /// set it in non-blocking mode and perform the conversion. + /// + /// # Errors + /// + /// Fails with `io::ErrorKind::InvalidInput` if the file is not a pipe or it + /// does not have write access. Also fails with any standard OS error if it occurs. + /// + /// # Panics + /// + /// This function panics if it is not called from within a runtime with + /// IO enabled. + /// + /// The runtime is usually set implicitly when this function is called + /// from a future driven by a tokio runtime, otherwise runtime can be set + /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + pub fn from_file(mut file: File) -> io::Result { + if !is_fifo(&file)? { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "not a pipe")); + } + + let flags = get_file_flags(&file)?; + if has_write_access(flags) { + set_nonblocking(&mut file, flags)?; + Sender::from_file_unchecked(file) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not in O_WRONLY or O_RDWR access mode", + )) + } + } + + /// Creates a new `Sender` from a [`File`] without checking pipe properties. + /// + /// This function is intended to construct a pipe from a File representing + /// a special FIFO file. The conversion assumes nothing about the underlying + /// file; it is left up to the user to make sure it is opened with write access, + /// represents a pipe and is set in non-blocking mode. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::fs::OpenOptions; + /// use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; + /// # use std::error::Error; + /// + /// const FIFO_NAME: &str = "path/to/a/fifo"; + /// + /// # async fn dox() -> Result<(), Box> { + /// let file = OpenOptions::new() + /// .write(true) + /// .custom_flags(libc::O_NONBLOCK) + /// .open(FIFO_NAME)?; + /// if file.metadata()?.file_type().is_fifo() { + /// let tx = pipe::Sender::from_file_unchecked(file)?; + /// /* use the Sender */ + /// } + /// # Ok(()) + /// # } + /// ``` + /// + /// # Panics + /// + /// This function panics if it is not called from within a runtime with + /// IO enabled. + /// + /// The runtime is usually set implicitly when this function is called + /// from a future driven by a tokio runtime, otherwise runtime can be set + /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + pub fn from_file_unchecked(file: File) -> io::Result { + let raw_fd = file.into_raw_fd(); + let mio_tx = unsafe { mio_pipe::Sender::from_raw_fd(raw_fd) }; + Sender::from_mio(mio_tx) + } + + /// Waits for any of the requested ready states. + /// + /// This function can be used instead of [`writable()`] to check the returned + /// ready set for [`Ready::WRITABLE`] and [`Ready::WRITE_CLOSED`] events. + /// + /// The function may complete without the pipe being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// [`writable()`]: Self::writable + /// + /// # Cancel safety + /// + /// This method is cancel safe. Once a readiness event occurs, the method + /// will continue to return immediately until the readiness event is + /// consumed by an attempt to write that fails with `WouldBlock` or + /// `Poll::Pending`. + pub async fn ready(&self, interest: Interest) -> io::Result { + let event = self.io.registration().readiness(interest).await?; + Ok(event.ready) + } + + /// Waits for the pipe to become writable. + /// + /// This function is equivalent to `ready(Interest::WRITABLE)` and is usually + /// paired with [`try_write()`]. + /// + /// [`try_write()`]: Self::try_write + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Open a writing end of a fifo + /// let tx = pipe::OpenOptions::new().open_sender("path/to/a/fifo")?; + /// + /// loop { + /// // Wait for the pipe to be writable + /// tx.writable().await?; + /// + /// // Try to write data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match tx.try_write(b"hello world") { + /// Ok(n) => { + /// break; + /// } + /// Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub async fn writable(&self) -> io::Result<()> { + self.ready(Interest::WRITABLE).await?; + Ok(()) + } + + /// Polls for write readiness. + /// + /// If the pipe is not currently ready for writing, this method will + /// store a clone of the `Waker` from the provided `Context`. When the pipe + /// becomes ready for writing, `Waker::wake` will be called on the waker. + /// + /// Note that on multiple calls to `poll_write_ready` or `poll_write`, only + /// the `Waker` from the `Context` passed to the most recent call is + /// scheduled to receive a wakeup. + /// + /// This function is intended for cases where creating and pinning a future + /// via [`writable`] is not feasible. Where possible, using [`writable`] is + /// preferred, as this supports polling from multiple tasks at once. + /// + /// [`writable`]: Self::writable + /// + /// # Return value + /// + /// The function returns: + /// + /// * `Poll::Pending` if the pipe is not ready for writing. + /// * `Poll::Ready(Ok(()))` if the pipe is ready for writing. + /// * `Poll::Ready(Err(e))` if an error is encountered. + /// + /// # Errors + /// + /// This function may encounter any standard I/O error except `WouldBlock`. + pub fn poll_write_ready(&self, cx: &mut Context<'_>) -> Poll> { + self.io.registration().poll_write_ready(cx).map_ok(|_| ()) + } + + /// Tries to write a buffer to the pipe, returning how many bytes were + /// written. + /// + /// The function will attempt to write the entire contents of `buf`, but + /// only part of the buffer may be written. If the length of `buf` is not + /// greater than `PIPE_BUF` (an OS constant, 4096 under Linux), then the + /// write is guaranteed to be atomic, i.e. either the entire content of + /// `buf` will be written or this method will fail with `WouldBlock`. There + /// is no such guarantee if `buf` is larger than `PIPE_BUF`. + /// + /// This function is usually paired with [`writable`]. + /// + /// [`writable`]: Self::writable + /// + /// # Return + /// + /// If data is successfully written, `Ok(n)` is returned, where `n` is the + /// number of bytes written. If the pipe is not ready to write data, + /// `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Open a writing end of a fifo + /// let tx = pipe::OpenOptions::new().open_sender("path/to/a/fifo")?; + /// + /// loop { + /// // Wait for the pipe to be writable + /// tx.writable().await?; + /// + /// // Try to write data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match tx.try_write(b"hello world") { + /// Ok(n) => { + /// break; + /// } + /// Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_write(&self, buf: &[u8]) -> io::Result { + self.io + .registration() + .try_io(Interest::WRITABLE, || (&*self.io).write(buf)) + } + + /// Tries to write several buffers to the pipe, returning how many bytes + /// were written. + /// + /// Data is written from each buffer in order, with the final buffer read + /// from possible being only partially consumed. This method behaves + /// equivalently to a single call to [`try_write()`] with concatenated + /// buffers. + /// + /// If the total length of buffers is not greater than `PIPE_BUF` (an OS + /// constant, 4096 under Linux), then the write is guaranteed to be atomic, + /// i.e. either the entire contents of buffers will be written or this + /// method will fail with `WouldBlock`. There is no such guarantee if the + /// total length of buffers is greater than `PIPE_BUF`. + /// + /// This function is usually paired with [`writable`]. + /// + /// [`try_write()`]: Self::try_write() + /// [`writable`]: Self::writable + /// + /// # Return + /// + /// If data is successfully written, `Ok(n)` is returned, where `n` is the + /// number of bytes written. If the pipe is not ready to write data, + /// `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Open a writing end of a fifo + /// let tx = pipe::OpenOptions::new().open_sender("path/to/a/fifo")?; + /// + /// let bufs = [io::IoSlice::new(b"hello "), io::IoSlice::new(b"world")]; + /// + /// loop { + /// // Wait for the pipe to be writable + /// tx.writable().await?; + /// + /// // Try to write data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match tx.try_write_vectored(&bufs) { + /// Ok(n) => { + /// break; + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_write_vectored(&self, buf: &[io::IoSlice<'_>]) -> io::Result { + self.io + .registration() + .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) + } +} + +impl AsyncWrite for Sender { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.io.poll_write(cx, buf) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll> { + self.io.poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } + + fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +impl AsRawFd for Sender { + fn as_raw_fd(&self) -> RawFd { + self.io.as_raw_fd() + } +} + +#[cfg(not(tokio_no_as_fd))] +impl AsFd for Sender { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} + +/// Reading end of a Unix pipe. +/// +/// It can be constructed from a FIFO file with [`OpenOptions::open_receiver`]. +/// +/// # Examples +/// +/// Receiving messages from a named pipe in a loop: +/// +/// ```no_run +/// use tokio::net::unix::pipe; +/// use tokio::io::{self, AsyncReadExt}; +/// +/// const FIFO_NAME: &str = "path/to/a/fifo"; +/// +/// # async fn dox() -> Result<(), Box> { +/// let mut rx = pipe::OpenOptions::new().open_receiver(FIFO_NAME)?; +/// loop { +/// let mut msg = vec![0; 256]; +/// match rx.read_exact(&mut msg).await { +/// Ok(_) => { +/// /* handle the message */ +/// } +/// Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { +/// // Writing end has been closed, we should reopen the pipe. +/// rx = pipe::OpenOptions::new().open_receiver(FIFO_NAME)?; +/// } +/// Err(e) => return Err(e.into()), +/// } +/// } +/// # } +/// ``` +/// +/// On Linux, you can use a `Receiver` in read-write access mode to implement +/// resilient reading from a named pipe. Unlike `Receiver` opened in read-only +/// mode, read from a pipe in read-write mode will not fail with `UnexpectedEof` +/// when the writing end is closed. This way, a `Receiver` can asynchronously +/// wait for the next writer to open the pipe. +/// +/// You should not use functions waiting for EOF such as [`read_to_end`] with +/// a `Receiver` in read-write access mode, since it **may wait forever**. +/// `Receiver` in this mode also holds an open writing end, which prevents +/// receiving EOF. +/// +/// To set the read-write access mode you can use `OpenOptions::read_write`. +/// Note that using read-write access mode with FIFO files is not defined by +/// the POSIX standard and it is only guaranteed to work on Linux. +/// +/// ```ignore +/// use tokio::net::unix::pipe; +/// use tokio::io::AsyncReadExt; +/// # use std::error::Error; +/// +/// const FIFO_NAME: &str = "path/to/a/fifo"; +/// +/// # async fn dox() -> Result<(), Box> { +/// let mut rx = pipe::OpenOptions::new() +/// .read_write(true) +/// .open_receiver(FIFO_NAME)?; +/// loop { +/// let mut msg = vec![0; 256]; +/// rx.read_exact(&mut msg).await?; +/// /* handle the message */ +/// } +/// # } +/// ``` +/// +/// [`read_to_end`]: crate::io::AsyncReadExt::read_to_end +#[derive(Debug)] +pub struct Receiver { + io: PollEvented, +} + +impl Receiver { + fn from_mio(mio_rx: mio_pipe::Receiver) -> io::Result { + let io = PollEvented::new_with_interest(mio_rx, Interest::READABLE)?; + Ok(Receiver { io }) + } + + /// Creates a new `Receiver` from a [`File`]. + /// + /// This function is intended to construct a pipe from a [`File`] representing + /// a special FIFO file. It will check if the file is a pipe and has read access, + /// set it in non-blocking mode and perform the conversion. + /// + /// # Errors + /// + /// Fails with `io::ErrorKind::InvalidInput` if the file is not a pipe or it + /// does not have read access. Also fails with any standard OS error if it occurs. + /// + /// # Panics + /// + /// This function panics if it is not called from within a runtime with + /// IO enabled. + /// + /// The runtime is usually set implicitly when this function is called + /// from a future driven by a tokio runtime, otherwise runtime can be set + /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + pub fn from_file(mut file: File) -> io::Result { + if !is_fifo(&file)? { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "not a pipe")); + } + + let flags = get_file_flags(&file)?; + if has_read_access(flags) { + set_nonblocking(&mut file, flags)?; + Receiver::from_file_unchecked(file) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not in O_RDONLY or O_RDWR access mode", + )) + } + } + + /// Creates a new `Receiver` from a [`File`] without checking pipe properties. + /// + /// This function is intended to construct a pipe from a File representing + /// a special FIFO file. The conversion assumes nothing about the underlying + /// file; it is left up to the user to make sure it is opened with read access, + /// represents a pipe and is set in non-blocking mode. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::fs::OpenOptions; + /// use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; + /// # use std::error::Error; + /// + /// const FIFO_NAME: &str = "path/to/a/fifo"; + /// + /// # async fn dox() -> Result<(), Box> { + /// let file = OpenOptions::new() + /// .read(true) + /// .custom_flags(libc::O_NONBLOCK) + /// .open(FIFO_NAME)?; + /// if file.metadata()?.file_type().is_fifo() { + /// let rx = pipe::Receiver::from_file_unchecked(file)?; + /// /* use the Receiver */ + /// } + /// # Ok(()) + /// # } + /// ``` + /// + /// # Panics + /// + /// This function panics if it is not called from within a runtime with + /// IO enabled. + /// + /// The runtime is usually set implicitly when this function is called + /// from a future driven by a tokio runtime, otherwise runtime can be set + /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + pub fn from_file_unchecked(file: File) -> io::Result { + let raw_fd = file.into_raw_fd(); + let mio_rx = unsafe { mio_pipe::Receiver::from_raw_fd(raw_fd) }; + Receiver::from_mio(mio_rx) + } + + /// Waits for any of the requested ready states. + /// + /// This function can be used instead of [`readable()`] to check the returned + /// ready set for [`Ready::READABLE`] and [`Ready::READ_CLOSED`] events. + /// + /// The function may complete without the pipe being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// [`readable()`]: Self::readable + /// + /// # Cancel safety + /// + /// This method is cancel safe. Once a readiness event occurs, the method + /// will continue to return immediately until the readiness event is + /// consumed by an attempt to read that fails with `WouldBlock` or + /// `Poll::Pending`. + pub async fn ready(&self, interest: Interest) -> io::Result { + let event = self.io.registration().readiness(interest).await?; + Ok(event.ready) + } + + /// Waits for the pipe to become readable. + /// + /// This function is equivalent to `ready(Interest::READABLE)` and is usually + /// paired with [`try_read()`]. + /// + /// [`try_read()`]: Self::try_read() + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Open a reading end of a fifo + /// let rx = pipe::OpenOptions::new().open_receiver("path/to/a/fifo")?; + /// + /// let mut msg = vec![0; 1024]; + /// + /// loop { + /// // Wait for the pipe to be readable + /// rx.readable().await?; + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match rx.try_read(&mut msg) { + /// Ok(n) => { + /// msg.truncate(n); + /// break; + /// } + /// Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// println!("GOT = {:?}", msg); + /// Ok(()) + /// } + /// ``` + pub async fn readable(&self) -> io::Result<()> { + self.ready(Interest::READABLE).await?; + Ok(()) + } + + /// Polls for read readiness. + /// + /// If the pipe is not currently ready for reading, this method will + /// store a clone of the `Waker` from the provided `Context`. When the pipe + /// becomes ready for reading, `Waker::wake` will be called on the waker. + /// + /// Note that on multiple calls to `poll_read_ready` or `poll_read`, only + /// the `Waker` from the `Context` passed to the most recent call is + /// scheduled to receive a wakeup. + /// + /// This function is intended for cases where creating and pinning a future + /// via [`readable`] is not feasible. Where possible, using [`readable`] is + /// preferred, as this supports polling from multiple tasks at once. + /// + /// [`readable`]: Self::readable + /// + /// # Return value + /// + /// The function returns: + /// + /// * `Poll::Pending` if the pipe is not ready for reading. + /// * `Poll::Ready(Ok(()))` if the pipe is ready for reading. + /// * `Poll::Ready(Err(e))` if an error is encountered. + /// + /// # Errors + /// + /// This function may encounter any standard I/O error except `WouldBlock`. + pub fn poll_read_ready(&self, cx: &mut Context<'_>) -> Poll> { + self.io.registration().poll_read_ready(cx).map_ok(|_| ()) + } + + /// Tries to read data from the pipe into the provided buffer, returning how + /// many bytes were read. + /// + /// Reads any pending data from the pipe but does not wait for new data + /// to arrive. On success, returns the number of bytes read. Because + /// `try_read()` is non-blocking, the buffer does not have to be stored by + /// the async task and can exist entirely on the stack. + /// + /// Usually [`readable()`] is used with this function. + /// + /// [`readable()`]: Self::readable() + /// + /// # Return + /// + /// If data is successfully read, `Ok(n)` is returned, where `n` is the + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The pipe's writing end is closed and will no longer write data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the pipe is not ready to read data, + /// `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Open a reading end of a fifo + /// let rx = pipe::OpenOptions::new().open_receiver("path/to/a/fifo")?; + /// + /// let mut msg = vec![0; 1024]; + /// + /// loop { + /// // Wait for the pipe to be readable + /// rx.readable().await?; + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match rx.try_read(&mut msg) { + /// Ok(n) => { + /// msg.truncate(n); + /// break; + /// } + /// Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// println!("GOT = {:?}", msg); + /// Ok(()) + /// } + /// ``` + pub fn try_read(&self, buf: &mut [u8]) -> io::Result { + self.io + .registration() + .try_io(Interest::READABLE, || (&*self.io).read(buf)) + } + + /// Tries to read data from the pipe into the provided buffers, returning + /// how many bytes were read. + /// + /// Data is copied to fill each buffer in order, with the final buffer + /// written to possibly being only partially filled. This method behaves + /// equivalently to a single call to [`try_read()`] with concatenated + /// buffers. + /// + /// Reads any pending data from the pipe but does not wait for new data + /// to arrive. On success, returns the number of bytes read. Because + /// `try_read_vectored()` is non-blocking, the buffer does not have to be + /// stored by the async task and can exist entirely on the stack. + /// + /// Usually, [`readable()`] is used with this function. + /// + /// [`try_read()`]: Self::try_read() + /// [`readable()`]: Self::readable() + /// + /// # Return + /// + /// If data is successfully read, `Ok(n)` is returned, where `n` is the + /// number of bytes read. `Ok(0)` indicates the pipe's writing end is + /// closed and will no longer write data. If the pipe is not ready to read + /// data `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Open a reading end of a fifo + /// let rx = pipe::OpenOptions::new().open_receiver("path/to/a/fifo")?; + /// + /// loop { + /// // Wait for the pipe to be readable + /// rx.readable().await?; + /// + /// // Creating the buffer **after** the `await` prevents it from + /// // being stored in the async task. + /// let mut buf_a = [0; 512]; + /// let mut buf_b = [0; 1024]; + /// let mut bufs = [ + /// io::IoSliceMut::new(&mut buf_a), + /// io::IoSliceMut::new(&mut buf_b), + /// ]; + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match rx.try_read_vectored(&mut bufs) { + /// Ok(0) => break, + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_read_vectored(&self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result { + self.io + .registration() + .try_io(Interest::READABLE, || (&*self.io).read_vectored(bufs)) + } + + cfg_io_util! { + /// Tries to read data from the pipe into the provided buffer, advancing the + /// buffer's internal cursor, returning how many bytes were read. + /// + /// Reads any pending data from the pipe but does not wait for new data + /// to arrive. On success, returns the number of bytes read. Because + /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by + /// the async task and can exist entirely on the stack. + /// + /// Usually, [`readable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: Self::readable + /// [`ready()`]: Self::ready + /// + /// # Return + /// + /// If data is successfully read, `Ok(n)` is returned, where `n` is the + /// number of bytes read. `Ok(0)` indicates the pipe's writing end is + /// closed and will no longer write data. If the pipe is not ready to read + /// data `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::unix::pipe; + /// use std::io; + /// + /// #[tokio::main] + /// async fn main() -> io::Result<()> { + /// // Open a reading end of a fifo + /// let rx = pipe::OpenOptions::new().open_receiver("path/to/a/fifo")?; + /// + /// loop { + /// // Wait for the pipe to be readable + /// rx.readable().await?; + /// + /// let mut buf = Vec::with_capacity(4096); + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match rx.try_read_buf(&mut buf) { + /// Ok(0) => break, + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_read_buf(&self, buf: &mut B) -> io::Result { + self.io.registration().try_io(Interest::READABLE, || { + use std::io::Read; + + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + + // Safety: `mio_pipe::Receiver` uses a `std::fs::File` underneath, + // which correctly handles reads into uninitialized memory. + let n = (&*self.io).read(dst)?; + + unsafe { + buf.advance_mut(n); + } + + Ok(n) + }) + } + } +} + +impl AsyncRead for Receiver { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + // Safety: `mio_pipe::Receiver` uses a `std::fs::File` underneath, + // which correctly handles reads into uninitialized memory. + unsafe { self.io.poll_read(cx, buf) } + } +} + +impl AsRawFd for Receiver { + fn as_raw_fd(&self) -> RawFd { + self.io.as_raw_fd() + } +} + +#[cfg(not(tokio_no_as_fd))] +impl AsFd for Receiver { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} + +/// Checks if file is a FIFO +fn is_fifo(file: &File) -> io::Result { + Ok(file.metadata()?.file_type().is_fifo()) +} + +/// Gets file descriptor's flags by fcntl. +fn get_file_flags(file: &File) -> io::Result { + let fd = file.as_raw_fd(); + let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if flags < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(flags) + } +} + +/// Checks for O_RDONLY or O_RDWR access mode. +fn has_read_access(flags: libc::c_int) -> bool { + let mode = flags & libc::O_ACCMODE; + mode == libc::O_RDONLY || mode == libc::O_RDWR +} + +/// Checks for O_WRONLY or O_RDWR access mode. +fn has_write_access(flags: libc::c_int) -> bool { + let mode = flags & libc::O_ACCMODE; + mode == libc::O_WRONLY || mode == libc::O_RDWR +} + +/// Sets file's flags with O_NONBLOCK by fcntl. +fn set_nonblocking(file: &mut File, current_flags: libc::c_int) -> io::Result<()> { + let fd = file.as_raw_fd(); + + let flags = current_flags | libc::O_NONBLOCK; + + if flags != current_flags { + let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flags) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + } + + Ok(()) +} diff --git a/third_party/rust/tokio/src/net/unix/split.rs b/third_party/rust/tokio/src/net/unix/split.rs index d4686c22d78c..6fc7067a7265 100644 --- a/third_party/rust/tokio/src/net/unix/split.rs +++ b/third_party/rust/tokio/src/net/unix/split.rs @@ -55,9 +55,20 @@ pub(crate) fn split(stream: &mut UnixStream) -> (ReadHalf<'_>, WriteHalf<'_>) { impl ReadHalf<'_> { /// Wait for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_read()`]. It can be used instead + /// of [`readable()`] to check the returned ready set for [`Ready::READABLE`] + /// and [`Ready::READ_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// This function is equivalent to [`UnixStream::ready`]. + /// + /// [`try_read()`]: Self::try_read + /// [`readable()`]: Self::readable /// /// # Cancel safety /// @@ -100,8 +111,12 @@ impl ReadHalf<'_> { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result { self.0.try_read(buf) @@ -174,9 +189,20 @@ impl ReadHalf<'_> { impl WriteHalf<'_> { /// Waits for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_write()`]. It can be used instead + /// of [`writable()`] to check the returned ready set for [`Ready::WRITABLE`] + /// and [`Ready::WRITE_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// This function is equivalent to [`UnixStream::ready`]. + /// + /// [`try_write()`]: Self::try_write + /// [`writable()`]: Self::writable /// /// # Cancel safety /// diff --git a/third_party/rust/tokio/src/net/unix/split_owned.rs b/third_party/rust/tokio/src/net/unix/split_owned.rs index 9c3a2a41777f..d1c6f93ded76 100644 --- a/third_party/rust/tokio/src/net/unix/split_owned.rs +++ b/third_party/rust/tokio/src/net/unix/split_owned.rs @@ -110,9 +110,20 @@ impl OwnedReadHalf { /// Waits for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_read()`]. It can be used instead + /// of [`readable()`] to check the returned ready set for [`Ready::READABLE`] + /// and [`Ready::READ_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// This function is equivalent to [`UnixStream::ready`]. + /// + /// [`try_read()`]: Self::try_read + /// [`readable()`]: Self::readable /// /// # Cancel safety /// @@ -155,8 +166,12 @@ impl OwnedReadHalf { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. pub fn try_read(&self, buf: &mut [u8]) -> io::Result { self.inner.try_read(buf) @@ -257,9 +272,20 @@ impl OwnedWriteHalf { /// Waits for any of the requested ready states. /// - /// This function is usually paired with `try_read()` or `try_write()`. It - /// can be used to concurrently read / write to the same socket on a single - /// task without splitting the socket. + /// This function is usually paired with [`try_write()`]. It can be used instead + /// of [`writable()`] to check the returned ready set for [`Ready::WRITABLE`] + /// and [`Ready::WRITE_CLOSED`] events. + /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// + /// This function is equivalent to [`UnixStream::ready`]. + /// + /// [`try_write()`]: Self::try_write + /// [`writable()`]: Self::writable /// /// # Cancel safety /// @@ -382,12 +408,12 @@ impl AsyncWrite for OwnedWriteHalf { impl AsRef for OwnedReadHalf { fn as_ref(&self) -> &UnixStream { - &*self.inner + &self.inner } } impl AsRef for OwnedWriteHalf { fn as_ref(&self) -> &UnixStream { - &*self.inner + &self.inner } } diff --git a/third_party/rust/tokio/src/net/unix/stream.rs b/third_party/rust/tokio/src/net/unix/stream.rs index 4e7ef87b4161..201645ba1d8e 100644 --- a/third_party/rust/tokio/src/net/unix/stream.rs +++ b/third_party/rust/tokio/src/net/unix/stream.rs @@ -5,10 +5,11 @@ use crate::net::unix::split_owned::{split_owned, OwnedReadHalf, OwnedWriteHalf}; use crate::net::unix::ucred::{self, UCred}; use crate::net::unix::SocketAddr; -use std::convert::TryFrom; use std::fmt; use std::io::{self, Read, Write}; use std::net::Shutdown; +#[cfg(not(tokio_no_as_fd))] +use std::os::unix::io::{AsFd, BorrowedFd}; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::os::unix::net; use std::path::Path; @@ -22,8 +23,8 @@ cfg_io_util! { cfg_net_unix! { /// A structure representing a connected Unix socket. /// - /// This socket can be connected directly with `UnixStream::connect` or accepted - /// from a listener with `UnixListener::incoming`. Additionally, a pair of + /// This socket can be connected directly with [`UnixStream::connect`] or accepted + /// from a listener with [`UnixListener::accept`]. Additionally, a pair of /// anonymous Unix sockets can be created with `UnixStream::pair`. /// /// To shut down the stream in the write direction, you can call the @@ -32,6 +33,8 @@ cfg_net_unix! { /// the stream in one direction. /// /// [`shutdown()`]: fn@crate::io::AsyncWriteExt::shutdown + /// [`UnixListener::accept`]: crate::net::UnixListener::accept + #[cfg_attr(docsrs, doc(alias = "uds"))] pub struct UnixStream { io: PollEvented, } @@ -65,6 +68,12 @@ impl UnixStream { /// can be used to concurrently read / write to the same socket on a single /// task without splitting the socket. /// + /// The function may complete without the socket being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Cancel safety /// /// This method is cancel safe. Once a readiness event occurs, the method @@ -239,8 +248,12 @@ impl UnixStream { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the stream's read half is closed - /// and will no longer yield data. If the stream is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The stream's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the stream is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -656,7 +669,7 @@ impl UnixStream { /// Tries to read or write from the socket using a user-provided IO operation. /// /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// should attempt to perform IO operation on the socket by manually /// calling the appropriate syscall. If the operation fails because the /// socket is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value @@ -675,6 +688,11 @@ impl UnixStream { /// defined on the Tokio `UnixStream` type, as this will mess with the /// readiness flag and can cause the socket to behave incorrectly. /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// /// [`readable()`]: UnixStream::readable() @@ -685,23 +703,85 @@ impl UnixStream { interest: Interest, f: impl FnOnce() -> io::Result, ) -> io::Result { - self.io.registration().try_io(interest, f) + self.io + .registration() + .try_io(interest, || self.io.try_io(f)) + } + + /// Reads or writes from the socket using a user-provided IO operation. + /// + /// The readiness of the socket is awaited and when the socket is ready, + /// the provided closure is called. The closure should attempt to perform + /// IO operation on the socket by manually calling the appropriate syscall. + /// If the operation fails because the socket is not actually ready, + /// then the closure should return a `WouldBlock` error. In such case the + /// readiness flag is cleared and the socket readiness is awaited again. + /// This loop is repeated until the closure returns an `Ok` or an error + /// other than `WouldBlock`. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the socket that failed due to the socket not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the socket to + /// behave incorrectly. + /// + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `UnixStream` type, as this will mess with the + /// readiness flag and can cause the socket to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + pub async fn async_io( + &self, + interest: Interest, + mut f: impl FnMut() -> io::Result, + ) -> io::Result { + self.io + .registration() + .async_io(interest, || self.io.try_io(&mut f)) + .await } /// Creates new `UnixStream` from a `std::os::unix::net::UnixStream`. /// /// This function is intended to be used to wrap a UnixStream from the - /// standard library in the Tokio equivalent. The conversion assumes - /// nothing about the underlying stream; it is left up to the user to set - /// it in non-blocking mode. + /// standard library in the Tokio equivalent. + /// + /// # Notes + /// + /// The caller is responsible for ensuring that the stream is in + /// non-blocking mode. Otherwise all I/O operations on the stream + /// will block the thread, which will cause unexpected behavior. + /// Non-blocking mode can be set using [`set_nonblocking`]. + /// + /// [`set_nonblocking`]: std::os::unix::net::UnixStream::set_nonblocking + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::UnixStream; + /// use std::os::unix::net::UnixStream as StdUnixStream; + /// # use std::error::Error; + /// + /// # async fn dox() -> Result<(), Box> { + /// let std_stream = StdUnixStream::connect("/path/to/the/socket")?; + /// std_stream.set_nonblocking(true)?; + /// let stream = UnixStream::from_std(std_stream)?; + /// # Ok(()) + /// # } + /// ``` /// /// # Panics /// - /// This function panics if thread-local runtime is not set. + /// This function panics if it is not called from within a runtime with + /// IO enabled. /// /// The runtime is usually set implicitly when this function is called /// from a future driven by a tokio runtime, otherwise runtime can be set /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function. + #[track_caller] pub fn from_std(stream: net::UnixStream) -> io::Result { let stream = mio::net::UnixStream::from_std(stream); let io = PollEvented::new(stream)?; @@ -958,3 +1038,10 @@ impl AsRawFd for UnixStream { self.io.as_raw_fd() } } + +#[cfg(not(tokio_no_as_fd))] +impl AsFd for UnixStream { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} diff --git a/third_party/rust/tokio/src/net/unix/ucred.rs b/third_party/rust/tokio/src/net/unix/ucred.rs index 865303b4cefa..edfab08ab087 100644 --- a/third_party/rust/tokio/src/net/unix/ucred.rs +++ b/third_party/rust/tokio/src/net/unix/ucred.rs @@ -1,24 +1,24 @@ -use libc::{gid_t, pid_t, uid_t}; +use crate::net::unix; /// Credentials of a process. #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] pub struct UCred { /// PID (process ID) of the process. - pid: Option, + pid: Option, /// UID (user ID) of the process. - uid: uid_t, + uid: unix::uid_t, /// GID (group ID) of the process. - gid: gid_t, + gid: unix::gid_t, } impl UCred { /// Gets UID (user ID) of the process. - pub fn uid(&self) -> uid_t { + pub fn uid(&self) -> unix::uid_t { self.uid } /// Gets GID (group ID) of the process. - pub fn gid(&self) -> gid_t { + pub fn gid(&self) -> unix::gid_t { self.gid } @@ -26,12 +26,17 @@ impl UCred { /// /// This is only implemented under Linux, Android, iOS, macOS, Solaris and /// Illumos. On other platforms this will always return `None`. - pub fn pid(&self) -> Option { + pub fn pid(&self) -> Option { self.pid } } -#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] +#[cfg(any( + target_os = "linux", + target_os = "redox", + target_os = "android", + target_os = "openbsd" +))] pub(crate) use self::impl_linux::get_peer_cred; #[cfg(any(target_os = "netbsd"))] @@ -46,16 +51,24 @@ pub(crate) use self::impl_macos::get_peer_cred; #[cfg(any(target_os = "solaris", target_os = "illumos"))] pub(crate) use self::impl_solaris::get_peer_cred; -#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] +#[cfg(target_os = "aix")] +pub(crate) use self::impl_aix::get_peer_cred; + +#[cfg(any( + target_os = "linux", + target_os = "redox", + target_os = "android", + target_os = "openbsd" +))] pub(crate) mod impl_linux { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::{c_void, getsockopt, socklen_t, SOL_SOCKET, SO_PEERCRED}; use std::{io, mem}; #[cfg(target_os = "openbsd")] use libc::sockpeercred as ucred; - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(any(target_os = "linux", target_os = "redox", target_os = "android"))] use libc::ucred; pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result { @@ -87,9 +100,9 @@ pub(crate) mod impl_linux { ); if ret == 0 && ucred_size as usize == mem::size_of::() { Ok(super::UCred { - uid: ucred.uid, - gid: ucred.gid, - pid: Some(ucred.pid), + uid: ucred.uid as unix::uid_t, + gid: ucred.gid as unix::gid_t, + pid: Some(ucred.pid as unix::pid_t), }) } else { Err(io::Error::last_os_error()) @@ -100,7 +113,7 @@ pub(crate) mod impl_linux { #[cfg(any(target_os = "netbsd"))] pub(crate) mod impl_netbsd { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::{c_void, getsockopt, socklen_t, unpcbid, LOCAL_PEEREID, SOL_SOCKET}; use std::io; @@ -129,9 +142,9 @@ pub(crate) mod impl_netbsd { ); if ret == 0 && unpcbid_size as usize == size_of::() { Ok(super::UCred { - uid: unpcbid.unp_euid, - gid: unpcbid.unp_egid, - pid: Some(unpcbid.unp_pid), + uid: unpcbid.unp_euid as unix::uid_t, + gid: unpcbid.unp_egid as unix::gid_t, + pid: Some(unpcbid.unp_pid as unix::pid_t), }) } else { Err(io::Error::last_os_error()) @@ -142,7 +155,7 @@ pub(crate) mod impl_netbsd { #[cfg(any(target_os = "dragonfly", target_os = "freebsd"))] pub(crate) mod impl_bsd { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::getpeereid; use std::io; @@ -160,8 +173,8 @@ pub(crate) mod impl_bsd { if ret == 0 { Ok(super::UCred { - uid: uid.assume_init(), - gid: gid.assume_init(), + uid: uid.assume_init() as unix::uid_t, + gid: gid.assume_init() as unix::gid_t, pid: None, }) } else { @@ -173,7 +186,7 @@ pub(crate) mod impl_bsd { #[cfg(any(target_os = "macos", target_os = "ios"))] pub(crate) mod impl_macos { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use libc::{c_void, getpeereid, getsockopt, pid_t, LOCAL_PEEREPID, SOL_LOCAL}; use std::io; @@ -207,9 +220,9 @@ pub(crate) mod impl_macos { if ret == 0 { Ok(super::UCred { - uid: uid.assume_init(), - gid: gid.assume_init(), - pid: Some(pid.assume_init()), + uid: uid.assume_init() as unix::uid_t, + gid: gid.assume_init() as unix::gid_t, + pid: Some(pid.assume_init() as unix::pid_t), }) } else { Err(io::Error::last_os_error()) @@ -220,7 +233,7 @@ pub(crate) mod impl_macos { #[cfg(any(target_os = "solaris", target_os = "illumos"))] pub(crate) mod impl_solaris { - use crate::net::unix::UnixStream; + use crate::net::unix::{self, UnixStream}; use std::io; use std::os::unix::io::AsRawFd; use std::ptr; @@ -240,9 +253,37 @@ pub(crate) mod impl_solaris { libc::ucred_free(cred); Ok(super::UCred { - uid, - gid, - pid: Some(pid), + uid: uid as unix::uid_t, + gid: gid as unix::gid_t, + pid: Some(pid as unix::pid_t), + }) + } else { + Err(io::Error::last_os_error()) + } + } + } +} + +#[cfg(target_os = "aix")] +pub(crate) mod impl_aix { + use crate::net::unix::UnixStream; + use std::io; + use std::os::unix::io::AsRawFd; + + pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result { + unsafe { + let raw_fd = sock.as_raw_fd(); + + let mut uid = std::mem::MaybeUninit::uninit(); + let mut gid = std::mem::MaybeUninit::uninit(); + + let ret = libc::getpeereid(raw_fd, uid.as_mut_ptr(), gid.as_mut_ptr()); + + if ret == 0 { + Ok(super::UCred { + uid: uid.assume_init(), + gid: gid.assume_init(), + pid: None, }) } else { Err(io::Error::last_os_error()) diff --git a/third_party/rust/tokio/src/net/windows/named_pipe.rs b/third_party/rust/tokio/src/net/windows/named_pipe.rs index 550fd4df2bc9..91f016267caf 100644 --- a/third_party/rust/tokio/src/net/windows/named_pipe.rs +++ b/third_party/rust/tokio/src/net/windows/named_pipe.rs @@ -10,27 +10,30 @@ use std::ptr; use std::task::{Context, Poll}; use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready}; +#[cfg(not(tokio_no_as_fd))] +use crate::os::windows::io::{AsHandle, BorrowedHandle}; use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle}; +cfg_io_util! { + use bytes::BufMut; +} + // Hide imports which are not used when generating documentation. #[cfg(not(docsrs))] mod doc { pub(super) use crate::os::windows::ffi::OsStrExt; - pub(super) use crate::winapi::shared::minwindef::{DWORD, FALSE}; - pub(super) use crate::winapi::um::fileapi; - pub(super) use crate::winapi::um::handleapi; - pub(super) use crate::winapi::um::namedpipeapi; - pub(super) use crate::winapi::um::winbase; - pub(super) use crate::winapi::um::winnt; - + pub(super) mod windows_sys { + pub(crate) use windows_sys::{ + Win32::Foundation::*, Win32::Storage::FileSystem::*, Win32::System::Pipes::*, + Win32::System::SystemServices::*, + }; + } pub(super) use mio::windows as mio_windows; } // NB: none of these shows up in public API, so don't document them. #[cfg(docsrs)] mod doc { - pub type DWORD = crate::doc::NotDefinedHere; - pub(super) mod mio_windows { pub type NamedPipe = crate::doc::NotDefinedHere; } @@ -97,7 +100,6 @@ use self::doc::*; /// # Ok(()) } /// ``` /// -/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY /// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes #[derive(Debug)] pub struct NamedPipeServer { @@ -188,17 +190,15 @@ impl NamedPipeServer { /// # Ok(()) } /// ``` pub async fn connect(&self) -> io::Result<()> { - loop { - match self.io.connect() { - Ok(()) => break, - Err(e) if e.kind() == io::ErrorKind::WouldBlock => { - self.io.registration().readiness(Interest::WRITABLE).await?; - } - Err(e) => return Err(e), + match self.io.connect() { + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.io + .registration() + .async_io(Interest::WRITABLE, || self.io.connect()) + .await } + x => x, } - - Ok(()) } /// Disconnects the server end of a named pipe instance from a client @@ -207,7 +207,7 @@ impl NamedPipeServer { /// ``` /// use tokio::io::AsyncWriteExt; /// use tokio::net::windows::named_pipe::{ClientOptions, ServerOptions}; - /// use winapi::shared::winerror; + /// use windows_sys::Win32::Foundation::ERROR_PIPE_NOT_CONNECTED; /// /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-disconnect"; /// @@ -227,7 +227,7 @@ impl NamedPipeServer { /// // Write fails with an OS-specific error after client has been /// // disconnected. /// let e = client.write(b"ping").await.unwrap_err(); - /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_NOT_CONNECTED as i32)); + /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_NOT_CONNECTED as i32)); /// # Ok(()) } /// ``` pub fn disconnect(&self) -> io::Result<()> { @@ -240,6 +240,12 @@ impl NamedPipeServer { /// can be used to concurrently read / write to the same pipe on a single /// task without splitting the pipe. /// + /// The function may complete without the pipe being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Examples /// /// Concurrently read and write to the pipe on the same task without @@ -399,8 +405,12 @@ impl NamedPipeServer { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the pipe's read half is closed - /// and will no longer yield data. If the pipe is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The pipe's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the pipe is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -528,6 +538,86 @@ impl NamedPipeServer { .try_io(Interest::READABLE, || (&*self.io).read_vectored(bufs)) } + cfg_io_util! { + /// Tries to read data from the stream into the provided buffer, advancing the + /// buffer's internal cursor, returning how many bytes were read. + /// + /// Receives any pending data from the pipe but does not wait for new data + /// to arrive. On success, returns the number of bytes read. Because + /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by + /// the async task and can exist entirely on the stack. + /// + /// Usually, [`readable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: NamedPipeServer::readable() + /// [`ready()`]: NamedPipeServer::ready() + /// + /// # Return + /// + /// If data is successfully read, `Ok(n)` is returned, where `n` is the + /// number of bytes read. `Ok(0)` indicates the stream's read half is closed + /// and will no longer yield data. If the stream is not ready to read data + /// `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::windows::named_pipe; + /// use std::error::Error; + /// use std::io; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-client-readable"; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let server = named_pipe::ServerOptions::new().create(PIPE_NAME)?; + /// + /// loop { + /// // Wait for the pipe to be readable + /// server.readable().await?; + /// + /// let mut buf = Vec::with_capacity(4096); + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match server.try_read_buf(&mut buf) { + /// Ok(0) => break, + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_read_buf(&self, buf: &mut B) -> io::Result { + self.io.registration().try_io(Interest::READABLE, || { + use std::io::Read; + + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + + // Safety: We trust `NamedPipeServer::read` to have filled up `n` bytes in the + // buffer. + let n = (&*self.io).read(dst)?; + + unsafe { + buf.advance_mut(n); + } + + Ok(n) + }) + } + } + /// Waits for the pipe to become writable. /// /// This function is equivalent to `ready(Interest::WRITABLE)` and is usually @@ -724,27 +814,32 @@ impl NamedPipeServer { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } - /// Tries to read or write from the socket using a user-provided IO operation. + /// Tries to read or write from the pipe using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// If the pipe is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the pipe by manually /// calling the appropriate syscall. If the operation fails because the - /// socket is not actually ready, then the closure should return a + /// pipe is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value /// of the closure is then returned by `try_io`. /// - /// If the socket is not ready, then the closure is not called + /// If the pipe is not ready, then the closure is not called /// and a `WouldBlock` error is returned. /// /// The closure should only return a `WouldBlock` error if it has performed - /// an IO operation on the socket that failed due to the socket not being + /// an IO operation on the pipe that failed due to the pipe not being /// ready. Returning a `WouldBlock` error in any other situation will - /// incorrectly clear the readiness flag, which can cause the socket to + /// incorrectly clear the readiness flag, which can cause the pipe to /// behave incorrectly. /// /// The closure should not perform the IO operation using any of the /// methods defined on the Tokio `NamedPipeServer` type, as this will mess with - /// the readiness flag and can cause the socket to behave incorrectly. + /// the readiness flag and can cause the pipe to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// @@ -758,6 +853,39 @@ impl NamedPipeServer { ) -> io::Result { self.io.registration().try_io(interest, f) } + + /// Reads or writes from the pipe using a user-provided IO operation. + /// + /// The readiness of the pipe is awaited and when the pipe is ready, + /// the provided closure is called. The closure should attempt to perform + /// IO operation on the pipe by manually calling the appropriate syscall. + /// If the operation fails because the pipe is not actually ready, + /// then the closure should return a `WouldBlock` error. In such case the + /// readiness flag is cleared and the pipe readiness is awaited again. + /// This loop is repeated until the closure returns an `Ok` or an error + /// other than `WouldBlock`. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the pipe that failed due to the pipe not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the pipe to + /// behave incorrectly. + /// + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `NamedPipeServer` type, as this will mess with the + /// readiness flag and can cause the pipe to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + pub async fn async_io( + &self, + interest: Interest, + f: impl FnMut() -> io::Result, + ) -> io::Result { + self.io.registration().async_io(interest, f).await + } } impl AsyncRead for NamedPipeServer { @@ -802,6 +930,13 @@ impl AsRawHandle for NamedPipeServer { } } +#[cfg(not(tokio_no_as_fd))] +impl AsHandle for NamedPipeServer { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + /// A [Windows named pipe] client. /// /// Constructed using [`ClientOptions::open`]. @@ -819,7 +954,7 @@ impl AsRawHandle for NamedPipeServer { /// use std::time::Duration; /// use tokio::net::windows::named_pipe::ClientOptions; /// use tokio::time; -/// use winapi::shared::winerror; +/// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY; /// /// const PIPE_NAME: &str = r"\\.\pipe\named-pipe-idiomatic-client"; /// @@ -827,7 +962,7 @@ impl AsRawHandle for NamedPipeServer { /// let client = loop { /// match ClientOptions::new().open(PIPE_NAME) { /// Ok(client) => break client, -/// Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), +/// Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), /// Err(e) => return Err(e), /// } /// @@ -838,7 +973,7 @@ impl AsRawHandle for NamedPipeServer { /// # Ok(()) } /// ``` /// -/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY +/// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html /// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes #[derive(Debug)] pub struct NamedPipeClient { @@ -902,6 +1037,12 @@ impl NamedPipeClient { /// can be used to concurrently read / write to the same pipe on a single /// task without splitting the pipe. /// + /// The function may complete without the pipe being ready. This is a + /// false-positive and attempting an operation will return with + /// `io::ErrorKind::WouldBlock`. The function can also return with an empty + /// [`Ready`] set, so you should always check the returned value and possibly + /// wait again if the requested states are not set. + /// /// # Examples /// /// Concurrently read and write to the pipe on the same task without @@ -1059,8 +1200,12 @@ impl NamedPipeClient { /// # Return /// /// If data is successfully read, `Ok(n)` is returned, where `n` is the - /// number of bytes read. `Ok(0)` indicates the pipe's read half is closed - /// and will no longer yield data. If the pipe is not ready to read data + /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios: + /// + /// 1. The pipe's read half is closed and will no longer yield data. + /// 2. The specified buffer was 0 bytes in length. + /// + /// If the pipe is not ready to read data, /// `Err(io::ErrorKind::WouldBlock)` is returned. /// /// # Examples @@ -1186,6 +1331,86 @@ impl NamedPipeClient { .try_io(Interest::READABLE, || (&*self.io).read_vectored(bufs)) } + cfg_io_util! { + /// Tries to read data from the stream into the provided buffer, advancing the + /// buffer's internal cursor, returning how many bytes were read. + /// + /// Receives any pending data from the pipe but does not wait for new data + /// to arrive. On success, returns the number of bytes read. Because + /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by + /// the async task and can exist entirely on the stack. + /// + /// Usually, [`readable()`] or [`ready()`] is used with this function. + /// + /// [`readable()`]: NamedPipeClient::readable() + /// [`ready()`]: NamedPipeClient::ready() + /// + /// # Return + /// + /// If data is successfully read, `Ok(n)` is returned, where `n` is the + /// number of bytes read. `Ok(0)` indicates the stream's read half is closed + /// and will no longer yield data. If the stream is not ready to read data + /// `Err(io::ErrorKind::WouldBlock)` is returned. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::net::windows::named_pipe; + /// use std::error::Error; + /// use std::io; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-client-readable"; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let client = named_pipe::ClientOptions::new().open(PIPE_NAME)?; + /// + /// loop { + /// // Wait for the pipe to be readable + /// client.readable().await?; + /// + /// let mut buf = Vec::with_capacity(4096); + /// + /// // Try to read data, this may still fail with `WouldBlock` + /// // if the readiness event is a false positive. + /// match client.try_read_buf(&mut buf) { + /// Ok(0) => break, + /// Ok(n) => { + /// println!("read {} bytes", n); + /// } + /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + /// continue; + /// } + /// Err(e) => { + /// return Err(e.into()); + /// } + /// } + /// } + /// + /// Ok(()) + /// } + /// ``` + pub fn try_read_buf(&self, buf: &mut B) -> io::Result { + self.io.registration().try_io(Interest::READABLE, || { + use std::io::Read; + + let dst = buf.chunk_mut(); + let dst = + unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit] as *mut [u8]) }; + + // Safety: We trust `NamedPipeClient::read` to have filled up `n` bytes in the + // buffer. + let n = (&*self.io).read(dst)?; + + unsafe { + buf.advance_mut(n); + } + + Ok(n) + }) + } + } + /// Waits for the pipe to become writable. /// /// This function is equivalent to `ready(Interest::WRITABLE)` and is usually @@ -1379,27 +1604,32 @@ impl NamedPipeClient { .try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf)) } - /// Tries to read or write from the socket using a user-provided IO operation. + /// Tries to read or write from the pipe using a user-provided IO operation. /// - /// If the socket is ready, the provided closure is called. The closure - /// should attempt to perform IO operation from the socket by manually + /// If the pipe is ready, the provided closure is called. The closure + /// should attempt to perform IO operation from the pipe by manually /// calling the appropriate syscall. If the operation fails because the - /// socket is not actually ready, then the closure should return a + /// pipe is not actually ready, then the closure should return a /// `WouldBlock` error and the readiness flag is cleared. The return value /// of the closure is then returned by `try_io`. /// - /// If the socket is not ready, then the closure is not called + /// If the pipe is not ready, then the closure is not called /// and a `WouldBlock` error is returned. /// /// The closure should only return a `WouldBlock` error if it has performed - /// an IO operation on the socket that failed due to the socket not being + /// an IO operation on the pipe that failed due to the pipe not being /// ready. Returning a `WouldBlock` error in any other situation will - /// incorrectly clear the readiness flag, which can cause the socket to + /// incorrectly clear the readiness flag, which can cause the pipe to /// behave incorrectly. /// /// The closure should not perform the IO operation using any of the methods /// defined on the Tokio `NamedPipeClient` type, as this will mess with the - /// readiness flag and can cause the socket to behave incorrectly. + /// readiness flag and can cause the pipe to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. /// /// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function. /// @@ -1413,6 +1643,39 @@ impl NamedPipeClient { ) -> io::Result { self.io.registration().try_io(interest, f) } + + /// Reads or writes from the pipe using a user-provided IO operation. + /// + /// The readiness of the pipe is awaited and when the pipe is ready, + /// the provided closure is called. The closure should attempt to perform + /// IO operation on the pipe by manually calling the appropriate syscall. + /// If the operation fails because the pipe is not actually ready, + /// then the closure should return a `WouldBlock` error. In such case the + /// readiness flag is cleared and the pipe readiness is awaited again. + /// This loop is repeated until the closure returns an `Ok` or an error + /// other than `WouldBlock`. + /// + /// The closure should only return a `WouldBlock` error if it has performed + /// an IO operation on the pipe that failed due to the pipe not being + /// ready. Returning a `WouldBlock` error in any other situation will + /// incorrectly clear the readiness flag, which can cause the pipe to + /// behave incorrectly. + /// + /// The closure should not perform the IO operation using any of the methods + /// defined on the Tokio `NamedPipeClient` type, as this will mess with the + /// readiness flag and can cause the pipe to behave incorrectly. + /// + /// This method is not intended to be used with combined interests. + /// The closure should perform only one type of IO operation, so it should not + /// require more than one ready state. This method may panic or sleep forever + /// if it is called with a combined interest. + pub async fn async_io( + &self, + interest: Interest, + f: impl FnMut() -> io::Result, + ) -> io::Result { + self.io.registration().async_io(interest, f).await + } } impl AsyncRead for NamedPipeClient { @@ -1457,17 +1720,11 @@ impl AsRawHandle for NamedPipeClient { } } -// Helper to set a boolean flag as a bitfield. -macro_rules! bool_flag { - ($f:expr, $t:expr, $flag:expr) => {{ - let current = $f; - - if $t { - $f = current | $flag; - } else { - $f = current & !$flag; - }; - }}; +#[cfg(not(tokio_no_as_fd))] +impl AsHandle for NamedPipeClient { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } } /// A builder structure for construct a named pipe with named pipe-specific @@ -1477,12 +1734,21 @@ macro_rules! bool_flag { /// See [`ServerOptions::create`]. #[derive(Debug, Clone)] pub struct ServerOptions { - open_mode: DWORD, - pipe_mode: DWORD, - max_instances: DWORD, - out_buffer_size: DWORD, - in_buffer_size: DWORD, - default_timeout: DWORD, + // dwOpenMode + access_inbound: bool, + access_outbound: bool, + first_pipe_instance: bool, + write_dac: bool, + write_owner: bool, + access_system_security: bool, + // dwPipeMode + pipe_mode: PipeMode, + reject_remote_clients: bool, + // other options + max_instances: u32, + out_buffer_size: u32, + in_buffer_size: u32, + default_timeout: u32, } impl ServerOptions { @@ -1499,9 +1765,15 @@ impl ServerOptions { /// ``` pub fn new() -> ServerOptions { ServerOptions { - open_mode: winbase::PIPE_ACCESS_DUPLEX | winbase::FILE_FLAG_OVERLAPPED, - pipe_mode: winbase::PIPE_TYPE_BYTE | winbase::PIPE_REJECT_REMOTE_CLIENTS, - max_instances: winbase::PIPE_UNLIMITED_INSTANCES, + access_inbound: true, + access_outbound: true, + first_pipe_instance: false, + write_dac: false, + write_owner: false, + access_system_security: false, + pipe_mode: PipeMode::Byte, + reject_remote_clients: true, + max_instances: windows_sys::PIPE_UNLIMITED_INSTANCES, out_buffer_size: 65536, in_buffer_size: 65536, default_timeout: 0, @@ -1513,15 +1785,11 @@ impl ServerOptions { /// The default pipe mode is [`PipeMode::Byte`]. See [`PipeMode`] for /// documentation of what each mode means. /// - /// This corresponding to specifying [`dwPipeMode`]. + /// This corresponds to specifying `PIPE_TYPE_` and `PIPE_READMODE_` in [`dwPipeMode`]. /// /// [`dwPipeMode`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea pub fn pipe_mode(&mut self, pipe_mode: PipeMode) -> &mut Self { - self.pipe_mode = match pipe_mode { - PipeMode::Byte => winbase::PIPE_TYPE_BYTE, - PipeMode::Message => winbase::PIPE_TYPE_MESSAGE, - }; - + self.pipe_mode = pipe_mode; self } @@ -1617,7 +1885,7 @@ impl ServerOptions { /// # Ok(()) } /// ``` pub fn access_inbound(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_INBOUND); + self.access_inbound = allowed; self } @@ -1715,7 +1983,7 @@ impl ServerOptions { /// # Ok(()) } /// ``` pub fn access_outbound(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_OUTBOUND); + self.access_outbound = allowed; self } @@ -1783,11 +2051,109 @@ impl ServerOptions { /// [`create`]: ServerOptions::create /// [`FILE_FLAG_FIRST_PIPE_INSTANCE`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea#pipe_first_pipe_instance pub fn first_pipe_instance(&mut self, first: bool) -> &mut Self { - bool_flag!( - self.open_mode, - first, - winbase::FILE_FLAG_FIRST_PIPE_INSTANCE - ); + self.first_pipe_instance = first; + self + } + + /// Requests permission to modify the pipe's discretionary access control list. + /// + /// This corresponds to setting [`WRITE_DAC`] in dwOpenMode. + /// + /// # Examples + /// + /// ``` + /// use std::{io, os::windows::prelude::AsRawHandle, ptr}; + // + /// use tokio::net::windows::named_pipe::ServerOptions; + /// use windows_sys::{ + /// Win32::Foundation::ERROR_SUCCESS, + /// Win32::Security::DACL_SECURITY_INFORMATION, + /// Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT}, + /// }; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe"; + /// + /// # #[tokio::main] async fn main() -> io::Result<()> { + /// let mut pipe_template = ServerOptions::new(); + /// pipe_template.write_dac(true); + /// let pipe = pipe_template.create(PIPE_NAME)?; + /// + /// unsafe { + /// assert_eq!( + /// ERROR_SUCCESS, + /// SetSecurityInfo( + /// pipe.as_raw_handle() as _, + /// SE_KERNEL_OBJECT, + /// DACL_SECURITY_INFORMATION, + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ) + /// ); + /// } + /// + /// # Ok(()) } + /// ``` + /// + /// ``` + /// use std::{io, os::windows::prelude::AsRawHandle, ptr}; + // + /// use tokio::net::windows::named_pipe::ServerOptions; + /// use windows_sys::{ + /// Win32::Foundation::ERROR_ACCESS_DENIED, + /// Win32::Security::DACL_SECURITY_INFORMATION, + /// Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT}, + /// }; + /// + /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe_fail"; + /// + /// # #[tokio::main] async fn main() -> io::Result<()> { + /// let mut pipe_template = ServerOptions::new(); + /// pipe_template.write_dac(false); + /// let pipe = pipe_template.create(PIPE_NAME)?; + /// + /// unsafe { + /// assert_eq!( + /// ERROR_ACCESS_DENIED, + /// SetSecurityInfo( + /// pipe.as_raw_handle() as _, + /// SE_KERNEL_OBJECT, + /// DACL_SECURITY_INFORMATION, + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ptr::null_mut(), + /// ) + /// ); + /// } + /// + /// # Ok(()) } + /// ``` + /// + /// [`WRITE_DAC`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea + pub fn write_dac(&mut self, requested: bool) -> &mut Self { + self.write_dac = requested; + self + } + + /// Requests permission to modify the pipe's owner. + /// + /// This corresponds to setting [`WRITE_OWNER`] in dwOpenMode. + /// + /// [`WRITE_OWNER`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea + pub fn write_owner(&mut self, requested: bool) -> &mut Self { + self.write_owner = requested; + self + } + + /// Requests permission to modify the pipe's system access control list. + /// + /// This corresponds to setting [`ACCESS_SYSTEM_SECURITY`] in dwOpenMode. + /// + /// [`ACCESS_SYSTEM_SECURITY`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea + pub fn access_system_security(&mut self, requested: bool) -> &mut Self { + self.access_system_security = requested; self } @@ -1798,7 +2164,7 @@ impl ServerOptions { /// /// [`PIPE_REJECT_REMOTE_CLIENTS`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea#pipe_reject_remote_clients pub fn reject_remote_clients(&mut self, reject: bool) -> &mut Self { - bool_flag!(self.pipe_mode, reject, winbase::PIPE_REJECT_REMOTE_CLIENTS); + self.reject_remote_clients = reject; self } @@ -1820,7 +2186,7 @@ impl ServerOptions { /// ``` /// use std::io; /// use tokio::net::windows::named_pipe::{ServerOptions, ClientOptions}; - /// use winapi::shared::winerror; + /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY; /// /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-max-instances"; /// @@ -1836,11 +2202,11 @@ impl ServerOptions { /// /// // Too many servers! /// let e = server.create(PIPE_NAME).unwrap_err(); - /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32)); + /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32)); /// /// // Still too many servers even if we specify a higher value! /// let e = server.max_instances(100).create(PIPE_NAME).unwrap_err(); - /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32)); + /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32)); /// # Ok(()) } /// ``` /// @@ -1856,9 +2222,10 @@ impl ServerOptions { /// let builder = ServerOptions::new().max_instances(255); /// # Ok(()) } /// ``` + #[track_caller] pub fn max_instances(&mut self, instances: usize) -> &mut Self { assert!(instances < 255, "cannot specify more than 254 instances"); - self.max_instances = instances as DWORD; + self.max_instances = instances as u32; self } @@ -1868,7 +2235,7 @@ impl ServerOptions { /// /// [`nOutBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea pub fn out_buffer_size(&mut self, buffer: u32) -> &mut Self { - self.out_buffer_size = buffer as DWORD; + self.out_buffer_size = buffer; self } @@ -1878,7 +2245,7 @@ impl ServerOptions { /// /// [`nInBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea pub fn in_buffer_size(&mut self, buffer: u32) -> &mut Self { - self.in_buffer_size = buffer as DWORD; + self.in_buffer_size = buffer; self } @@ -1935,7 +2302,7 @@ impl ServerOptions { /// /// [`create`]: ServerOptions::create /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew - /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES + /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html pub unsafe fn create_with_security_attributes_raw( &self, addr: impl AsRef, @@ -1943,10 +2310,46 @@ impl ServerOptions { ) -> io::Result { let addr = encode_addr(addr); - let h = namedpipeapi::CreateNamedPipeW( + let pipe_mode = { + let mut mode = if matches!(self.pipe_mode, PipeMode::Message) { + windows_sys::PIPE_TYPE_MESSAGE | windows_sys::PIPE_READMODE_MESSAGE + } else { + windows_sys::PIPE_TYPE_BYTE | windows_sys::PIPE_READMODE_BYTE + }; + if self.reject_remote_clients { + mode |= windows_sys::PIPE_REJECT_REMOTE_CLIENTS; + } else { + mode |= windows_sys::PIPE_ACCEPT_REMOTE_CLIENTS; + } + mode + }; + let open_mode = { + let mut mode = windows_sys::FILE_FLAG_OVERLAPPED; + if self.access_inbound { + mode |= windows_sys::PIPE_ACCESS_INBOUND; + } + if self.access_outbound { + mode |= windows_sys::PIPE_ACCESS_OUTBOUND; + } + if self.first_pipe_instance { + mode |= windows_sys::FILE_FLAG_FIRST_PIPE_INSTANCE; + } + if self.write_dac { + mode |= windows_sys::WRITE_DAC; + } + if self.write_owner { + mode |= windows_sys::WRITE_OWNER; + } + if self.access_system_security { + mode |= windows_sys::ACCESS_SYSTEM_SECURITY; + } + mode + }; + + let h = windows_sys::CreateNamedPipeW( addr.as_ptr(), - self.open_mode, - self.pipe_mode, + open_mode, + pipe_mode, self.max_instances, self.out_buffer_size, self.in_buffer_size, @@ -1954,11 +2357,11 @@ impl ServerOptions { attrs as *mut _, ); - if h == handleapi::INVALID_HANDLE_VALUE { + if h == windows_sys::INVALID_HANDLE_VALUE { return Err(io::Error::last_os_error()); } - NamedPipeServer::from_raw_handle(h) + NamedPipeServer::from_raw_handle(h as _) } } @@ -1968,8 +2371,10 @@ impl ServerOptions { /// See [`ClientOptions::open`]. #[derive(Debug, Clone)] pub struct ClientOptions { - desired_access: DWORD, - security_qos_flags: DWORD, + generic_read: bool, + generic_write: bool, + security_qos_flags: u32, + pipe_mode: PipeMode, } impl ClientOptions { @@ -1988,8 +2393,11 @@ impl ClientOptions { /// ``` pub fn new() -> Self { Self { - desired_access: winnt::GENERIC_READ | winnt::GENERIC_WRITE, - security_qos_flags: winbase::SECURITY_IDENTIFICATION | winbase::SECURITY_SQOS_PRESENT, + generic_read: true, + generic_write: true, + security_qos_flags: windows_sys::SECURITY_IDENTIFICATION + | windows_sys::SECURITY_SQOS_PRESENT, + pipe_mode: PipeMode::Byte, } } @@ -2000,7 +2408,7 @@ impl ClientOptions { /// [`GENERIC_READ`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew pub fn read(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.desired_access, allowed, winnt::GENERIC_READ); + self.generic_read = allowed; self } @@ -2011,7 +2419,7 @@ impl ClientOptions { /// [`GENERIC_WRITE`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew pub fn write(&mut self, allowed: bool) -> &mut Self { - bool_flag!(self.desired_access, allowed, winnt::GENERIC_WRITE); + self.generic_write = allowed; self } @@ -2034,11 +2442,20 @@ impl ClientOptions { /// automatically when using this method. /// /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea - /// [`SECURITY_IDENTIFICATION`]: crate::winapi::um::winbase::SECURITY_IDENTIFICATION + /// [`SECURITY_IDENTIFICATION`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Storage/FileSystem/constant.SECURITY_IDENTIFICATION.html /// [Impersonation Levels]: https://docs.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level pub fn security_qos_flags(&mut self, flags: u32) -> &mut Self { // See: https://github.com/rust-lang/rust/pull/58216 - self.security_qos_flags = flags | winbase::SECURITY_SQOS_PRESENT; + self.security_qos_flags = flags | windows_sys::SECURITY_SQOS_PRESENT; + self + } + + /// The pipe mode. + /// + /// The default pipe mode is [`PipeMode::Byte`]. See [`PipeMode`] for + /// documentation of what each mode means. + pub fn pipe_mode(&mut self, pipe_mode: PipeMode) -> &mut Self { + self.pipe_mode = pipe_mode; self } @@ -2063,8 +2480,7 @@ impl ClientOptions { /// but the server is not currently waiting for a connection. Please see the /// examples for how to check for this error. /// - /// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY - /// [`winapi`]: crate::winapi + /// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html /// [enabled I/O]: crate::runtime::Builder::enable_io /// [Tokio Runtime]: crate::runtime::Runtime /// @@ -2075,7 +2491,7 @@ impl ClientOptions { /// use std::time::Duration; /// use tokio::net::windows::named_pipe::ClientOptions; /// use tokio::time; - /// use winapi::shared::winerror; + /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY; /// /// const PIPE_NAME: &str = r"\\.\pipe\mynamedpipe"; /// @@ -2083,7 +2499,7 @@ impl ClientOptions { /// let client = loop { /// match ClientOptions::new().open(PIPE_NAME) { /// Ok(client) => break client, - /// Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), + /// Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), /// Err(e) => return Err(e), /// } /// @@ -2113,7 +2529,7 @@ impl ClientOptions { /// /// [`open`]: ClientOptions::open /// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew - /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES + /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html pub unsafe fn open_with_security_attributes_raw( &self, addr: impl AsRef, @@ -2121,29 +2537,50 @@ impl ClientOptions { ) -> io::Result { let addr = encode_addr(addr); + let desired_access = { + let mut access = 0; + if self.generic_read { + access |= windows_sys::GENERIC_READ; + } + if self.generic_write { + access |= windows_sys::GENERIC_WRITE; + } + access + }; + // NB: We could use a platform specialized `OpenOptions` here, but since - // we have access to winapi it ultimately doesn't hurt to use + // we have access to windows_sys it ultimately doesn't hurt to use // `CreateFile` explicitly since it allows the use of our already // well-structured wide `addr` to pass into CreateFileW. - let h = fileapi::CreateFileW( + let h = windows_sys::CreateFileW( addr.as_ptr(), - self.desired_access, + desired_access, 0, attrs as *mut _, - fileapi::OPEN_EXISTING, + windows_sys::OPEN_EXISTING, self.get_flags(), - ptr::null_mut(), + 0, ); - if h == handleapi::INVALID_HANDLE_VALUE { + if h == windows_sys::INVALID_HANDLE_VALUE { return Err(io::Error::last_os_error()); } - NamedPipeClient::from_raw_handle(h) + if matches!(self.pipe_mode, PipeMode::Message) { + let mode = windows_sys::PIPE_READMODE_MESSAGE; + let result = + windows_sys::SetNamedPipeHandleState(h, &mode, ptr::null_mut(), ptr::null_mut()); + + if result == 0 { + return Err(io::Error::last_os_error()); + } + } + + NamedPipeClient::from_raw_handle(h as _) } fn get_flags(&self) -> u32 { - self.security_qos_flags | winbase::FILE_FLAG_OVERLAPPED + self.security_qos_flags | windows_sys::FILE_FLAG_OVERLAPPED } } @@ -2156,16 +2593,19 @@ pub enum PipeMode { /// Data is written to the pipe as a stream of bytes. The pipe does not /// distinguish bytes written during different write operations. /// - /// Corresponds to [`PIPE_TYPE_BYTE`][crate::winapi::um::winbase::PIPE_TYPE_BYTE]. + /// Corresponds to [`PIPE_TYPE_BYTE`]. + /// + /// [`PIPE_TYPE_BYTE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_BYTE.html Byte, /// Data is written to the pipe as a stream of messages. The pipe treats the /// bytes written during each write operation as a message unit. Any reading /// on a named pipe returns [`ERROR_MORE_DATA`] when a message is not read /// completely. /// - /// Corresponds to [`PIPE_TYPE_MESSAGE`][crate::winapi::um::winbase::PIPE_TYPE_MESSAGE]. + /// Corresponds to [`PIPE_TYPE_MESSAGE`]. /// - /// [`ERROR_MORE_DATA`]: crate::winapi::shared::winerror::ERROR_MORE_DATA + /// [`ERROR_MORE_DATA`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_MORE_DATA.html + /// [`PIPE_TYPE_MESSAGE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_MESSAGE.html Message, } @@ -2175,11 +2615,15 @@ pub enum PipeMode { pub enum PipeEnd { /// The named pipe refers to the client end of a named pipe instance. /// - /// Corresponds to [`PIPE_CLIENT_END`][crate::winapi::um::winbase::PIPE_CLIENT_END]. + /// Corresponds to [`PIPE_CLIENT_END`]. + /// + /// [`PIPE_CLIENT_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_CLIENT_END.html Client, /// The named pipe refers to the server end of a named pipe instance. /// - /// Corresponds to [`PIPE_SERVER_END`][crate::winapi::um::winbase::PIPE_SERVER_END]. + /// Corresponds to [`PIPE_SERVER_END`]. + /// + /// [`PIPE_SERVER_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_SERVER_END.html Server, } @@ -2217,26 +2661,26 @@ unsafe fn named_pipe_info(handle: RawHandle) -> io::Result { let mut in_buffer_size = 0; let mut max_instances = 0; - let result = namedpipeapi::GetNamedPipeInfo( - handle, + let result = windows_sys::GetNamedPipeInfo( + handle as _, &mut flags, &mut out_buffer_size, &mut in_buffer_size, &mut max_instances, ); - if result == FALSE { + if result == 0 { return Err(io::Error::last_os_error()); } let mut end = PipeEnd::Client; let mut mode = PipeMode::Byte; - if flags & winbase::PIPE_SERVER_END != 0 { + if flags & windows_sys::PIPE_SERVER_END != 0 { end = PipeEnd::Server; } - if flags & winbase::PIPE_TYPE_MESSAGE != 0 { + if flags & windows_sys::PIPE_TYPE_MESSAGE != 0 { mode = PipeMode::Message; } diff --git a/third_party/rust/tokio/src/park/either.rs b/third_party/rust/tokio/src/park/either.rs deleted file mode 100644 index ee02ec158b01..000000000000 --- a/third_party/rust/tokio/src/park/either.rs +++ /dev/null @@ -1,74 +0,0 @@ -#![cfg_attr(not(feature = "full"), allow(dead_code))] - -use crate::park::{Park, Unpark}; - -use std::fmt; -use std::time::Duration; - -pub(crate) enum Either { - A(A), - B(B), -} - -impl Park for Either -where - A: Park, - B: Park, -{ - type Unpark = Either; - type Error = Either; - - fn unpark(&self) -> Self::Unpark { - match self { - Either::A(a) => Either::A(a.unpark()), - Either::B(b) => Either::B(b.unpark()), - } - } - - fn park(&mut self) -> Result<(), Self::Error> { - match self { - Either::A(a) => a.park().map_err(Either::A), - Either::B(b) => b.park().map_err(Either::B), - } - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - match self { - Either::A(a) => a.park_timeout(duration).map_err(Either::A), - Either::B(b) => b.park_timeout(duration).map_err(Either::B), - } - } - - fn shutdown(&mut self) { - match self { - Either::A(a) => a.shutdown(), - Either::B(b) => b.shutdown(), - } - } -} - -impl Unpark for Either -where - A: Unpark, - B: Unpark, -{ - fn unpark(&self) { - match self { - Either::A(a) => a.unpark(), - Either::B(b) => b.unpark(), - } - } -} - -impl fmt::Debug for Either -where - A: fmt::Debug, - B: fmt::Debug, -{ - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Either::A(a) => a.fmt(fmt), - Either::B(b) => b.fmt(fmt), - } - } -} diff --git a/third_party/rust/tokio/src/park/mod.rs b/third_party/rust/tokio/src/park/mod.rs deleted file mode 100644 index 87d04ff78eae..000000000000 --- a/third_party/rust/tokio/src/park/mod.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Abstraction over blocking and unblocking the current thread. -//! -//! Provides an abstraction over blocking the current thread. This is similar to -//! the park / unpark constructs provided by `std` but made generic. This allows -//! embedding custom functionality to perform when the thread is blocked. -//! -//! A blocked `Park` instance is unblocked by calling `unpark` on its -//! `Unpark` handle. -//! -//! The `ParkThread` struct implements `Park` using `thread::park` to put the -//! thread to sleep. The Tokio reactor also implements park, but uses -//! `mio::Poll` to block the thread instead. -//! -//! The `Park` trait is composable. A timer implementation might decorate a -//! `Park` implementation by checking if any timeouts have elapsed after the -//! inner `Park` implementation unblocks. -//! -//! # Model -//! -//! Conceptually, each `Park` instance has an associated token, which is -//! initially not present: -//! -//! * The `park` method blocks the current thread unless or until the token is -//! available, at which point it atomically consumes the token. -//! * The `unpark` method atomically makes the token available if it wasn't -//! already. -//! -//! Some things to note: -//! -//! * If `unpark` is called before `park`, the next call to `park` will -//! **not** block the thread. -//! * **Spurious** wakeups are permitted, i.e., the `park` method may unblock -//! even if `unpark` was not called. -//! * `park_timeout` does the same as `park` but allows specifying a maximum -//! time to block the thread for. - -cfg_rt! { - pub(crate) mod either; -} - -#[cfg(any(feature = "rt", feature = "sync"))] -pub(crate) mod thread; - -use std::fmt::Debug; -use std::sync::Arc; -use std::time::Duration; - -/// Blocks the current thread. -pub(crate) trait Park { - /// Unpark handle type for the `Park` implementation. - type Unpark: Unpark; - - /// Error returned by `park`. - type Error: Debug; - - /// Gets a new `Unpark` handle associated with this `Park` instance. - fn unpark(&self) -> Self::Unpark; - - /// Blocks the current thread unless or until the token is available. - /// - /// A call to `park` does not guarantee that the thread will remain blocked - /// forever, and callers should be prepared for this possibility. This - /// function may wakeup spuriously for any reason. - /// - /// # Panics - /// - /// This function **should** not panic, but ultimately, panics are left as - /// an implementation detail. Refer to the documentation for the specific - /// `Park` implementation. - fn park(&mut self) -> Result<(), Self::Error>; - - /// Parks the current thread for at most `duration`. - /// - /// This function is the same as `park` but allows specifying a maximum time - /// to block the thread for. - /// - /// Same as `park`, there is no guarantee that the thread will remain - /// blocked for any amount of time. Spurious wakeups are permitted for any - /// reason. - /// - /// # Panics - /// - /// This function **should** not panic, but ultimately, panics are left as - /// an implementation detail. Refer to the documentation for the specific - /// `Park` implementation. - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error>; - - /// Releases all resources holded by the parker for proper leak-free shutdown. - fn shutdown(&mut self); -} - -/// Unblock a thread blocked by the associated `Park` instance. -pub(crate) trait Unpark: Sync + Send + 'static { - /// Unblocks a thread that is blocked by the associated `Park` handle. - /// - /// Calling `unpark` atomically makes available the unpark token, if it is - /// not already available. - /// - /// # Panics - /// - /// This function **should** not panic, but ultimately, panics are left as - /// an implementation detail. Refer to the documentation for the specific - /// `Unpark` implementation. - fn unpark(&self); -} - -impl Unpark for Box { - fn unpark(&self) { - (**self).unpark() - } -} - -impl Unpark for Arc { - fn unpark(&self) { - (**self).unpark() - } -} diff --git a/third_party/rust/tokio/src/park/thread.rs b/third_party/rust/tokio/src/park/thread.rs deleted file mode 100644 index 27ce20243965..000000000000 --- a/third_party/rust/tokio/src/park/thread.rs +++ /dev/null @@ -1,346 +0,0 @@ -#![cfg_attr(not(feature = "full"), allow(dead_code))] - -use crate::loom::sync::atomic::AtomicUsize; -use crate::loom::sync::{Arc, Condvar, Mutex}; -use crate::park::{Park, Unpark}; - -use std::sync::atomic::Ordering::SeqCst; -use std::time::Duration; - -#[derive(Debug)] -pub(crate) struct ParkThread { - inner: Arc, -} - -pub(crate) type ParkError = (); - -/// Unblocks a thread that was blocked by `ParkThread`. -#[derive(Clone, Debug)] -pub(crate) struct UnparkThread { - inner: Arc, -} - -#[derive(Debug)] -struct Inner { - state: AtomicUsize, - mutex: Mutex<()>, - condvar: Condvar, -} - -const EMPTY: usize = 0; -const PARKED: usize = 1; -const NOTIFIED: usize = 2; - -thread_local! { - static CURRENT_PARKER: ParkThread = ParkThread::new(); -} - -// ==== impl ParkThread ==== - -impl ParkThread { - pub(crate) fn new() -> Self { - Self { - inner: Arc::new(Inner { - state: AtomicUsize::new(EMPTY), - mutex: Mutex::new(()), - condvar: Condvar::new(), - }), - } - } -} - -impl Park for ParkThread { - type Unpark = UnparkThread; - type Error = ParkError; - - fn unpark(&self) -> Self::Unpark { - let inner = self.inner.clone(); - UnparkThread { inner } - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.inner.park(); - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.inner.park_timeout(duration); - Ok(()) - } - - fn shutdown(&mut self) { - self.inner.shutdown(); - } -} - -// ==== impl Inner ==== - -impl Inner { - /// Parks the current thread for at most `dur`. - fn park(&self) { - // If we were previously notified then we consume this notification and - // return quickly. - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - return; - } - - // Otherwise we need to coordinate going to sleep - let mut m = self.mutex.lock(); - - match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { - Ok(_) => {} - Err(NOTIFIED) => { - // We must read here, even though we know it will be `NOTIFIED`. - // This is because `unpark` may have been called again since we read - // `NOTIFIED` in the `compare_exchange` above. We must perform an - // acquire operation that synchronizes with that `unpark` to observe - // any writes it made before the call to unpark. To do that we must - // read from the write it made to `state`. - let old = self.state.swap(EMPTY, SeqCst); - debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); - - return; - } - Err(actual) => panic!("inconsistent park state; actual = {}", actual), - } - - loop { - m = self.condvar.wait(m).unwrap(); - - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - // got a notification - return; - } - - // spurious wakeup, go back to sleep - } - } - - fn park_timeout(&self, dur: Duration) { - // Like `park` above we have a fast path for an already-notified thread, - // and afterwards we start coordinating for a sleep. Return quickly. - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - return; - } - - if dur == Duration::from_millis(0) { - return; - } - - let m = self.mutex.lock(); - - match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { - Ok(_) => {} - Err(NOTIFIED) => { - // We must read again here, see `park`. - let old = self.state.swap(EMPTY, SeqCst); - debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); - - return; - } - Err(actual) => panic!("inconsistent park_timeout state; actual = {}", actual), - } - - // Wait with a timeout, and if we spuriously wake up or otherwise wake up - // from a notification, we just want to unconditionally set the state back to - // empty, either consuming a notification or un-flagging ourselves as - // parked. - let (_m, _result) = self.condvar.wait_timeout(m, dur).unwrap(); - - match self.state.swap(EMPTY, SeqCst) { - NOTIFIED => {} // got a notification, hurray! - PARKED => {} // no notification, alas - n => panic!("inconsistent park_timeout state: {}", n), - } - } - - fn unpark(&self) { - // To ensure the unparked thread will observe any writes we made before - // this call, we must perform a release operation that `park` can - // synchronize with. To do that we must write `NOTIFIED` even if `state` - // is already `NOTIFIED`. That is why this must be a swap rather than a - // compare-and-swap that returns if it reads `NOTIFIED` on failure. - match self.state.swap(NOTIFIED, SeqCst) { - EMPTY => return, // no one was waiting - NOTIFIED => return, // already unparked - PARKED => {} // gotta go wake someone up - _ => panic!("inconsistent state in unpark"), - } - - // There is a period between when the parked thread sets `state` to - // `PARKED` (or last checked `state` in the case of a spurious wake - // up) and when it actually waits on `cvar`. If we were to notify - // during this period it would be ignored and then when the parked - // thread went to sleep it would never wake up. Fortunately, it has - // `lock` locked at this stage so we can acquire `lock` to wait until - // it is ready to receive the notification. - // - // Releasing `lock` before the call to `notify_one` means that when the - // parked thread wakes it doesn't get woken only to have to wait for us - // to release `lock`. - drop(self.mutex.lock()); - - self.condvar.notify_one() - } - - fn shutdown(&self) { - self.condvar.notify_all(); - } -} - -impl Default for ParkThread { - fn default() -> Self { - Self::new() - } -} - -// ===== impl UnparkThread ===== - -impl Unpark for UnparkThread { - fn unpark(&self) { - self.inner.unpark(); - } -} - -use std::future::Future; -use std::marker::PhantomData; -use std::mem; -use std::rc::Rc; -use std::task::{RawWaker, RawWakerVTable, Waker}; - -/// Blocks the current thread using a condition variable. -#[derive(Debug)] -pub(crate) struct CachedParkThread { - _anchor: PhantomData>, -} - -impl CachedParkThread { - /// Creates a new `ParkThread` handle for the current thread. - /// - /// This type cannot be moved to other threads, so it should be created on - /// the thread that the caller intends to park. - pub(crate) fn new() -> CachedParkThread { - CachedParkThread { - _anchor: PhantomData, - } - } - - pub(crate) fn get_unpark(&self) -> Result { - self.with_current(|park_thread| park_thread.unpark()) - } - - /// Gets a reference to the `ParkThread` handle for this thread. - fn with_current(&self, f: F) -> Result - where - F: FnOnce(&ParkThread) -> R, - { - CURRENT_PARKER.try_with(|inner| f(inner)).map_err(|_| ()) - } - - pub(crate) fn block_on(&mut self, f: F) -> Result { - use std::task::Context; - use std::task::Poll::Ready; - - // `get_unpark()` should not return a Result - let waker = self.get_unpark()?.into_waker(); - let mut cx = Context::from_waker(&waker); - - pin!(f); - - loop { - if let Ready(v) = crate::coop::budget(|| f.as_mut().poll(&mut cx)) { - return Ok(v); - } - - self.park()?; - } - } -} - -impl Park for CachedParkThread { - type Unpark = UnparkThread; - type Error = ParkError; - - fn unpark(&self) -> Self::Unpark { - self.get_unpark().unwrap() - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.with_current(|park_thread| park_thread.inner.park())?; - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.with_current(|park_thread| park_thread.inner.park_timeout(duration))?; - Ok(()) - } - - fn shutdown(&mut self) { - let _ = self.with_current(|park_thread| park_thread.inner.shutdown()); - } -} - -impl UnparkThread { - pub(crate) fn into_waker(self) -> Waker { - unsafe { - let raw = unparker_to_raw_waker(self.inner); - Waker::from_raw(raw) - } - } -} - -impl Inner { - #[allow(clippy::wrong_self_convention)] - fn into_raw(this: Arc) -> *const () { - Arc::into_raw(this) as *const () - } - - unsafe fn from_raw(ptr: *const ()) -> Arc { - Arc::from_raw(ptr as *const Inner) - } -} - -unsafe fn unparker_to_raw_waker(unparker: Arc) -> RawWaker { - RawWaker::new( - Inner::into_raw(unparker), - &RawWakerVTable::new(clone, wake, wake_by_ref, drop_waker), - ) -} - -unsafe fn clone(raw: *const ()) -> RawWaker { - let unparker = Inner::from_raw(raw); - - // Increment the ref count - mem::forget(unparker.clone()); - - unparker_to_raw_waker(unparker) -} - -unsafe fn drop_waker(raw: *const ()) { - let _ = Inner::from_raw(raw); -} - -unsafe fn wake(raw: *const ()) { - let unparker = Inner::from_raw(raw); - unparker.unpark(); -} - -unsafe fn wake_by_ref(raw: *const ()) { - let unparker = Inner::from_raw(raw); - unparker.unpark(); - - // We don't actually own a reference to the unparker - mem::forget(unparker); -} diff --git a/third_party/rust/tokio/src/process/mod.rs b/third_party/rust/tokio/src/process/mod.rs index 4e1a21dd4496..d68a68995ecb 100644 --- a/third_party/rust/tokio/src/process/mod.rs +++ b/third_party/rust/tokio/src/process/mod.rs @@ -111,7 +111,7 @@ //! let mut cmd = Command::new("sort"); //! //! // Specifying that we want pipe both the output and the input. -//! // Similarily to capturing the output, by configuring the pipe +//! // Similarly to capturing the output, by configuring the pipe //! // to stdin it can now be used as an asynchronous writer. //! cmd.stdout(Stdio::piped()); //! cmd.stdin(Stdio::piped()); @@ -156,7 +156,6 @@ //! ```no_run //! use tokio::join; //! use tokio::process::Command; -//! use std::convert::TryInto; //! use std::process::Stdio; //! //! #[tokio::main] @@ -217,7 +216,7 @@ //! from being spawned. //! //! The tokio runtime will, on a best-effort basis, attempt to reap and clean up -//! any process which it has spawned. No additional guarantees are made with regards +//! any process which it has spawned. No additional guarantees are made with regard to //! how quickly or how often this procedure will take place. //! //! It is recommended to avoid dropping a [`Child`] process handle before it has been @@ -245,22 +244,26 @@ mod kill; use crate::io::{AsyncRead, AsyncWrite, ReadBuf}; use crate::process::kill::Kill; -use std::convert::TryInto; use std::ffi::OsStr; use std::future::Future; use std::io; -#[cfg(unix)] -use std::os::unix::process::CommandExt; -#[cfg(windows)] -use std::os::windows::io::{AsRawHandle, RawHandle}; -#[cfg(windows)] -use std::os::windows::process::CommandExt; use std::path::Path; use std::pin::Pin; use std::process::{Command as StdCommand, ExitStatus, Output, Stdio}; use std::task::Context; use std::task::Poll; +#[cfg(unix)] +use std::os::unix::process::CommandExt; +#[cfg(windows)] +use std::os::windows::process::CommandExt; + +cfg_windows! { + use crate::os::windows::io::{AsRawHandle, RawHandle}; + #[cfg(not(tokio_no_as_fd))] + use crate::os::windows::io::{AsHandle, BorrowedHandle}; +} + /// This structure mimics the API of [`std::process::Command`] found in the standard library, but /// replaces functions that create a process with an asynchronous variant. The main provided /// asynchronous functions are [spawn](Command::spawn), [status](Command::status), and @@ -309,7 +312,8 @@ impl Command { /// /// ```no_run /// use tokio::process::Command; - /// let command = Command::new("sh"); + /// let mut command = Command::new("sh"); + /// # let _ = command.output(); // assert borrow checker /// ``` /// /// [rust-lang/rust#37519]: https://github.com/rust-lang/rust/issues/37519 @@ -328,16 +332,20 @@ impl Command { /// Only one argument can be passed per use. So instead of: /// /// ```no_run - /// tokio::process::Command::new("sh") - /// .arg("-C /path/to/repo"); + /// let mut command = tokio::process::Command::new("sh"); + /// command.arg("-C /path/to/repo"); + /// + /// # let _ = command.output(); // assert borrow checker /// ``` /// /// usage would be: /// /// ```no_run - /// tokio::process::Command::new("sh") - /// .arg("-C") - /// .arg("/path/to/repo"); + /// let mut command = tokio::process::Command::new("sh"); + /// command.arg("-C"); + /// command.arg("/path/to/repo"); + /// + /// # let _ = command.output(); // assert borrow checker /// ``` /// /// To pass multiple arguments see [`args`]. @@ -349,11 +357,15 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// - /// let command = Command::new("ls") + /// let output = Command::new("ls") /// .arg("-l") - /// .arg("-a"); + /// .arg("-a") + /// .output().await.unwrap(); + /// # } + /// /// ``` pub fn arg>(&mut self, arg: S) -> &mut Command { self.std.arg(arg); @@ -371,10 +383,13 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// - /// let command = Command::new("ls") - /// .args(&["-l", "-a"]); + /// let output = Command::new("ls") + /// .args(&["-l", "-a"]) + /// .output().await.unwrap(); + /// # } /// ``` pub fn args(&mut self, args: I) -> &mut Command where @@ -385,6 +400,22 @@ impl Command { self } + /// Append literal text to the command line without any quoting or escaping. + /// + /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow + /// `CommandLineToArgvW` escaping rules. + /// + /// **Note**: This is an [unstable API][unstable] but will be stabilised once + /// tokio's MSRV is sufficiently new. See [the documentation on + /// unstable features][unstable] for details about using unstable features. + #[cfg(windows)] + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(all(windows, tokio_unstable))))] + pub fn raw_arg>(&mut self, text_to_append_as_is: S) -> &mut Command { + self.std.raw_arg(text_to_append_as_is); + self + } + /// Inserts or updates an environment variable mapping. /// /// Note that environment variable names are case-insensitive (but case-preserving) on Windows, @@ -395,10 +426,13 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// - /// let command = Command::new("ls") - /// .env("PATH", "/bin"); + /// let output = Command::new("ls") + /// .env("PATH", "/bin") + /// .output().await.unwrap(); + /// # } /// ``` pub fn env(&mut self, key: K, val: V) -> &mut Command where @@ -416,6 +450,7 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// use std::process::{Stdio}; /// use std::env; @@ -426,11 +461,13 @@ impl Command { /// k == "TERM" || k == "TZ" || k == "LANG" || k == "PATH" /// ).collect(); /// - /// let command = Command::new("printenv") + /// let output = Command::new("printenv") /// .stdin(Stdio::null()) /// .stdout(Stdio::inherit()) /// .env_clear() - /// .envs(&filtered_env); + /// .envs(&filtered_env) + /// .output().await.unwrap(); + /// # } /// ``` pub fn envs(&mut self, vars: I) -> &mut Command where @@ -449,10 +486,13 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// - /// let command = Command::new("ls") - /// .env_remove("PATH"); + /// let output = Command::new("ls") + /// .env_remove("PATH") + /// .output().await.unwrap(); + /// # } /// ``` pub fn env_remove>(&mut self, key: K) -> &mut Command { self.std.env_remove(key); @@ -466,10 +506,13 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// - /// let command = Command::new("ls") - /// .env_clear(); + /// let output = Command::new("ls") + /// .env_clear() + /// .output().await.unwrap(); + /// # } /// ``` pub fn env_clear(&mut self) -> &mut Command { self.std.env_clear(); @@ -493,10 +536,13 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// - /// let command = Command::new("ls") - /// .current_dir("/bin"); + /// let output = Command::new("ls") + /// .current_dir("/bin") + /// .output().await.unwrap(); + /// # } /// ``` pub fn current_dir>(&mut self, dir: P) -> &mut Command { self.std.current_dir(dir); @@ -516,11 +562,14 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use std::process::{Stdio}; /// use tokio::process::Command; /// - /// let command = Command::new("ls") - /// .stdin(Stdio::null()); + /// let output = Command::new("ls") + /// .stdin(Stdio::null()) + /// .output().await.unwrap(); + /// # } /// ``` pub fn stdin>(&mut self, cfg: T) -> &mut Command { self.std.stdin(cfg); @@ -540,11 +589,14 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// use std::process::Stdio; /// - /// let command = Command::new("ls") - /// .stdout(Stdio::null()); + /// let output = Command::new("ls") + /// .stdout(Stdio::null()) + /// .output().await.unwrap(); + /// # } /// ``` pub fn stdout>(&mut self, cfg: T) -> &mut Command { self.std.stdout(cfg); @@ -564,11 +616,14 @@ impl Command { /// Basic usage: /// /// ```no_run + /// # async fn test() { // allow using await /// use tokio::process::Command; /// use std::process::{Stdio}; /// - /// let command = Command::new("ls") - /// .stderr(Stdio::null()); + /// let output = Command::new("ls") + /// .stderr(Stdio::null()) + /// .output().await.unwrap(); + /// # } /// ``` pub fn stderr>(&mut self, cfg: T) -> &mut Command { self.std.stderr(cfg); @@ -595,7 +650,7 @@ impl Command { /// operation, the resulting zombie process cannot be `.await`ed inside of the /// destructor to avoid blocking other tasks. The tokio runtime will, on a /// best-effort basis, attempt to reap and clean up such processes in the - /// background, but makes no additional guarantees are made with regards + /// background, but no additional guarantees are made with regard to /// how quickly or how often this procedure will take place. /// /// If stronger guarantees are required, it is recommended to avoid dropping @@ -606,16 +661,16 @@ impl Command { self } - /// Sets the [process creation flags][1] to be passed to `CreateProcess`. - /// - /// These will always be ORed with `CREATE_UNICODE_ENVIRONMENT`. - /// - /// [1]: https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863(v=vs.85).aspx - #[cfg(windows)] - #[cfg_attr(docsrs, doc(cfg(windows)))] - pub fn creation_flags(&mut self, flags: u32) -> &mut Command { - self.std.creation_flags(flags); - self + cfg_windows! { + /// Sets the [process creation flags][1] to be passed to `CreateProcess`. + /// + /// These will always be ORed with `CREATE_UNICODE_ENVIRONMENT`. + /// + /// [1]: https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863(v=vs.85).aspx + pub fn creation_flags(&mut self, flags: u32) -> &mut Command { + self.std.creation_flags(flags); + self + } } /// Sets the child process's user ID. This translates to a @@ -690,6 +745,39 @@ impl Command { self } + /// Sets the process group ID (PGID) of the child process. Equivalent to a + /// setpgid call in the child process, but may be more efficient. + /// + /// Process groups determine which processes receive signals. + /// + /// **Note**: This is an [unstable API][unstable] but will be stabilised once + /// tokio's MSRV is sufficiently new. See [the documentation on + /// unstable features][unstable] for details about using unstable features. + /// + /// If you want similar behaviour without using this unstable feature you can + /// create a [`std::process::Command`] and convert that into a + /// [`tokio::process::Command`] using the `From` trait. + /// + /// [unstable]: crate#unstable-features + /// [`tokio::process::Command`]: crate::process::Command + /// + /// ```no_run + /// # async fn test() { // allow using await + /// use tokio::process::Command; + /// + /// let output = Command::new("ls") + /// .process_group(0) + /// .output().await.unwrap(); + /// # } + /// ``` + #[cfg(unix)] + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(all(unix, tokio_unstable))))] + pub fn process_group(&mut self, pgroup: i32) -> &mut Command { + self.std.process_group(pgroup); + self + } + /// Executes the command as a child process, returning a handle to it. /// /// By default, stdin, stdout and stderr are inherited from the parent. @@ -742,7 +830,7 @@ impl Command { /// from being spawned. /// /// The tokio runtime will, on a best-effort basis, attempt to reap and clean up - /// any process which it has spawned. No additional guarantees are made with regards + /// any process which it has spawned. No additional guarantees are made with regard to /// how quickly or how often this procedure will take place. /// /// It is recommended to avoid dropping a [`Child`] process handle before it has been @@ -923,8 +1011,9 @@ where type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + ready!(crate::trace::trace_leaf(cx)); // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let ret = Pin::new(&mut self.inner).poll(cx); @@ -1013,13 +1102,14 @@ impl Child { } } - /// Extracts the raw handle of the process associated with this child while - /// it is still running. Returns `None` if the child has exited. - #[cfg(windows)] - pub fn raw_handle(&self) -> Option { - match &self.child { - FusedChild::Child(c) => Some(c.inner.as_raw_handle()), - FusedChild::Done(_) => None, + cfg_windows! { + /// Extracts the raw handle of the process associated with this child while + /// it is still running. Returns `None` if the child has exited. + pub fn raw_handle(&self) -> Option { + match &self.child { + FusedChild::Child(c) => Some(c.inner.as_raw_handle()), + FusedChild::Done(_) => None, + } } } @@ -1254,7 +1344,7 @@ impl ChildStdin { } impl ChildStdout { - /// Creates an asynchronous `ChildStderr` from a synchronous one. + /// Creates an asynchronous `ChildStdout` from a synchronous one. /// /// # Errors /// @@ -1285,41 +1375,51 @@ impl ChildStderr { impl AsyncWrite for ChildStdin { fn poll_write( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { - self.inner.poll_write(cx, buf) + Pin::new(&mut self.inner).poll_write(cx, buf) } - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) } - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.inner).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() } } impl AsyncRead for ChildStdout { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll> { - // Safety: pipes support reading into uninitialized memory - unsafe { self.inner.poll_read(cx, buf) } + Pin::new(&mut self.inner).poll_read(cx, buf) } } impl AsyncRead for ChildStderr { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll> { - // Safety: pipes support reading into uninitialized memory - unsafe { self.inner.poll_read(cx, buf) } + Pin::new(&mut self.inner).poll_read(cx, buf) } } @@ -1349,6 +1449,8 @@ impl TryInto for ChildStderr { #[cfg(unix)] mod sys { + #[cfg(not(tokio_no_as_fd))] + use std::os::unix::io::{AsFd, BorrowedFd}; use std::os::unix::io::{AsRawFd, RawFd}; use super::{ChildStderr, ChildStdin, ChildStdout}; @@ -1359,42 +1461,79 @@ mod sys { } } + #[cfg(not(tokio_no_as_fd))] + impl AsFd for ChildStdin { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } + impl AsRawFd for ChildStdout { fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() } } + #[cfg(not(tokio_no_as_fd))] + impl AsFd for ChildStdout { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } + impl AsRawFd for ChildStderr { fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsFd for ChildStderr { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } + } } -#[cfg(windows)] -mod sys { - use std::os::windows::io::{AsRawHandle, RawHandle}; - - use super::{ChildStderr, ChildStdin, ChildStdout}; - +cfg_windows! { impl AsRawHandle for ChildStdin { fn as_raw_handle(&self) -> RawHandle { self.inner.as_raw_handle() } } + #[cfg(not(tokio_no_as_fd))] + impl AsHandle for ChildStdin { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } + } + impl AsRawHandle for ChildStdout { fn as_raw_handle(&self) -> RawHandle { self.inner.as_raw_handle() } } + #[cfg(not(tokio_no_as_fd))] + impl AsHandle for ChildStdout { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } + } + impl AsRawHandle for ChildStderr { fn as_raw_handle(&self) -> RawHandle { self.inner.as_raw_handle() } } + + #[cfg(not(tokio_no_as_fd))] + impl AsHandle for ChildStderr { + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } + } } #[cfg(all(test, not(loom)))] diff --git a/third_party/rust/tokio/src/process/unix/mod.rs b/third_party/rust/tokio/src/process/unix/mod.rs index 576fe6cb47f2..41857f49c8f9 100644 --- a/third_party/rust/tokio/src/process/unix/mod.rs +++ b/third_party/rust/tokio/src/process/unix/mod.rs @@ -21,27 +21,26 @@ //! processes in general aren't scalable (e.g. millions) so it shouldn't be that //! bad in theory... -pub(crate) mod driver; - pub(crate) mod orphan; use orphan::{OrphanQueue, OrphanQueueImpl, Wait}; mod reap; use reap::Reaper; -use crate::io::PollEvented; +use crate::io::{AsyncRead, AsyncWrite, PollEvented, ReadBuf}; use crate::process::kill::Kill; use crate::process::SpawnedChild; -use crate::signal::unix::driver::Handle as SignalHandle; +use crate::runtime::signal::Handle as SignalHandle; use crate::signal::unix::{signal, Signal, SignalKind}; use mio::event::Source; use mio::unix::SourceFd; -use once_cell::sync::Lazy; use std::fmt; use std::fs::File; use std::future::Future; use std::io; +#[cfg(not(tokio_no_as_fd))] +use std::os::unix::io::{AsFd, BorrowedFd}; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::pin::Pin; use std::process::{Child as StdChild, ExitStatus, Stdio}; @@ -64,25 +63,41 @@ impl Kill for StdChild { } } -static ORPHAN_QUEUE: Lazy> = Lazy::new(OrphanQueueImpl::new); +cfg_not_has_const_mutex_new! { + fn get_orphan_queue() -> &'static OrphanQueueImpl { + use crate::util::once_cell::OnceCell; + + static ORPHAN_QUEUE: OnceCell> = OnceCell::new(); + + ORPHAN_QUEUE.get(OrphanQueueImpl::new) + } +} + +cfg_has_const_mutex_new! { + fn get_orphan_queue() -> &'static OrphanQueueImpl { + static ORPHAN_QUEUE: OrphanQueueImpl = OrphanQueueImpl::new(); + + &ORPHAN_QUEUE + } +} pub(crate) struct GlobalOrphanQueue; impl fmt::Debug for GlobalOrphanQueue { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - ORPHAN_QUEUE.fmt(fmt) + get_orphan_queue().fmt(fmt) } } impl GlobalOrphanQueue { - fn reap_orphans(handle: &SignalHandle) { - ORPHAN_QUEUE.reap_orphans(handle) + pub(crate) fn reap_orphans(handle: &SignalHandle) { + get_orphan_queue().reap_orphans(handle) } } impl OrphanQueue for GlobalOrphanQueue { fn push_orphan(&self, orphan: StdChild) { - ORPHAN_QUEUE.push_orphan(orphan) + get_orphan_queue().push_orphan(orphan) } } @@ -143,7 +158,7 @@ impl Future for Child { #[derive(Debug)] pub(crate) struct Pipe { - // Actually a pipe and not a File. However, we are reusing `File` to get + // Actually a pipe is not a File. However, we are reusing `File` to get // close on drop. This is a similar trick as `mio`. fd: File, } @@ -169,6 +184,10 @@ impl<'a> io::Write for &'a Pipe { fn flush(&mut self) -> io::Result<()> { (&self.fd).flush() } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + (&self.fd).write_vectored(bufs) + } } impl AsRawFd for Pipe { @@ -177,8 +196,15 @@ impl AsRawFd for Pipe { } } -pub(crate) fn convert_to_stdio(io: PollEvented) -> io::Result { - let mut fd = io.into_inner()?.fd; +#[cfg(not(tokio_no_as_fd))] +impl AsFd for Pipe { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} + +pub(crate) fn convert_to_stdio(io: ChildStdio) -> io::Result { + let mut fd = io.inner.into_inner()?.fd; // Ensure that the fd to be inherited is set to *blocking* mode, as this // is the default that virtually all programs expect to have. Those @@ -213,7 +239,69 @@ impl Source for Pipe { } } -pub(crate) type ChildStdio = PollEvented; +pub(crate) struct ChildStdio { + inner: PollEvented, +} + +impl fmt::Debug for ChildStdio { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(fmt) + } +} + +impl AsRawFd for ChildStdio { + fn as_raw_fd(&self) -> RawFd { + self.inner.as_raw_fd() + } +} + +#[cfg(not(tokio_no_as_fd))] +impl AsFd for ChildStdio { + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} + +impl AsyncWrite for ChildStdio { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.inner.poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll> { + self.inner.poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + true + } +} + +impl AsyncRead for ChildStdio { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + // Safety: pipes support reading into uninitialized memory + unsafe { self.inner.poll_read(cx, buf) } + } +} fn set_nonblocking(fd: &mut T, nonblocking: bool) -> io::Result<()> { unsafe { @@ -238,7 +326,7 @@ fn set_nonblocking(fd: &mut T, nonblocking: bool) -> io::Result<()> Ok(()) } -pub(super) fn stdio(io: T) -> io::Result> +pub(super) fn stdio(io: T) -> io::Result where T: IntoRawFd, { @@ -246,5 +334,5 @@ where let mut pipe = Pipe::from(io); set_nonblocking(&mut pipe, true)?; - PollEvented::new(pipe) + PollEvented::new(pipe).map(|inner| ChildStdio { inner }) } diff --git a/third_party/rust/tokio/src/process/unix/orphan.rs b/third_party/rust/tokio/src/process/unix/orphan.rs index 0e52530c37b2..340719603bee 100644 --- a/third_party/rust/tokio/src/process/unix/orphan.rs +++ b/third_party/rust/tokio/src/process/unix/orphan.rs @@ -1,5 +1,5 @@ use crate::loom::sync::{Mutex, MutexGuard}; -use crate::signal::unix::driver::Handle as SignalHandle; +use crate::runtime::signal::Handle as SignalHandle; use crate::signal::unix::{signal_with_handle, SignalKind}; use crate::sync::watch; use std::io; @@ -43,10 +43,21 @@ pub(crate) struct OrphanQueueImpl { } impl OrphanQueueImpl { - pub(crate) fn new() -> Self { - Self { - sigchild: Mutex::new(None), - queue: Mutex::new(Vec::new()), + cfg_not_has_const_mutex_new! { + pub(crate) fn new() -> Self { + Self { + sigchild: Mutex::new(None), + queue: Mutex::new(Vec::new()), + } + } + } + + cfg_has_const_mutex_new! { + pub(crate) const fn new() -> Self { + Self { + sigchild: Mutex::const_new(None), + queue: Mutex::const_new(Vec::new()), + } } } @@ -120,8 +131,8 @@ where #[cfg(all(test, not(loom)))] pub(crate) mod test { use super::*; - use crate::io::driver::Driver as IoDriver; - use crate::signal::unix::driver::{Driver as SignalDriver, Handle as SignalHandle}; + use crate::runtime::io::Driver as IoDriver; + use crate::runtime::signal::{Driver as SignalDriver, Handle as SignalHandle}; use crate::sync::watch; use std::cell::{Cell, RefCell}; use std::io; @@ -283,7 +294,8 @@ pub(crate) mod test { #[cfg_attr(miri, ignore)] // Miri does not support epoll. #[test] fn does_not_register_signal_if_queue_empty() { - let signal_driver = IoDriver::new().and_then(SignalDriver::new).unwrap(); + let (io_driver, io_handle) = IoDriver::new(1024).unwrap(); + let signal_driver = SignalDriver::new(io_driver, &io_handle).unwrap(); let handle = signal_driver.handle(); let orphanage = OrphanQueueImpl::new(); diff --git a/third_party/rust/tokio/src/process/windows.rs b/third_party/rust/tokio/src/process/windows.rs index 136d5b0cab86..eb412cdc55fd 100644 --- a/third_party/rust/tokio/src/process/windows.rs +++ b/third_party/rust/tokio/src/process/windows.rs @@ -15,29 +15,30 @@ //! `RegisterWaitForSingleObject` and then wait on the other end of the oneshot //! from then on out. -use crate::io::PollEvented; +use crate::io::{blocking::Blocking, AsyncRead, AsyncWrite, ReadBuf}; use crate::process::kill::Kill; use crate::process::SpawnedChild; use crate::sync::oneshot; -use mio::windows::NamedPipe; use std::fmt; +use std::fs::File as StdFile; use std::future::Future; use std::io; -use std::os::windows::prelude::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle}; +use std::os::windows::prelude::{AsRawHandle, IntoRawHandle, RawHandle}; use std::pin::Pin; use std::process::Stdio; use std::process::{Child as StdChild, Command as StdCommand, ExitStatus}; -use std::ptr; -use std::task::Context; -use std::task::Poll; -use winapi::shared::minwindef::{DWORD, FALSE}; -use winapi::um::handleapi::{DuplicateHandle, INVALID_HANDLE_VALUE}; -use winapi::um::processthreadsapi::GetCurrentProcess; -use winapi::um::threadpoollegacyapiset::UnregisterWaitEx; -use winapi::um::winbase::{RegisterWaitForSingleObject, INFINITE}; -use winapi::um::winnt::{ - BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, PVOID, WT_EXECUTEINWAITTHREAD, WT_EXECUTEONLYONCE, +use std::sync::Arc; +use std::task::{Context, Poll}; + +use windows_sys::{ + Win32::Foundation::{ + DuplicateHandle, BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, INVALID_HANDLE_VALUE, + }, + Win32::System::Threading::{ + GetCurrentProcess, RegisterWaitForSingleObject, UnregisterWaitEx, INFINITE, + WT_EXECUTEINWAITTHREAD, WT_EXECUTEONLYONCE, + }, }; #[must_use = "futures do nothing unless polled"] @@ -119,11 +120,11 @@ impl Future for Child { } let (tx, rx) = oneshot::channel(); let ptr = Box::into_raw(Box::new(Some(tx))); - let mut wait_object = ptr::null_mut(); + let mut wait_object = 0; let rc = unsafe { RegisterWaitForSingleObject( &mut wait_object, - inner.child.as_raw_handle(), + inner.child.as_raw_handle() as _, Some(callback), ptr as *mut _, INFINITE, @@ -162,37 +163,106 @@ impl Drop for Waiting { } } -unsafe extern "system" fn callback(ptr: PVOID, _timer_fired: BOOLEAN) { +unsafe extern "system" fn callback(ptr: *mut std::ffi::c_void, _timer_fired: BOOLEAN) { let complete = &mut *(ptr as *mut Option>); let _ = complete.take().unwrap().send(()); } -pub(crate) type ChildStdio = PollEvented; +#[derive(Debug)] +struct ArcFile(Arc); -pub(super) fn stdio(io: T) -> io::Result> +impl io::Read for ArcFile { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + (&*self.0).read(bytes) + } +} + +impl io::Write for ArcFile { + fn write(&mut self, bytes: &[u8]) -> io::Result { + (&*self.0).write(bytes) + } + + fn flush(&mut self) -> io::Result<()> { + (&*self.0).flush() + } +} + +#[derive(Debug)] +pub(crate) struct ChildStdio { + // Used for accessing the raw handle, even if the io version is busy + raw: Arc, + // For doing I/O operations asynchronously + io: Blocking, +} + +impl AsRawHandle for ChildStdio { + fn as_raw_handle(&self) -> RawHandle { + self.raw.as_raw_handle() + } +} + +impl AsyncRead for ChildStdio { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.io).poll_read(cx, buf) + } +} + +impl AsyncWrite for ChildStdio { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.io).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.io).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.io).poll_shutdown(cx) + } +} + +pub(super) fn stdio(io: T) -> io::Result where T: IntoRawHandle, { - let pipe = unsafe { NamedPipe::from_raw_handle(io.into_raw_handle()) }; - PollEvented::new(pipe) + use std::os::windows::prelude::FromRawHandle; + + let raw = Arc::new(unsafe { StdFile::from_raw_handle(io.into_raw_handle()) }); + let io = Blocking::new(ArcFile(raw.clone())); + Ok(ChildStdio { raw, io }) } -pub(crate) fn convert_to_stdio(io: PollEvented) -> io::Result { - let named_pipe = io.into_inner()?; +pub(crate) fn convert_to_stdio(child_stdio: ChildStdio) -> io::Result { + let ChildStdio { raw, io } = child_stdio; + drop(io); // Try to drop the Arc count here + + Arc::try_unwrap(raw) + .or_else(|raw| duplicate_handle(&*raw)) + .map(Stdio::from) +} + +fn duplicate_handle(io: &T) -> io::Result { + use std::os::windows::prelude::FromRawHandle; - // Mio does not implement `IntoRawHandle` for `NamedPipe`, so we'll manually - // duplicate the handle here... unsafe { let mut dup_handle = INVALID_HANDLE_VALUE; let cur_proc = GetCurrentProcess(); let status = DuplicateHandle( cur_proc, - named_pipe.as_raw_handle(), + io.as_raw_handle() as _, cur_proc, &mut dup_handle, - 0 as DWORD, - FALSE, + 0, + 0, DUPLICATE_SAME_ACCESS, ); @@ -200,6 +270,6 @@ pub(crate) fn convert_to_stdio(io: PollEvented) -> io::Result return Err(io::Error::last_os_error()); } - Ok(Stdio::from_raw_handle(dup_handle)) + Ok(StdFile::from_raw_handle(dup_handle as _)) } } diff --git a/third_party/rust/tokio/src/runtime/basic_scheduler.rs b/third_party/rust/tokio/src/runtime/basic_scheduler.rs deleted file mode 100644 index 401f55b3f2fa..000000000000 --- a/third_party/rust/tokio/src/runtime/basic_scheduler.rs +++ /dev/null @@ -1,574 +0,0 @@ -use crate::future::poll_fn; -use crate::loom::sync::atomic::AtomicBool; -use crate::loom::sync::{Arc, Mutex}; -use crate::park::{Park, Unpark}; -use crate::runtime::context::EnterGuard; -use crate::runtime::driver::Driver; -use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task}; -use crate::runtime::Callback; -use crate::runtime::{MetricsBatch, SchedulerMetrics, WorkerMetrics}; -use crate::sync::notify::Notify; -use crate::util::atomic_cell::AtomicCell; -use crate::util::{waker_ref, Wake, WakerRef}; - -use std::cell::RefCell; -use std::collections::VecDeque; -use std::fmt; -use std::future::Future; -use std::sync::atomic::Ordering::{AcqRel, Release}; -use std::task::Poll::{Pending, Ready}; -use std::time::Duration; - -/// Executes tasks on the current thread -pub(crate) struct BasicScheduler { - /// Core scheduler data is acquired by a thread entering `block_on`. - core: AtomicCell, - - /// Notifier for waking up other threads to steal the - /// driver. - notify: Notify, - - /// Sendable task spawner - spawner: Spawner, - - /// This is usually None, but right before dropping the BasicScheduler, it - /// is changed to `Some` with the context being the runtime's own context. - /// This ensures that any tasks dropped in the `BasicScheduler`s destructor - /// run in that runtime's context. - context_guard: Option, -} - -/// Data required for executing the scheduler. The struct is passed around to -/// a function that will perform the scheduling work and acts as a capability token. -struct Core { - /// Scheduler run queue - tasks: VecDeque>>, - - /// Sendable task spawner - spawner: Spawner, - - /// Current tick - tick: u8, - - /// Runtime driver - /// - /// The driver is removed before starting to park the thread - driver: Option, - - /// Metrics batch - metrics: MetricsBatch, -} - -#[derive(Clone)] -pub(crate) struct Spawner { - shared: Arc, -} - -/// Scheduler state shared between threads. -struct Shared { - /// Remote run queue. None if the `Runtime` has been dropped. - queue: Mutex>>>>, - - /// Collection of all active tasks spawned onto this executor. - owned: OwnedTasks>, - - /// Unpark the blocked thread. - unpark: ::Unpark, - - /// Indicates whether the blocked on thread was woken. - woken: AtomicBool, - - /// Callback for a worker parking itself - before_park: Option, - - /// Callback for a worker unparking itself - after_unpark: Option, - - /// Keeps track of various runtime metrics. - scheduler_metrics: SchedulerMetrics, - - /// This scheduler only has one worker. - worker_metrics: WorkerMetrics, -} - -/// Thread-local context. -struct Context { - /// Handle to the spawner - spawner: Spawner, - - /// Scheduler core, enabling the holder of `Context` to execute the - /// scheduler. - core: RefCell>>, -} - -/// Initial queue capacity. -const INITIAL_CAPACITY: usize = 64; - -/// Max number of tasks to poll per tick. -#[cfg(loom)] -const MAX_TASKS_PER_TICK: usize = 4; -#[cfg(not(loom))] -const MAX_TASKS_PER_TICK: usize = 61; - -/// How often to check the remote queue first. -const REMOTE_FIRST_INTERVAL: u8 = 31; - -// Tracks the current BasicScheduler. -scoped_thread_local!(static CURRENT: Context); - -impl BasicScheduler { - pub(crate) fn new( - driver: Driver, - before_park: Option, - after_unpark: Option, - ) -> BasicScheduler { - let unpark = driver.unpark(); - - let spawner = Spawner { - shared: Arc::new(Shared { - queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))), - owned: OwnedTasks::new(), - unpark, - woken: AtomicBool::new(false), - before_park, - after_unpark, - scheduler_metrics: SchedulerMetrics::new(), - worker_metrics: WorkerMetrics::new(), - }), - }; - - let core = AtomicCell::new(Some(Box::new(Core { - tasks: VecDeque::with_capacity(INITIAL_CAPACITY), - spawner: spawner.clone(), - tick: 0, - driver: Some(driver), - metrics: MetricsBatch::new(), - }))); - - BasicScheduler { - core, - notify: Notify::new(), - spawner, - context_guard: None, - } - } - - pub(crate) fn spawner(&self) -> &Spawner { - &self.spawner - } - - pub(crate) fn block_on(&self, future: F) -> F::Output { - pin!(future); - - // Attempt to steal the scheduler core and block_on the future if we can - // there, otherwise, lets select on a notification that the core is - // available or the future is complete. - loop { - if let Some(core) = self.take_core() { - return core.block_on(future); - } else { - let mut enter = crate::runtime::enter(false); - - let notified = self.notify.notified(); - pin!(notified); - - if let Some(out) = enter - .block_on(poll_fn(|cx| { - if notified.as_mut().poll(cx).is_ready() { - return Ready(None); - } - - if let Ready(out) = future.as_mut().poll(cx) { - return Ready(Some(out)); - } - - Pending - })) - .expect("Failed to `Enter::block_on`") - { - return out; - } - } - } - } - - fn take_core(&self) -> Option> { - let core = self.core.take()?; - - Some(CoreGuard { - context: Context { - spawner: self.spawner.clone(), - core: RefCell::new(Some(core)), - }, - basic_scheduler: self, - }) - } - - pub(super) fn set_context_guard(&mut self, guard: EnterGuard) { - self.context_guard = Some(guard); - } -} - -impl Drop for BasicScheduler { - fn drop(&mut self) { - // Avoid a double panic if we are currently panicking and - // the lock may be poisoned. - - let core = match self.take_core() { - Some(core) => core, - None if std::thread::panicking() => return, - None => panic!("Oh no! We never placed the Core back, this is a bug!"), - }; - - core.enter(|mut core, context| { - // Drain the OwnedTasks collection. This call also closes the - // collection, ensuring that no tasks are ever pushed after this - // call returns. - context.spawner.shared.owned.close_and_shutdown_all(); - - // Drain local queue - // We already shut down every task, so we just need to drop the task. - while let Some(task) = core.pop_task() { - drop(task); - } - - // Drain remote queue and set it to None - let remote_queue = core.spawner.shared.queue.lock().take(); - - // Using `Option::take` to replace the shared queue with `None`. - // We already shut down every task, so we just need to drop the task. - if let Some(remote_queue) = remote_queue { - for task in remote_queue { - drop(task); - } - } - - assert!(context.spawner.shared.owned.is_empty()); - - // Submit metrics - core.metrics.submit(&core.spawner.shared.worker_metrics); - - (core, ()) - }); - } -} - -impl fmt::Debug for BasicScheduler { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("BasicScheduler").finish() - } -} - -// ===== impl Core ===== - -impl Core { - fn pop_task(&mut self) -> Option>> { - let ret = self.tasks.pop_front(); - self.spawner - .shared - .worker_metrics - .set_queue_depth(self.tasks.len()); - ret - } - - fn push_task(&mut self, task: task::Notified>) { - self.tasks.push_back(task); - self.metrics.inc_local_schedule_count(); - self.spawner - .shared - .worker_metrics - .set_queue_depth(self.tasks.len()); - } -} - -// ===== impl Context ===== - -impl Context { - /// Execute the closure with the given scheduler core stored in the - /// thread-local context. - fn run_task(&self, mut core: Box, f: impl FnOnce() -> R) -> (Box, R) { - core.metrics.incr_poll_count(); - self.enter(core, || crate::coop::budget(f)) - } - - /// Blocks the current thread until an event is received by the driver, - /// including I/O events, timer events, ... - fn park(&self, mut core: Box) -> Box { - let mut driver = core.driver.take().expect("driver missing"); - - if let Some(f) = &self.spawner.shared.before_park { - // Incorrect lint, the closures are actually different types so `f` - // cannot be passed as an argument to `enter`. - #[allow(clippy::redundant_closure)] - let (c, _) = self.enter(core, || f()); - core = c; - } - - // This check will fail if `before_park` spawns a task for us to run - // instead of parking the thread - if core.tasks.is_empty() { - // Park until the thread is signaled - core.metrics.about_to_park(); - core.metrics.submit(&core.spawner.shared.worker_metrics); - - let (c, _) = self.enter(core, || { - driver.park().expect("failed to park"); - }); - - core = c; - core.metrics.returned_from_park(); - } - - if let Some(f) = &self.spawner.shared.after_unpark { - // Incorrect lint, the closures are actually different types so `f` - // cannot be passed as an argument to `enter`. - #[allow(clippy::redundant_closure)] - let (c, _) = self.enter(core, || f()); - core = c; - } - - core.driver = Some(driver); - core - } - - /// Checks the driver for new events without blocking the thread. - fn park_yield(&self, mut core: Box) -> Box { - let mut driver = core.driver.take().expect("driver missing"); - - core.metrics.submit(&core.spawner.shared.worker_metrics); - let (mut core, _) = self.enter(core, || { - driver - .park_timeout(Duration::from_millis(0)) - .expect("failed to park"); - }); - - core.driver = Some(driver); - core - } - - fn enter(&self, core: Box, f: impl FnOnce() -> R) -> (Box, R) { - // Store the scheduler core in the thread-local context - // - // A drop-guard is employed at a higher level. - *self.core.borrow_mut() = Some(core); - - // Execute the closure while tracking the execution budget - let ret = f(); - - // Take the scheduler core back - let core = self.core.borrow_mut().take().expect("core missing"); - (core, ret) - } -} - -// ===== impl Spawner ===== - -impl Spawner { - /// Spawns a future onto the basic scheduler - pub(crate) fn spawn(&self, future: F) -> JoinHandle - where - F: crate::future::Future + Send + 'static, - F::Output: Send + 'static, - { - let (handle, notified) = self.shared.owned.bind(future, self.shared.clone()); - - if let Some(notified) = notified { - self.shared.schedule(notified); - } - - handle - } - - fn pop(&self) -> Option>> { - match self.shared.queue.lock().as_mut() { - Some(queue) => queue.pop_front(), - None => None, - } - } - - fn waker_ref(&self) -> WakerRef<'_> { - // Set woken to true when enter block_on, ensure outer future - // be polled for the first time when enter loop - self.shared.woken.store(true, Release); - waker_ref(&self.shared) - } - - // reset woken to false and return original value - pub(crate) fn reset_woken(&self) -> bool { - self.shared.woken.swap(false, AcqRel) - } -} - -cfg_metrics! { - impl Spawner { - pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { - &self.shared.scheduler_metrics - } - - pub(crate) fn injection_queue_depth(&self) -> usize { - // TODO: avoid having to lock. The multi-threaded injection queue - // could probably be used here. - self.shared.queue.lock() - .as_ref() - .map(|queue| queue.len()) - .unwrap_or(0) - } - - pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { - assert_eq!(0, worker); - &self.shared.worker_metrics - } - } -} - -impl fmt::Debug for Spawner { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Spawner").finish() - } -} - -// ===== impl Shared ===== - -impl Schedule for Arc { - fn release(&self, task: &Task) -> Option> { - self.owned.remove(task) - } - - fn schedule(&self, task: task::Notified) { - CURRENT.with(|maybe_cx| match maybe_cx { - Some(cx) if Arc::ptr_eq(self, &cx.spawner.shared) => { - let mut core = cx.core.borrow_mut(); - - // If `None`, the runtime is shutting down, so there is no need - // to schedule the task. - if let Some(core) = core.as_mut() { - core.push_task(task); - } - } - _ => { - // Track that a task was scheduled from **outside** of the runtime. - self.scheduler_metrics.inc_remote_schedule_count(); - - // If the queue is None, then the runtime has shut down. We - // don't need to do anything with the notification in that case. - let mut guard = self.queue.lock(); - if let Some(queue) = guard.as_mut() { - queue.push_back(task); - drop(guard); - self.unpark.unpark(); - } - } - }); - } -} - -impl Wake for Shared { - fn wake(arc_self: Arc) { - Wake::wake_by_ref(&arc_self) - } - - /// Wake by reference - fn wake_by_ref(arc_self: &Arc) { - arc_self.woken.store(true, Release); - arc_self.unpark.unpark(); - } -} - -// ===== CoreGuard ===== - -/// Used to ensure we always place the `Core` value back into its slot in -/// `BasicScheduler`, even if the future panics. -struct CoreGuard<'a> { - context: Context, - basic_scheduler: &'a BasicScheduler, -} - -impl CoreGuard<'_> { - fn block_on(self, future: F) -> F::Output { - self.enter(|mut core, context| { - let _enter = crate::runtime::enter(false); - let waker = context.spawner.waker_ref(); - let mut cx = std::task::Context::from_waker(&waker); - - pin!(future); - - 'outer: loop { - if core.spawner.reset_woken() { - let (c, res) = context.enter(core, || { - crate::coop::budget(|| future.as_mut().poll(&mut cx)) - }); - - core = c; - - if let Ready(v) = res { - return (core, v); - } - } - - for _ in 0..MAX_TASKS_PER_TICK { - // Get and increment the current tick - let tick = core.tick; - core.tick = core.tick.wrapping_add(1); - - let entry = if tick % REMOTE_FIRST_INTERVAL == 0 { - core.spawner.pop().or_else(|| core.tasks.pop_front()) - } else { - core.tasks.pop_front().or_else(|| core.spawner.pop()) - }; - - let task = match entry { - Some(entry) => entry, - None => { - core = context.park(core); - - // Try polling the `block_on` future next - continue 'outer; - } - }; - - let task = context.spawner.shared.owned.assert_owner(task); - - let (c, _) = context.run_task(core, || { - task.run(); - }); - - core = c; - } - - // Yield to the driver, this drives the timer and pulls any - // pending I/O events. - core = context.park_yield(core); - } - }) - } - - /// Enters the scheduler context. This sets the queue and other necessary - /// scheduler state in the thread-local. - fn enter(self, f: F) -> R - where - F: FnOnce(Box, &Context) -> (Box, R), - { - // Remove `core` from `context` to pass into the closure. - let core = self.context.core.borrow_mut().take().expect("core missing"); - - // Call the closure and place `core` back - let (core, ret) = CURRENT.set(&self.context, || f(core, &self.context)); - - *self.context.core.borrow_mut() = Some(core); - - ret - } -} - -impl Drop for CoreGuard<'_> { - fn drop(&mut self) { - if let Some(core) = self.context.core.borrow_mut().take() { - // Replace old scheduler back into the state to allow - // other threads to pick it up and drive it. - self.basic_scheduler.core.set(core); - - // Wake up other possible threads that could steal the driver. - self.basic_scheduler.notify.notify_one() - } - } -} diff --git a/third_party/rust/tokio/src/runtime/blocking/mod.rs b/third_party/rust/tokio/src/runtime/blocking/mod.rs index 15fe05c9ade2..c42924be77df 100644 --- a/third_party/rust/tokio/src/runtime/blocking/mod.rs +++ b/third_party/rust/tokio/src/runtime/blocking/mod.rs @@ -4,16 +4,19 @@ //! compilation. mod pool; -pub(crate) use pool::{spawn_blocking, BlockingPool, Mandatory, Spawner, Task}; +pub(crate) use pool::{spawn_blocking, BlockingPool, Spawner}; cfg_fs! { pub(crate) use pool::spawn_mandatory_blocking; } +cfg_trace! { + pub(crate) use pool::Mandatory; +} + mod schedule; mod shutdown; mod task; -pub(crate) use schedule::NoopSchedule; pub(crate) use task::BlockingTask; use crate::runtime::Builder; @@ -21,28 +24,3 @@ use crate::runtime::Builder; pub(crate) fn create_blocking_pool(builder: &Builder, thread_cap: usize) -> BlockingPool { BlockingPool::new(builder, thread_cap) } - -/* -cfg_not_blocking_impl! { - use crate::runtime::Builder; - use std::time::Duration; - - #[derive(Debug, Clone)] - pub(crate) struct BlockingPool {} - - pub(crate) use BlockingPool as Spawner; - - pub(crate) fn create_blocking_pool(_builder: &Builder, _thread_cap: usize) -> BlockingPool { - BlockingPool {} - } - - impl BlockingPool { - pub(crate) fn spawner(&self) -> &BlockingPool { - self - } - - pub(crate) fn shutdown(&mut self, _duration: Option) { - } - } -} -*/ diff --git a/third_party/rust/tokio/src/runtime/blocking/pool.rs b/third_party/rust/tokio/src/runtime/blocking/pool.rs index daf1f63fac34..a23b0a0d2de6 100644 --- a/third_party/rust/tokio/src/runtime/blocking/pool.rs +++ b/third_party/rust/tokio/src/runtime/blocking/pool.rs @@ -2,15 +2,16 @@ use crate::loom::sync::{Arc, Condvar, Mutex}; use crate::loom::thread; -use crate::runtime::blocking::schedule::NoopSchedule; -use crate::runtime::blocking::shutdown; +use crate::runtime::blocking::schedule::BlockingSchedule; +use crate::runtime::blocking::{shutdown, BlockingTask}; use crate::runtime::builder::ThreadNameFn; -use crate::runtime::context; use crate::runtime::task::{self, JoinHandle}; use crate::runtime::{Builder, Callback, Handle}; use std::collections::{HashMap, VecDeque}; use std::fmt; +use std::io; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; pub(crate) struct BlockingPool { @@ -23,6 +24,53 @@ pub(crate) struct Spawner { inner: Arc, } +#[derive(Default)] +pub(crate) struct SpawnerMetrics { + num_threads: AtomicUsize, + num_idle_threads: AtomicUsize, + queue_depth: AtomicUsize, +} + +impl SpawnerMetrics { + fn num_threads(&self) -> usize { + self.num_threads.load(Ordering::Relaxed) + } + + fn num_idle_threads(&self) -> usize { + self.num_idle_threads.load(Ordering::Relaxed) + } + + cfg_metrics! { + fn queue_depth(&self) -> usize { + self.queue_depth.load(Ordering::Relaxed) + } + } + + fn inc_num_threads(&self) { + self.num_threads.fetch_add(1, Ordering::Relaxed); + } + + fn dec_num_threads(&self) { + self.num_threads.fetch_sub(1, Ordering::Relaxed); + } + + fn inc_num_idle_threads(&self) { + self.num_idle_threads.fetch_add(1, Ordering::Relaxed); + } + + fn dec_num_idle_threads(&self) -> usize { + self.num_idle_threads.fetch_sub(1, Ordering::Relaxed) + } + + fn inc_queue_depth(&self) { + self.queue_depth.fetch_add(1, Ordering::Relaxed); + } + + fn dec_queue_depth(&self) { + self.queue_depth.fetch_sub(1, Ordering::Relaxed); + } +} + struct Inner { /// State shared between worker threads. shared: Mutex, @@ -47,12 +95,13 @@ struct Inner { // Customizable wait timeout. keep_alive: Duration, + + // Metrics about the pool. + metrics: SpawnerMetrics, } struct Shared { queue: VecDeque, - num_th: usize, - num_idle: u32, num_notify: u32, shutdown: bool, shutdown_tx: Option, @@ -71,7 +120,7 @@ struct Shared { } pub(crate) struct Task { - task: task::UnownedTask, + task: task::UnownedTask, mandatory: Mandatory, } @@ -82,8 +131,27 @@ pub(crate) enum Mandatory { NonMandatory, } +pub(crate) enum SpawnError { + /// Pool is shutting down and the task was not scheduled + ShuttingDown, + /// There are no worker threads available to take the task + /// and the OS failed to spawn a new one + NoThreads(io::Error), +} + +impl From for io::Error { + fn from(e: SpawnError) -> Self { + match e { + SpawnError::ShuttingDown => { + io::Error::new(io::ErrorKind::Other, "blocking pool shutting down") + } + SpawnError::NoThreads(e) => e, + } + } +} + impl Task { - pub(crate) fn new(task: task::UnownedTask, mandatory: Mandatory) -> Task { + pub(crate) fn new(task: task::UnownedTask, mandatory: Mandatory) -> Task { Task { task, mandatory } } @@ -104,12 +172,14 @@ const KEEP_ALIVE: Duration = Duration::from_secs(10); /// Runs the provided function on an executor dedicated to blocking operations. /// Tasks will be scheduled as non-mandatory, meaning they may not get executed /// in case of runtime shutdown. +#[track_caller] +#[cfg_attr(tokio_wasi, allow(dead_code))] pub(crate) fn spawn_blocking(func: F) -> JoinHandle where F: FnOnce() -> R + Send + 'static, R: Send + 'static, { - let rt = context::current(); + let rt = Handle::current(); rt.spawn_blocking(func) } @@ -127,8 +197,8 @@ cfg_fs! { F: FnOnce() -> R + Send + 'static, R: Send + 'static, { - let rt = context::current(); - rt.spawn_mandatory_blocking(func) + let rt = Handle::current(); + rt.inner.blocking_spawner().spawn_mandatory_blocking(&rt, func) } } @@ -144,8 +214,6 @@ impl BlockingPool { inner: Arc::new(Inner { shared: Mutex::new(Shared { queue: VecDeque::new(), - num_th: 0, - num_idle: 0, num_notify: 0, shutdown: false, shutdown_tx: Some(shutdown_tx), @@ -160,6 +228,7 @@ impl BlockingPool { before_stop: builder.before_stop.clone(), thread_cap, keep_alive, + metrics: Default::default(), }), }, shutdown_rx, @@ -219,7 +288,106 @@ impl fmt::Debug for BlockingPool { // ===== impl Spawner ===== impl Spawner { - pub(crate) fn spawn(&self, task: Task, rt: &Handle) -> Result<(), ()> { + #[track_caller] + pub(crate) fn spawn_blocking(&self, rt: &Handle, func: F) -> JoinHandle + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + let (join_handle, spawn_result) = + if cfg!(debug_assertions) && std::mem::size_of::() > 2048 { + self.spawn_blocking_inner(Box::new(func), Mandatory::NonMandatory, None, rt) + } else { + self.spawn_blocking_inner(func, Mandatory::NonMandatory, None, rt) + }; + + match spawn_result { + Ok(()) => join_handle, + // Compat: do not panic here, return the join_handle even though it will never resolve + Err(SpawnError::ShuttingDown) => join_handle, + Err(SpawnError::NoThreads(e)) => { + panic!("OS can't spawn worker thread: {}", e) + } + } + } + + cfg_fs! { + #[track_caller] + #[cfg_attr(any( + all(loom, not(test)), // the function is covered by loom tests + test + ), allow(dead_code))] + pub(crate) fn spawn_mandatory_blocking(&self, rt: &Handle, func: F) -> Option> + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + let (join_handle, spawn_result) = if cfg!(debug_assertions) && std::mem::size_of::() > 2048 { + self.spawn_blocking_inner( + Box::new(func), + Mandatory::Mandatory, + None, + rt, + ) + } else { + self.spawn_blocking_inner( + func, + Mandatory::Mandatory, + None, + rt, + ) + }; + + if spawn_result.is_ok() { + Some(join_handle) + } else { + None + } + } + } + + #[track_caller] + pub(crate) fn spawn_blocking_inner( + &self, + func: F, + is_mandatory: Mandatory, + name: Option<&str>, + rt: &Handle, + ) -> (JoinHandle, Result<(), SpawnError>) + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + let fut = BlockingTask::new(func); + let id = task::Id::next(); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let fut = { + use tracing::Instrument; + let location = std::panic::Location::caller(); + let span = tracing::trace_span!( + target: "tokio::task::blocking", + "runtime.spawn", + kind = %"blocking", + task.name = %name.unwrap_or_default(), + task.id = id.as_u64(), + "fn" = %std::any::type_name::(), + loc.file = location.file(), + loc.line = location.line(), + loc.col = location.column(), + ); + fut.instrument(span) + }; + + #[cfg(not(all(tokio_unstable, feature = "tracing")))] + let _ = name; + + let (task, handle) = task::unowned(fut, BlockingSchedule::new(rt), id); + + let spawned = self.spawn_task(Task::new(task, is_mandatory), rt); + (handle, spawned) + } + + fn spawn_task(&self, task: Task, rt: &Handle) -> Result<(), SpawnError> { let mut shared = self.inner.shared.lock(); if shared.shutdown { @@ -229,28 +397,44 @@ impl Spawner { task.task.shutdown(); // no need to even push this task; it would never get picked up - return Err(()); + return Err(SpawnError::ShuttingDown); } shared.queue.push_back(task); + self.inner.metrics.inc_queue_depth(); - if shared.num_idle == 0 { + if self.inner.metrics.num_idle_threads() == 0 { // No threads are able to process the task. - if shared.num_th == self.inner.thread_cap { + if self.inner.metrics.num_threads() == self.inner.thread_cap { // At max number of threads } else { - shared.num_th += 1; assert!(shared.shutdown_tx.is_some()); let shutdown_tx = shared.shutdown_tx.clone(); if let Some(shutdown_tx) = shutdown_tx { let id = shared.worker_thread_index; - shared.worker_thread_index += 1; - let handle = self.spawn_thread(shutdown_tx, rt, id); - - shared.worker_threads.insert(id, handle); + match self.spawn_thread(shutdown_tx, rt, id) { + Ok(handle) => { + self.inner.metrics.inc_num_threads(); + shared.worker_thread_index += 1; + shared.worker_threads.insert(id, handle); + } + Err(ref e) + if is_temporary_os_thread_error(e) + && self.inner.metrics.num_threads() > 0 => + { + // OS temporarily failed to spawn a new thread. + // The task will be picked up eventually by a currently + // busy thread. + } + Err(e) => { + // The OS refused to spawn the thread and there is no thread + // to pick up the task that has just been pushed to the queue. + return Err(SpawnError::NoThreads(e)); + } + } } } } else { @@ -259,7 +443,7 @@ impl Spawner { // exactly. Thread libraries may generate spurious // wakeups, this counter is used to keep us in a // consistent state. - shared.num_idle -= 1; + self.inner.metrics.dec_num_idle_threads(); shared.num_notify += 1; self.inner.condvar.notify_one(); } @@ -272,7 +456,7 @@ impl Spawner { shutdown_tx: shutdown::Sender, rt: &Handle, id: usize, - ) -> thread::JoinHandle<()> { + ) -> std::io::Result> { let mut builder = thread::Builder::new().name((self.inner.thread_name)()); if let Some(stack_size) = self.inner.stack_size { @@ -281,17 +465,37 @@ impl Spawner { let rt = rt.clone(); - builder - .spawn(move || { - // Only the reference should be moved into the closure - let _enter = crate::runtime::context::enter(rt.clone()); - rt.blocking_spawner.inner.run(id); - drop(shutdown_tx); - }) - .expect("OS can't spawn a new worker thread") + builder.spawn(move || { + // Only the reference should be moved into the closure + let _enter = rt.enter(); + rt.inner.blocking_spawner().inner.run(id); + drop(shutdown_tx); + }) } } +cfg_metrics! { + impl Spawner { + pub(crate) fn num_threads(&self) -> usize { + self.inner.metrics.num_threads() + } + + pub(crate) fn num_idle_threads(&self) -> usize { + self.inner.metrics.num_idle_threads() + } + + pub(crate) fn queue_depth(&self) -> usize { + self.inner.metrics.queue_depth() + } + } +} + +// Tells whether the error when spawning a thread is temporary. +#[inline] +fn is_temporary_os_thread_error(error: &std::io::Error) -> bool { + matches!(error.kind(), std::io::ErrorKind::WouldBlock) +} + impl Inner { fn run(&self, worker_thread_id: usize) { if let Some(f) = &self.after_start { @@ -304,6 +508,7 @@ impl Inner { 'main: loop { // BUSY while let Some(task) = shared.queue.pop_front() { + self.metrics.dec_queue_depth(); drop(shared); task.run(); @@ -311,7 +516,7 @@ impl Inner { } // IDLE - shared.num_idle += 1; + self.metrics.inc_num_idle_threads(); while !shared.shutdown { let lock_result = self.condvar.wait_timeout(shared, self.keep_alive).unwrap(); @@ -345,6 +550,7 @@ impl Inner { if shared.shutdown { // Drain the queue while let Some(task) = shared.queue.pop_front() { + self.metrics.dec_queue_depth(); drop(shared); task.shutdown_or_run_if_mandatory(); @@ -355,7 +561,7 @@ impl Inner { // Work was produced, and we "took" it (by decrementing num_notify). // This means that num_idle was decremented once for our wakeup. // But, since we are exiting, we need to "undo" that, as we'll stay idle. - shared.num_idle += 1; + self.metrics.inc_num_idle_threads(); // NOTE: Technically we should also do num_notify++ and notify again, // but since we're shutting down anyway, that won't be necessary. break; @@ -363,17 +569,17 @@ impl Inner { } // Thread exit - shared.num_th -= 1; + self.metrics.dec_num_threads(); // num_idle should now be tracked exactly, panic // with a descriptive message if it is not the // case. - shared.num_idle = shared - .num_idle - .checked_sub(1) - .expect("num_idle underflowed on thread exit"); + let prev_idle = self.metrics.dec_num_idle_threads(); + if prev_idle < self.metrics.num_idle_threads() { + panic!("num_idle_threads underflowed on thread exit") + } - if shared.shutdown && shared.num_th == 0 { + if shared.shutdown && self.metrics.num_threads() == 0 { self.condvar.notify_one(); } diff --git a/third_party/rust/tokio/src/runtime/blocking/schedule.rs b/third_party/rust/tokio/src/runtime/blocking/schedule.rs index 54252241d944..edf775be8be7 100644 --- a/third_party/rust/tokio/src/runtime/blocking/schedule.rs +++ b/third_party/rust/tokio/src/runtime/blocking/schedule.rs @@ -1,15 +1,52 @@ +#[cfg(feature = "test-util")] +use crate::runtime::scheduler; use crate::runtime::task::{self, Task}; +use crate::runtime::Handle; -/// `task::Schedule` implementation that does nothing. This is unique to the -/// blocking scheduler as tasks scheduled are not really futures but blocking -/// operations. +/// `task::Schedule` implementation that does nothing (except some bookkeeping +/// in test-util builds). This is unique to the blocking scheduler as tasks +/// scheduled are not really futures but blocking operations. /// /// We avoid storing the task by forgetting it in `bind` and re-materializing it -/// in `release. -pub(crate) struct NoopSchedule; +/// in `release`. +pub(crate) struct BlockingSchedule { + #[cfg(feature = "test-util")] + handle: Handle, +} -impl task::Schedule for NoopSchedule { +impl BlockingSchedule { + #[cfg_attr(not(feature = "test-util"), allow(unused_variables))] + pub(crate) fn new(handle: &Handle) -> Self { + #[cfg(feature = "test-util")] + { + match &handle.inner { + scheduler::Handle::CurrentThread(handle) => { + handle.driver.clock.inhibit_auto_advance(); + } + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Handle::MultiThread(_) => {} + } + } + BlockingSchedule { + #[cfg(feature = "test-util")] + handle: handle.clone(), + } + } +} + +impl task::Schedule for BlockingSchedule { fn release(&self, _task: &Task) -> Option> { + #[cfg(feature = "test-util")] + { + match &self.handle.inner { + scheduler::Handle::CurrentThread(handle) => { + handle.driver.clock.allow_auto_advance(); + handle.driver.unpark(); + } + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Handle::MultiThread(_) => {} + } + } None } diff --git a/third_party/rust/tokio/src/runtime/blocking/shutdown.rs b/third_party/rust/tokio/src/runtime/blocking/shutdown.rs index e6f4674183ec..fe5abae076d6 100644 --- a/third_party/rust/tokio/src/runtime/blocking/shutdown.rs +++ b/third_party/rust/tokio/src/runtime/blocking/shutdown.rs @@ -35,13 +35,13 @@ impl Receiver { /// /// If the timeout has elapsed, it returns `false`, otherwise it returns `true`. pub(crate) fn wait(&mut self, timeout: Option) -> bool { - use crate::runtime::enter::try_enter; + use crate::runtime::context::try_enter_blocking_region; if timeout == Some(Duration::from_nanos(0)) { return false; } - let mut e = match try_enter(false) { + let mut e = match try_enter_blocking_region() { Some(enter) => enter, _ => { if std::thread::panicking() { diff --git a/third_party/rust/tokio/src/runtime/blocking/task.rs b/third_party/rust/tokio/src/runtime/blocking/task.rs index 0b7803a6c0b3..c4461754005d 100644 --- a/third_party/rust/tokio/src/runtime/blocking/task.rs +++ b/third_party/rust/tokio/src/runtime/blocking/task.rs @@ -37,7 +37,7 @@ where // currently goes through Task::poll(), and so is subject to budgeting. That isn't really // what we want; a blocking task may itself want to run tasks (it might be a Worker!), so // we want it to start without any budgeting. - crate::coop::stop(); + crate::runtime::coop::stop(); Poll::Ready(func()) } diff --git a/third_party/rust/tokio/src/runtime/builder.rs b/third_party/rust/tokio/src/runtime/builder.rs index 91c365fd5162..af9e0e172f31 100644 --- a/third_party/rust/tokio/src/runtime/builder.rs +++ b/third_party/rust/tokio/src/runtime/builder.rs @@ -1,5 +1,6 @@ use crate::runtime::handle::Handle; -use crate::runtime::{blocking, driver, Callback, Runtime, Spawner}; +use crate::runtime::{blocking, driver, Callback, HistogramBuilder, Runtime}; +use crate::util::rand::{RngSeed, RngSeedGenerator}; use std::fmt; use std::io; @@ -43,6 +44,7 @@ pub struct Builder { /// Whether or not to enable the I/O driver enable_io: bool, + nevents: usize, /// Whether or not to enable the time driver enable_time: bool, @@ -78,13 +80,124 @@ pub struct Builder { /// Customizable keep alive timeout for BlockingPool pub(super) keep_alive: Option, + + /// How many ticks before pulling a task from the global/remote queue? + /// + /// When `None`, the value is unspecified and behavior details are left to + /// the scheduler. Each scheduler flavor could choose to either pick its own + /// default value or use some other strategy to decide when to poll from the + /// global queue. For example, the multi-threaded scheduler uses a + /// self-tuning strategy based on mean task poll times. + pub(super) global_queue_interval: Option, + + /// How many ticks before yielding to the driver for timer and I/O events? + pub(super) event_interval: u32, + + /// When true, the multi-threade scheduler LIFO slot should not be used. + /// + /// This option should only be exposed as unstable. + pub(super) disable_lifo_slot: bool, + + /// Specify a random number generator seed to provide deterministic results + pub(super) seed_generator: RngSeedGenerator, + + /// When true, enables task poll count histogram instrumentation. + pub(super) metrics_poll_count_histogram_enable: bool, + + /// Configures the task poll count histogram + pub(super) metrics_poll_count_histogram: HistogramBuilder, + + #[cfg(tokio_unstable)] + pub(super) unhandled_panic: UnhandledPanic, +} + +cfg_unstable! { + /// How the runtime should respond to unhandled panics. + /// + /// Instances of `UnhandledPanic` are passed to `Builder::unhandled_panic` + /// to configure the runtime behavior when a spawned task panics. + /// + /// See [`Builder::unhandled_panic`] for more details. + #[derive(Debug, Clone)] + #[non_exhaustive] + pub enum UnhandledPanic { + /// The runtime should ignore panics on spawned tasks. + /// + /// The panic is forwarded to the task's [`JoinHandle`] and all spawned + /// tasks continue running normally. + /// + /// This is the default behavior. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{self, UnhandledPanic}; + /// + /// # pub fn main() { + /// let rt = runtime::Builder::new_current_thread() + /// .unhandled_panic(UnhandledPanic::Ignore) + /// .build() + /// .unwrap(); + /// + /// let task1 = rt.spawn(async { panic!("boom"); }); + /// let task2 = rt.spawn(async { + /// // This task completes normally + /// "done" + /// }); + /// + /// rt.block_on(async { + /// // The panic on the first task is forwarded to the `JoinHandle` + /// assert!(task1.await.is_err()); + /// + /// // The second task completes normally + /// assert!(task2.await.is_ok()); + /// }) + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + Ignore, + + /// The runtime should immediately shutdown if a spawned task panics. + /// + /// The runtime will immediately shutdown even if the panicked task's + /// [`JoinHandle`] is still available. All further spawned tasks will be + /// immediately dropped and call to [`Runtime::block_on`] will panic. + /// + /// # Examples + /// + /// ```should_panic + /// use tokio::runtime::{self, UnhandledPanic}; + /// + /// # pub fn main() { + /// let rt = runtime::Builder::new_current_thread() + /// .unhandled_panic(UnhandledPanic::ShutdownRuntime) + /// .build() + /// .unwrap(); + /// + /// rt.spawn(async { panic!("boom"); }); + /// rt.spawn(async { + /// // This task never completes. + /// }); + /// + /// rt.block_on(async { + /// // Do some work + /// # loop { tokio::task::yield_now().await; } + /// }) + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + ShutdownRuntime, + } } pub(crate) type ThreadNameFn = std::sync::Arc String + Send + Sync + 'static>; +#[derive(Clone, Copy)] pub(crate) enum Kind { CurrentThread, - #[cfg(feature = "rt-multi-thread")] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] MultiThread, } @@ -98,28 +211,38 @@ impl Builder { /// /// [`LocalSet`]: crate::task::LocalSet pub fn new_current_thread() -> Builder { - Builder::new(Kind::CurrentThread) + #[cfg(loom)] + const EVENT_INTERVAL: u32 = 4; + // The number `61` is fairly arbitrary. I believe this value was copied from golang. + #[cfg(not(loom))] + const EVENT_INTERVAL: u32 = 61; + + Builder::new(Kind::CurrentThread, EVENT_INTERVAL) } - /// Returns a new builder with the multi thread scheduler selected. - /// - /// Configuration methods can be chained on the return value. - #[cfg(feature = "rt-multi-thread")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] - pub fn new_multi_thread() -> Builder { - Builder::new(Kind::MultiThread) + cfg_not_wasi! { + /// Returns a new builder with the multi thread scheduler selected. + /// + /// Configuration methods can be chained on the return value. + #[cfg(feature = "rt-multi-thread")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] + pub fn new_multi_thread() -> Builder { + // The number `61` is fairly arbitrary. I believe this value was copied from golang. + Builder::new(Kind::MultiThread, 61) + } } /// Returns a new runtime builder initialized with default configuration /// values. /// /// Configuration methods can be chained on the return value. - pub(crate) fn new(kind: Kind) -> Builder { + pub(crate) fn new(kind: Kind, event_interval: u32) -> Builder { Builder { kind, // I/O defaults to "off" enable_io: false, + nevents: 1024, // Time defaults to "off" enable_time: false, @@ -127,6 +250,7 @@ impl Builder { // The clock starts not-paused start_paused: false, + // Read from environment variable first in multi-threaded mode. // Default to lazy auto-detection (one thread per CPU core) worker_threads: None, @@ -145,6 +269,22 @@ impl Builder { after_unpark: None, keep_alive: None, + + // Defaults for these values depend on the scheduler kind, so we get them + // as parameters. + global_queue_interval: None, + event_interval, + + seed_generator: RngSeedGenerator::new(RngSeed::new()), + + #[cfg(tokio_unstable)] + unhandled_panic: UnhandledPanic::Ignore, + + metrics_poll_count_histogram_enable: false, + + metrics_poll_count_histogram: Default::default(), + + disable_lifo_slot: false, } } @@ -165,7 +305,11 @@ impl Builder { /// .unwrap(); /// ``` pub fn enable_all(&mut self) -> &mut Self { - #[cfg(any(feature = "net", feature = "process", all(unix, feature = "signal")))] + #[cfg(any( + feature = "net", + all(unix, feature = "process"), + all(unix, feature = "signal") + ))] self.enable_io(); #[cfg(feature = "time")] self.enable_time(); @@ -178,15 +322,13 @@ impl Builder { /// This can be any number above 0 though it is advised to keep this value /// on the smaller side. /// + /// This will override the value read from environment variable `TOKIO_WORKER_THREADS`. + /// /// # Default /// /// The default value is the number of cores available to the system. /// - /// # Panic - /// - /// When using the `current_thread` runtime this method will panic, since - /// those variants do not allow setting worker thread counts. - /// + /// When using the `current_thread` runtime this method has no effect. /// /// # Examples /// @@ -219,9 +361,10 @@ impl Builder { /// rt.block_on(async move {}); /// ``` /// - /// # Panic + /// # Panics /// /// This will panic if `val` is not larger than `0`. + #[track_caller] pub fn worker_threads(&mut self, val: usize) -> &mut Self { assert!(val > 0, "Worker threads cannot be set to 0"); self.worker_threads = Some(val); @@ -237,7 +380,7 @@ impl Builder { /// /// The default value is 512. /// - /// # Panic + /// # Panics /// /// This will panic if `val` is not larger than `0`. /// @@ -249,6 +392,7 @@ impl Builder { /// [`spawn_blocking`]: fn@crate::task::spawn_blocking /// [`worker_threads`]: Self::worker_threads /// [`thread_keep_alive`]: Self::thread_keep_alive + #[track_caller] #[cfg_attr(docsrs, doc(alias = "max_threads"))] pub fn max_blocking_threads(&mut self, val: usize) -> &mut Self { assert!(val > 0, "Max blocking threads cannot be set to 0"); @@ -286,7 +430,6 @@ impl Builder { /// ``` /// # use tokio::runtime; /// # use std::sync::atomic::{AtomicUsize, Ordering}; - /// /// # pub fn main() { /// let rt = runtime::Builder::new_multi_thread() /// .thread_name_fn(|| { @@ -338,7 +481,6 @@ impl Builder { /// /// ``` /// # use tokio::runtime; - /// /// # pub fn main() { /// let runtime = runtime::Builder::new_multi_thread() /// .on_thread_start(|| { @@ -364,7 +506,6 @@ impl Builder { /// /// ``` /// # use tokio::runtime; - /// /// # pub fn main() { /// let runtime = runtime::Builder::new_multi_thread() /// .on_thread_stop(|| { @@ -473,7 +614,6 @@ impl Builder { /// /// ``` /// # use tokio::runtime; - /// /// # pub fn main() { /// let runtime = runtime::Builder::new_multi_thread() /// .on_thread_unpark(|| { @@ -513,8 +653,8 @@ impl Builder { /// ``` pub fn build(&mut self) -> io::Result { match &self.kind { - Kind::CurrentThread => self.build_basic_runtime(), - #[cfg(feature = "rt-multi-thread")] + Kind::CurrentThread => self.build_current_thread_runtime(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] Kind::MultiThread => self.build_threaded_runtime(), } } @@ -523,12 +663,13 @@ impl Builder { driver::Cfg { enable_pause_time: match self.kind { Kind::CurrentThread => true, - #[cfg(feature = "rt-multi-thread")] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] Kind::MultiThread => false, }, enable_io: self.enable_io, enable_time: self.enable_time, start_paused: self.start_paused, + nevents: self.nevents, } } @@ -542,7 +683,6 @@ impl Builder { /// ``` /// # use tokio::runtime; /// # use std::time::Duration; - /// /// # pub fn main() { /// let rt = runtime::Builder::new_multi_thread() /// .thread_keep_alive(Duration::from_millis(100)) @@ -554,35 +694,385 @@ impl Builder { self } - fn build_basic_runtime(&mut self) -> io::Result { - use crate::runtime::{BasicScheduler, Kind}; + /// Sets the number of scheduler ticks after which the scheduler will poll the global + /// task queue. + /// + /// A scheduler "tick" roughly corresponds to one `poll` invocation on a task. + /// + /// By default the global queue interval is: + /// + /// * `31` for the current-thread scheduler. + /// * `61` for the multithreaded scheduler. + /// + /// Schedulers have a local queue of already-claimed tasks, and a global queue of incoming + /// tasks. Setting the interval to a smaller value increases the fairness of the scheduler, + /// at the cost of more synchronization overhead. That can be beneficial for prioritizing + /// getting started on new work, especially if tasks frequently yield rather than complete + /// or await on further I/O. Conversely, a higher value prioritizes existing work, and + /// is a good choice when most tasks quickly complete polling. + /// + /// # Examples + /// + /// ``` + /// # use tokio::runtime; + /// # pub fn main() { + /// let rt = runtime::Builder::new_multi_thread() + /// .global_queue_interval(31) + /// .build(); + /// # } + /// ``` + pub fn global_queue_interval(&mut self, val: u32) -> &mut Self { + self.global_queue_interval = Some(val); + self + } - let (driver, resources) = driver::Driver::new(self.get_cfg())?; + /// Sets the number of scheduler ticks after which the scheduler will poll for + /// external events (timers, I/O, and so on). + /// + /// A scheduler "tick" roughly corresponds to one `poll` invocation on a task. + /// + /// By default, the event interval is `61` for all scheduler types. + /// + /// Setting the event interval determines the effective "priority" of delivering + /// these external events (which may wake up additional tasks), compared to + /// executing tasks that are currently ready to run. A smaller value is useful + /// when tasks frequently spend a long time in polling, or frequently yield, + /// which can result in overly long delays picking up I/O events. Conversely, + /// picking up new events requires extra synchronization and syscall overhead, + /// so if tasks generally complete their polling quickly, a higher event interval + /// will minimize that overhead while still keeping the scheduler responsive to + /// events. + /// + /// # Examples + /// + /// ``` + /// # use tokio::runtime; + /// # pub fn main() { + /// let rt = runtime::Builder::new_multi_thread() + /// .event_interval(31) + /// .build(); + /// # } + /// ``` + pub fn event_interval(&mut self, val: u32) -> &mut Self { + self.event_interval = val; + self + } - // And now put a single-threaded scheduler on top of the timer. When - // there are no futures ready to do something, it'll let the timer or - // the reactor to generate some new stimuli for the futures to continue - // in their life. - let scheduler = - BasicScheduler::new(driver, self.before_park.clone(), self.after_unpark.clone()); - let spawner = Spawner::Basic(scheduler.spawner().clone()); + cfg_unstable! { + /// Configure how the runtime responds to an unhandled panic on a + /// spawned task. + /// + /// By default, an unhandled panic (i.e. a panic not caught by + /// [`std::panic::catch_unwind`]) has no impact on the runtime's + /// execution. The panic is error value is forwarded to the task's + /// [`JoinHandle`] and all other spawned tasks continue running. + /// + /// The `unhandled_panic` option enables configuring this behavior. + /// + /// * `UnhandledPanic::Ignore` is the default behavior. Panics on + /// spawned tasks have no impact on the runtime's execution. + /// * `UnhandledPanic::ShutdownRuntime` will force the runtime to + /// shutdown immediately when a spawned task panics even if that + /// task's `JoinHandle` has not been dropped. All other spawned tasks + /// will immediately terminate and further calls to + /// [`Runtime::block_on`] will panic. + /// + /// # Unstable + /// + /// This option is currently unstable and its implementation is + /// incomplete. The API may change or be removed in the future. See + /// tokio-rs/tokio#4516 for more details. + /// + /// # Examples + /// + /// The following demonstrates a runtime configured to shutdown on + /// panic. The first spawned task panics and results in the runtime + /// shutting down. The second spawned task never has a chance to + /// execute. The call to `block_on` will panic due to the runtime being + /// forcibly shutdown. + /// + /// ```should_panic + /// use tokio::runtime::{self, UnhandledPanic}; + /// + /// # pub fn main() { + /// let rt = runtime::Builder::new_current_thread() + /// .unhandled_panic(UnhandledPanic::ShutdownRuntime) + /// .build() + /// .unwrap(); + /// + /// rt.spawn(async { panic!("boom"); }); + /// rt.spawn(async { + /// // This task never completes. + /// }); + /// + /// rt.block_on(async { + /// // Do some work + /// # loop { tokio::task::yield_now().await; } + /// }) + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + pub fn unhandled_panic(&mut self, behavior: UnhandledPanic) -> &mut Self { + self.unhandled_panic = behavior; + self + } + + /// Disables the LIFO task scheduler heuristic. + /// + /// The multi-threaded scheduler includes a heuristic for optimizing + /// message-passing patterns. This heuristic results in the **last** + /// scheduled task being polled first. + /// + /// To implement this heuristic, each worker thread has a slot which + /// holds the task that should be polled next. However, this slot cannot + /// be stolen by other worker threads, which can result in lower total + /// throughput when tasks tend to have longer poll times. + /// + /// This configuration option will disable this heuristic resulting in + /// all scheduled tasks being pushed into the worker-local queue, which + /// is stealable. + /// + /// Consider trying this option when the task "scheduled" time is high + /// but the runtime is underutilized. Use tokio-rs/tokio-metrics to + /// collect this data. + /// + /// # Unstable + /// + /// This configuration option is considered a workaround for the LIFO + /// slot not being stealable. When the slot becomes stealable, we will + /// revisit whether or not this option is necessary. See + /// tokio-rs/tokio#4941. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_multi_thread() + /// .disable_lifo_slot() + /// .build() + /// .unwrap(); + /// ``` + pub fn disable_lifo_slot(&mut self) -> &mut Self { + self.disable_lifo_slot = true; + self + } + + /// Specifies the random number generation seed to use within all + /// threads associated with the runtime being built. + /// + /// This option is intended to make certain parts of the runtime + /// deterministic (e.g. the [`tokio::select!`] macro). In the case of + /// [`tokio::select!`] it will ensure that the order that branches are + /// polled is deterministic. + /// + /// In addition to the code specifying `rng_seed` and interacting with + /// the runtime, the internals of Tokio and the Rust compiler may affect + /// the sequences of random numbers. In order to ensure repeatable + /// results, the version of Tokio, the versions of all other + /// dependencies that interact with Tokio, and the Rust compiler version + /// should also all remain constant. + /// + /// # Examples + /// + /// ``` + /// # use tokio::runtime::{self, RngSeed}; + /// # pub fn main() { + /// let seed = RngSeed::from_bytes(b"place your seed here"); + /// let rt = runtime::Builder::new_current_thread() + /// .rng_seed(seed) + /// .build(); + /// # } + /// ``` + /// + /// [`tokio::select!`]: crate::select + pub fn rng_seed(&mut self, seed: RngSeed) -> &mut Self { + self.seed_generator = RngSeedGenerator::new(seed); + self + } + } + + cfg_metrics! { + /// Enables tracking the distribution of task poll times. + /// + /// Task poll times are not instrumented by default as doing so requires + /// calling [`Instant::now()`] twice per task poll, which could add + /// measurable overhead. Use the [`Handle::metrics()`] to access the + /// metrics data. + /// + /// The histogram uses fixed bucket sizes. In other words, the histogram + /// buckets are not dynamic based on input values. Use the + /// `metrics_poll_count_histogram_` builder methods to configure the + /// histogram details. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_multi_thread() + /// .enable_metrics_poll_count_histogram() + /// .build() + /// .unwrap(); + /// # // Test default values here + /// # fn us(n: u64) -> std::time::Duration { std::time::Duration::from_micros(n) } + /// # let m = rt.handle().metrics(); + /// # assert_eq!(m.poll_count_histogram_num_buckets(), 10); + /// # assert_eq!(m.poll_count_histogram_bucket_range(0), us(0)..us(100)); + /// # assert_eq!(m.poll_count_histogram_bucket_range(1), us(100)..us(200)); + /// ``` + /// + /// [`Handle::metrics()`]: crate::runtime::Handle::metrics + /// [`Instant::now()`]: std::time::Instant::now + pub fn enable_metrics_poll_count_histogram(&mut self) -> &mut Self { + self.metrics_poll_count_histogram_enable = true; + self + } + + /// Sets the histogram scale for tracking the distribution of task poll + /// times. + /// + /// Tracking the distribution of task poll times can be done using a + /// linear or log scale. When using linear scale, each histogram bucket + /// will represent the same range of poll times. When using log scale, + /// each histogram bucket will cover a range twice as big as the + /// previous bucket. + /// + /// **Default:** linear scale. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{self, HistogramScale}; + /// + /// let rt = runtime::Builder::new_multi_thread() + /// .enable_metrics_poll_count_histogram() + /// .metrics_poll_count_histogram_scale(HistogramScale::Log) + /// .build() + /// .unwrap(); + /// ``` + pub fn metrics_poll_count_histogram_scale(&mut self, histogram_scale: crate::runtime::HistogramScale) -> &mut Self { + self.metrics_poll_count_histogram.scale = histogram_scale; + self + } + + /// Sets the histogram resolution for tracking the distribution of task + /// poll times. + /// + /// The resolution is the histogram's first bucket's range. When using a + /// linear histogram scale, each bucket will cover the same range. When + /// using a log scale, each bucket will cover a range twice as big as + /// the previous bucket. In the log case, the resolution represents the + /// smallest bucket range. + /// + /// Note that, when using log scale, the resolution is rounded up to the + /// nearest power of 2 in nanoseconds. + /// + /// **Default:** 100 microseconds. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// use std::time::Duration; + /// + /// let rt = runtime::Builder::new_multi_thread() + /// .enable_metrics_poll_count_histogram() + /// .metrics_poll_count_histogram_resolution(Duration::from_micros(100)) + /// .build() + /// .unwrap(); + /// ``` + pub fn metrics_poll_count_histogram_resolution(&mut self, resolution: Duration) -> &mut Self { + assert!(resolution > Duration::from_secs(0)); + // Sanity check the argument and also make the cast below safe. + assert!(resolution <= Duration::from_secs(1)); + + let resolution = resolution.as_nanos() as u64; + self.metrics_poll_count_histogram.resolution = resolution; + self + } + + /// Sets the number of buckets for the histogram tracking the + /// distribution of task poll times. + /// + /// The last bucket tracks all greater values that fall out of other + /// ranges. So, configuring the histogram using a linear scale, + /// resolution of 50ms, and 10 buckets, the 10th bucket will track task + /// polls that take more than 450ms to complete. + /// + /// **Default:** 10 + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_multi_thread() + /// .enable_metrics_poll_count_histogram() + /// .metrics_poll_count_histogram_buckets(15) + /// .build() + /// .unwrap(); + /// ``` + pub fn metrics_poll_count_histogram_buckets(&mut self, buckets: usize) -> &mut Self { + self.metrics_poll_count_histogram.num_buckets = buckets; + self + } + } + + fn build_current_thread_runtime(&mut self) -> io::Result { + use crate::runtime::scheduler::{self, CurrentThread}; + use crate::runtime::{runtime::Scheduler, Config}; + + let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; // Blocking pool let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads); let blocking_spawner = blocking_pool.spawner().clone(); - Ok(Runtime { - kind: Kind::CurrentThread(scheduler), - handle: Handle { - spawner, - io_handle: resources.io_handle, - time_handle: resources.time_handle, - signal_handle: resources.signal_handle, - clock: resources.clock, - blocking_spawner, + // Generate a rng seed for this runtime. + let seed_generator_1 = self.seed_generator.next_generator(); + let seed_generator_2 = self.seed_generator.next_generator(); + + // And now put a single-threaded scheduler on top of the timer. When + // there are no futures ready to do something, it'll let the timer or + // the reactor to generate some new stimuli for the futures to continue + // in their life. + let (scheduler, handle) = CurrentThread::new( + driver, + driver_handle, + blocking_spawner, + seed_generator_2, + Config { + before_park: self.before_park.clone(), + after_unpark: self.after_unpark.clone(), + global_queue_interval: self.global_queue_interval, + event_interval: self.event_interval, + #[cfg(tokio_unstable)] + unhandled_panic: self.unhandled_panic.clone(), + disable_lifo_slot: self.disable_lifo_slot, + seed_generator: seed_generator_1, + metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(), }, + ); + + let handle = Handle { + inner: scheduler::Handle::CurrentThread(handle), + }; + + Ok(Runtime::from_parts( + Scheduler::CurrentThread(scheduler), + handle, blocking_pool, - }) + )) + } + + fn metrics_poll_count_histogram_builder(&self) -> Option { + if self.metrics_poll_count_histogram_enable { + Some(self.metrics_poll_count_histogram.clone()) + } else { + None + } } } @@ -607,6 +1097,25 @@ cfg_io_driver! { self.enable_io = true; self } + + /// Enables the I/O driver and configures the max number of events to be + /// processed per tick. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_current_thread() + /// .enable_io() + /// .max_io_events_per_tick(1024) + /// .build() + /// .unwrap(); + /// ``` + pub fn max_io_events_per_tick(&mut self, capacity: usize) -> &mut Self { + self.nevents = capacity; + self + } } } @@ -662,39 +1171,48 @@ cfg_rt_multi_thread! { impl Builder { fn build_threaded_runtime(&mut self) -> io::Result { use crate::loom::sys::num_cpus; - use crate::runtime::{Kind, ThreadPool}; - use crate::runtime::park::Parker; + use crate::runtime::{Config, runtime::Scheduler}; + use crate::runtime::scheduler::{self, MultiThread}; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - let (driver, resources) = driver::Driver::new(self.get_cfg())?; - - let (scheduler, launch) = ThreadPool::new(core_threads, Parker::new(driver), self.before_park.clone(), self.after_unpark.clone()); - let spawner = Spawner::ThreadPool(scheduler.spawner().clone()); + let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; // Create the blocking pool - let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads); + let blocking_pool = + blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads); let blocking_spawner = blocking_pool.spawner().clone(); - // Create the runtime handle - let handle = Handle { - spawner, - io_handle: resources.io_handle, - time_handle: resources.time_handle, - signal_handle: resources.signal_handle, - clock: resources.clock, + // Generate a rng seed for this runtime. + let seed_generator_1 = self.seed_generator.next_generator(); + let seed_generator_2 = self.seed_generator.next_generator(); + + let (scheduler, handle, launch) = MultiThread::new( + core_threads, + driver, + driver_handle, blocking_spawner, - }; + seed_generator_2, + Config { + before_park: self.before_park.clone(), + after_unpark: self.after_unpark.clone(), + global_queue_interval: self.global_queue_interval, + event_interval: self.event_interval, + #[cfg(tokio_unstable)] + unhandled_panic: self.unhandled_panic.clone(), + disable_lifo_slot: self.disable_lifo_slot, + seed_generator: seed_generator_1, + metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(), + }, + ); + + let handle = Handle { inner: scheduler::Handle::MultiThread(handle) }; // Spawn the thread pool workers - let _enter = crate::runtime::context::enter(handle.clone()); + let _enter = handle.enter(); launch.launch(); - Ok(Runtime { - kind: Kind::ThreadPool(scheduler), - handle, - blocking_pool, - }) + Ok(Runtime::from_parts(Scheduler::MultiThread(scheduler), handle, blocking_pool)) } } } diff --git a/third_party/rust/tokio/src/runtime/config.rs b/third_party/rust/tokio/src/runtime/config.rs new file mode 100644 index 000000000000..c42e4fe5a806 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/config.rs @@ -0,0 +1,37 @@ +#![cfg_attr(any(not(feature = "full"), tokio_wasm), allow(dead_code))] +use crate::runtime::Callback; +use crate::util::RngSeedGenerator; + +pub(crate) struct Config { + /// How many ticks before pulling a task from the global/remote queue? + pub(crate) global_queue_interval: Option, + + /// How many ticks before yielding to the driver for timer and I/O events? + pub(crate) event_interval: u32, + + /// Callback for a worker parking itself + pub(crate) before_park: Option, + + /// Callback for a worker unparking itself + pub(crate) after_unpark: Option, + + /// The multi-threaded scheduler includes a per-worker LIFO slot used to + /// store the last scheduled task. This can improve certain usage patterns, + /// especially message passing between tasks. However, this LIFO slot is not + /// currently stealable. + /// + /// Eventually, the LIFO slot **will** become stealable, however as a + /// stop-gap, this unstable option lets users disable the LIFO task. + pub(crate) disable_lifo_slot: bool, + + /// Random number generator seed to configure runtimes to act in a + /// deterministic way. + pub(crate) seed_generator: RngSeedGenerator, + + /// How to build poll time histograms + pub(crate) metrics_poll_count_histogram: Option, + + #[cfg(tokio_unstable)] + /// How to respond to unhandled task panics. + pub(crate) unhandled_panic: crate::runtime::UnhandledPanic, +} diff --git a/third_party/rust/tokio/src/runtime/context.rs b/third_party/rust/tokio/src/runtime/context.rs index 1f44a5340267..5943e9aa9778 100644 --- a/third_party/rust/tokio/src/runtime/context.rs +++ b/third_party/rust/tokio/src/runtime/context.rs @@ -1,111 +1,191 @@ -//! Thread local runtime context -use crate::runtime::{Handle, TryCurrentError}; +use crate::loom::thread::AccessError; +use crate::runtime::coop; -use std::cell::RefCell; +use std::cell::Cell; -thread_local! { - static CONTEXT: RefCell> = RefCell::new(None) -} +#[cfg(any(feature = "rt", feature = "macros"))] +use crate::util::rand::FastRand; -pub(crate) fn try_current() -> Result { - match CONTEXT.try_with(|ctx| ctx.borrow().clone()) { - Ok(Some(handle)) => Ok(handle), - Ok(None) => Err(TryCurrentError::new_no_context()), - Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()), +cfg_rt! { + mod blocking; + pub(crate) use blocking::{disallow_block_in_place, try_enter_blocking_region, BlockingRegionGuard}; + + mod current; + pub(crate) use current::{with_current, try_set_current, SetCurrentGuard}; + + mod runtime; + pub(crate) use runtime::{EnterRuntime, enter_runtime}; + + mod scoped; + use scoped::Scoped; + + use crate::runtime::{scheduler, task::Id}; + + use std::task::Waker; + + cfg_taskdump! { + use crate::runtime::task::trace; } } -pub(crate) fn current() -> Handle { - match try_current() { - Ok(handle) => handle, - Err(e) => panic!("{}", e), - } +cfg_rt_multi_thread! { + mod runtime_mt; + pub(crate) use runtime_mt::{current_enter_context, exit_runtime}; } -cfg_io_driver! { - pub(crate) fn io_handle() -> crate::runtime::driver::IoHandle { - match CONTEXT.try_with(|ctx| { - let ctx = ctx.borrow(); - ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).io_handle.clone() - }) { - Ok(io_handle) => io_handle, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), +struct Context { + /// Uniquely identifies the current thread + #[cfg(feature = "rt")] + thread_id: Cell>, + + /// Handle to the runtime scheduler running on the current thread. + #[cfg(feature = "rt")] + current: current::HandleCell, + + /// Handle to the scheduler's internal "context" + #[cfg(feature = "rt")] + scheduler: Scoped, + + #[cfg(feature = "rt")] + current_task_id: Cell>, + + /// Tracks if the current thread is currently driving a runtime. + /// Note, that if this is set to "entered", the current scheduler + /// handle may not reference the runtime currently executing. This + /// is because other runtime handles may be set to current from + /// within a runtime. + #[cfg(feature = "rt")] + runtime: Cell, + + #[cfg(any(feature = "rt", feature = "macros"))] + rng: Cell>, + + /// Tracks the amount of "work" a task may still do before yielding back to + /// the sheduler + budget: Cell, + + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + ))] + trace: trace::Context, +} + +tokio_thread_local! { + static CONTEXT: Context = const { + Context { + #[cfg(feature = "rt")] + thread_id: Cell::new(None), + + /// Tracks the current runtime handle to use when spawning, + /// accessing drivers, etc... + #[cfg(feature = "rt")] + current: current::HandleCell::new(), + + /// Tracks the current scheduler internal context + #[cfg(feature = "rt")] + scheduler: Scoped::new(), + + #[cfg(feature = "rt")] + current_task_id: Cell::new(None), + + /// Tracks if the current thread is currently driving a runtime. + /// Note, that if this is set to "entered", the current scheduler + /// handle may not reference the runtime currently executing. This + /// is because other runtime handles may be set to current from + /// within a runtime. + #[cfg(feature = "rt")] + runtime: Cell::new(EnterRuntime::NotEntered), + + #[cfg(any(feature = "rt", feature = "macros"))] + rng: Cell::new(None), + + budget: Cell::new(coop::Budget::unconstrained()), + + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64" + ) + ))] + trace: trace::Context::new(), } } } -cfg_signal_internal! { - #[cfg(unix)] - pub(crate) fn signal_handle() -> crate::runtime::driver::SignalHandle { - match CONTEXT.try_with(|ctx| { - let ctx = ctx.borrow(); - ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).signal_handle.clone() - }) { - Ok(signal_handle) => signal_handle, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), - } - } +#[cfg(any(feature = "macros", all(feature = "sync", feature = "rt")))] +pub(crate) fn thread_rng_n(n: u32) -> u32 { + CONTEXT.with(|ctx| { + let mut rng = ctx.rng.get().unwrap_or_else(FastRand::new); + let ret = rng.fastrand_n(n); + ctx.rng.set(Some(rng)); + ret + }) } -cfg_time! { - pub(crate) fn time_handle() -> crate::runtime::driver::TimeHandle { - match CONTEXT.try_with(|ctx| { - let ctx = ctx.borrow(); - ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).time_handle.clone() - }) { - Ok(time_handle) => time_handle, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), - } - } - - cfg_test_util! { - pub(crate) fn clock() -> Option { - match CONTEXT.try_with(|ctx| (*ctx.borrow()).as_ref().map(|ctx| ctx.clock.clone())) { - Ok(clock) => clock, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), - } - } - } +pub(super) fn budget(f: impl FnOnce(&Cell) -> R) -> Result { + CONTEXT.try_with(|ctx| f(&ctx.budget)) } cfg_rt! { - pub(crate) fn spawn_handle() -> Option { - match CONTEXT.try_with(|ctx| (*ctx.borrow()).as_ref().map(|ctx| ctx.spawner.clone())) { - Ok(spawner) => spawner, - Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + use crate::runtime::ThreadId; + + pub(crate) fn thread_id() -> Result { + CONTEXT.try_with(|ctx| { + match ctx.thread_id.get() { + Some(id) => id, + None => { + let id = ThreadId::next(); + ctx.thread_id.set(Some(id)); + id + } + } + }) + } + + pub(crate) fn set_current_task_id(id: Option) -> Option { + CONTEXT.try_with(|ctx| ctx.current_task_id.replace(id)).unwrap_or(None) + } + + pub(crate) fn current_task_id() -> Option { + CONTEXT.try_with(|ctx| ctx.current_task_id.get()).unwrap_or(None) + } + + #[track_caller] + pub(crate) fn defer(waker: &Waker) { + with_scheduler(|maybe_scheduler| { + if let Some(scheduler) = maybe_scheduler { + scheduler.defer(waker); + } else { + // Called from outside of the runtime, immediately wake the + // task. + waker.wake_by_ref(); + } + }); + } + + pub(super) fn set_scheduler(v: &scheduler::Context, f: impl FnOnce() -> R) -> R { + CONTEXT.with(|c| c.scheduler.set(v, f)) + } + + #[track_caller] + pub(super) fn with_scheduler(f: impl FnOnce(Option<&scheduler::Context>) -> R) -> R { + CONTEXT.with(|c| c.scheduler.with(f)) + } + + cfg_taskdump! { + /// SAFETY: Callers of this function must ensure that trace frames always + /// form a valid linked list. + pub(crate) unsafe fn with_trace(f: impl FnOnce(&trace::Context) -> R) -> Option { + CONTEXT.try_with(|c| f(&c.trace)).ok() } } } - -/// Sets this [`Handle`] as the current active [`Handle`]. -/// -/// [`Handle`]: Handle -pub(crate) fn enter(new: Handle) -> EnterGuard { - match try_enter(new) { - Some(guard) => guard, - None => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), - } -} - -/// Sets this [`Handle`] as the current active [`Handle`]. -/// -/// [`Handle`]: Handle -pub(crate) fn try_enter(new: Handle) -> Option { - CONTEXT - .try_with(|ctx| { - let old = ctx.borrow_mut().replace(new); - EnterGuard(old) - }) - .ok() -} - -#[derive(Debug)] -pub(crate) struct EnterGuard(Option); - -impl Drop for EnterGuard { - fn drop(&mut self) { - CONTEXT.with(|ctx| { - *ctx.borrow_mut() = self.0.take(); - }); - } -} diff --git a/third_party/rust/tokio/src/runtime/context/blocking.rs b/third_party/rust/tokio/src/runtime/context/blocking.rs new file mode 100644 index 000000000000..8ae4f570e855 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/context/blocking.rs @@ -0,0 +1,121 @@ +use super::{EnterRuntime, CONTEXT}; + +use crate::loom::thread::AccessError; +use crate::util::markers::NotSendOrSync; + +use std::marker::PhantomData; +use std::time::Duration; + +/// Guard tracking that a caller has entered a blocking region. +#[must_use] +pub(crate) struct BlockingRegionGuard { + _p: PhantomData, +} + +pub(crate) struct DisallowBlockInPlaceGuard(bool); + +pub(crate) fn try_enter_blocking_region() -> Option { + CONTEXT + .try_with(|c| { + if c.runtime.get().is_entered() { + None + } else { + Some(BlockingRegionGuard::new()) + } + // If accessing the thread-local fails, the thread is terminating + // and thread-locals are being destroyed. Because we don't know if + // we are currently in a runtime or not, we default to being + // permissive. + }) + .unwrap_or_else(|_| Some(BlockingRegionGuard::new())) +} + +/// Disallows blocking in the current runtime context until the guard is dropped. +pub(crate) fn disallow_block_in_place() -> DisallowBlockInPlaceGuard { + let reset = CONTEXT.with(|c| { + if let EnterRuntime::Entered { + allow_block_in_place: true, + } = c.runtime.get() + { + c.runtime.set(EnterRuntime::Entered { + allow_block_in_place: false, + }); + true + } else { + false + } + }); + + DisallowBlockInPlaceGuard(reset) +} + +impl BlockingRegionGuard { + pub(super) fn new() -> BlockingRegionGuard { + BlockingRegionGuard { _p: PhantomData } + } + + /// Blocks the thread on the specified future, returning the value with + /// which that future completes. + pub(crate) fn block_on(&mut self, f: F) -> Result + where + F: std::future::Future, + { + use crate::runtime::park::CachedParkThread; + + let mut park = CachedParkThread::new(); + park.block_on(f) + } + + /// Blocks the thread on the specified future for **at most** `timeout` + /// + /// If the future completes before `timeout`, the result is returned. If + /// `timeout` elapses, then `Err` is returned. + pub(crate) fn block_on_timeout(&mut self, f: F, timeout: Duration) -> Result + where + F: std::future::Future, + { + use crate::runtime::park::CachedParkThread; + use std::task::Context; + use std::task::Poll::Ready; + use std::time::Instant; + + let mut park = CachedParkThread::new(); + let waker = park.waker().map_err(|_| ())?; + let mut cx = Context::from_waker(&waker); + + pin!(f); + let when = Instant::now() + timeout; + + loop { + if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) { + return Ok(v); + } + + let now = Instant::now(); + + if now >= when { + return Err(()); + } + + park.park_timeout(when - now); + } + } +} + +impl Drop for DisallowBlockInPlaceGuard { + fn drop(&mut self) { + if self.0 { + // XXX: Do we want some kind of assertion here, or is "best effort" okay? + CONTEXT.with(|c| { + if let EnterRuntime::Entered { + allow_block_in_place: false, + } = c.runtime.get() + { + c.runtime.set(EnterRuntime::Entered { + allow_block_in_place: true, + }); + } + }) + } + } +} diff --git a/third_party/rust/tokio/src/runtime/context/current.rs b/third_party/rust/tokio/src/runtime/context/current.rs new file mode 100644 index 000000000000..c3dc5c899421 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/context/current.rs @@ -0,0 +1,99 @@ +use super::{Context, CONTEXT}; + +use crate::runtime::{scheduler, TryCurrentError}; +use crate::util::markers::SyncNotSend; + +use std::cell::{Cell, RefCell}; +use std::marker::PhantomData; + +#[derive(Debug)] +#[must_use] +pub(crate) struct SetCurrentGuard { + // The previous handle + prev: Option, + + // The depth for this guard + depth: usize, + + // Don't let the type move across threads. + _p: PhantomData, +} + +pub(super) struct HandleCell { + /// Current handle + handle: RefCell>, + + /// Tracks the number of nested calls to `try_set_current`. + depth: Cell, +} + +/// Sets this [`Handle`] as the current active [`Handle`]. +/// +/// [`Handle`]: crate::runtime::scheduler::Handle +pub(crate) fn try_set_current(handle: &scheduler::Handle) -> Option { + CONTEXT.try_with(|ctx| ctx.set_current(handle)).ok() +} + +pub(crate) fn with_current(f: F) -> Result +where + F: FnOnce(&scheduler::Handle) -> R, +{ + match CONTEXT.try_with(|ctx| ctx.current.handle.borrow().as_ref().map(f)) { + Ok(Some(ret)) => Ok(ret), + Ok(None) => Err(TryCurrentError::new_no_context()), + Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()), + } +} + +impl Context { + pub(super) fn set_current(&self, handle: &scheduler::Handle) -> SetCurrentGuard { + let old_handle = self.current.handle.borrow_mut().replace(handle.clone()); + let depth = self.current.depth.get(); + + if depth == usize::MAX { + panic!("reached max `enter` depth"); + } + + let depth = depth + 1; + self.current.depth.set(depth); + + SetCurrentGuard { + prev: old_handle, + depth, + _p: PhantomData, + } + } +} + +impl HandleCell { + pub(super) const fn new() -> HandleCell { + HandleCell { + handle: RefCell::new(None), + depth: Cell::new(0), + } + } +} + +impl Drop for SetCurrentGuard { + fn drop(&mut self) { + CONTEXT.with(|ctx| { + let depth = ctx.current.depth.get(); + + if depth != self.depth { + if !std::thread::panicking() { + panic!( + "`EnterGuard` values dropped out of order. Guards returned by \ + `tokio::runtime::Handle::enter()` must be dropped in the reverse \ + order as they were acquired." + ); + } else { + // Just return... this will leave handles in a wonky state though... + return; + } + } + + *ctx.current.handle.borrow_mut() = self.prev.take(); + ctx.current.depth.set(depth - 1); + }); + } +} diff --git a/third_party/rust/tokio/src/runtime/context/runtime.rs b/third_party/rust/tokio/src/runtime/context/runtime.rs new file mode 100644 index 000000000000..f2e29899a445 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/context/runtime.rs @@ -0,0 +1,99 @@ +use super::{BlockingRegionGuard, SetCurrentGuard, CONTEXT}; + +use crate::runtime::scheduler; +use crate::util::rand::{FastRand, RngSeed}; + +use std::fmt; + +#[derive(Debug, Clone, Copy)] +#[must_use] +pub(crate) enum EnterRuntime { + /// Currently in a runtime context. + #[cfg_attr(not(feature = "rt"), allow(dead_code))] + Entered { allow_block_in_place: bool }, + + /// Not in a runtime context **or** a blocking region. + NotEntered, +} + +/// Guard tracking that a caller has entered a runtime context. +#[must_use] +pub(crate) struct EnterRuntimeGuard { + /// Tracks that the current thread has entered a blocking function call. + pub(crate) blocking: BlockingRegionGuard, + + #[allow(dead_code)] // Only tracking the guard. + pub(crate) handle: SetCurrentGuard, + + // Tracks the previous random number generator seed + old_seed: RngSeed, +} + +/// Marks the current thread as being within the dynamic extent of an +/// executor. +#[track_caller] +pub(crate) fn enter_runtime(handle: &scheduler::Handle, allow_block_in_place: bool, f: F) -> R +where + F: FnOnce(&mut BlockingRegionGuard) -> R, +{ + let maybe_guard = CONTEXT.with(|c| { + if c.runtime.get().is_entered() { + None + } else { + // Set the entered flag + c.runtime.set(EnterRuntime::Entered { + allow_block_in_place, + }); + + // Generate a new seed + let rng_seed = handle.seed_generator().next_seed(); + + // Swap the RNG seed + let mut rng = c.rng.get().unwrap_or_else(FastRand::new); + let old_seed = rng.replace_seed(rng_seed); + c.rng.set(Some(rng)); + + Some(EnterRuntimeGuard { + blocking: BlockingRegionGuard::new(), + handle: c.set_current(handle), + old_seed, + }) + } + }); + + if let Some(mut guard) = maybe_guard { + return f(&mut guard.blocking); + } + + panic!( + "Cannot start a runtime from within a runtime. This happens \ + because a function (like `block_on`) attempted to block the \ + current thread while the thread is being used to drive \ + asynchronous tasks." + ); +} + +impl fmt::Debug for EnterRuntimeGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Enter").finish() + } +} + +impl Drop for EnterRuntimeGuard { + fn drop(&mut self) { + CONTEXT.with(|c| { + assert!(c.runtime.get().is_entered()); + c.runtime.set(EnterRuntime::NotEntered); + // Replace the previous RNG seed + let mut rng = c.rng.get().unwrap_or_else(FastRand::new); + rng.replace_seed(self.old_seed.clone()); + c.rng.set(Some(rng)); + }); + } +} + +impl EnterRuntime { + pub(crate) fn is_entered(self) -> bool { + matches!(self, EnterRuntime::Entered { .. }) + } +} diff --git a/third_party/rust/tokio/src/runtime/context/runtime_mt.rs b/third_party/rust/tokio/src/runtime/context/runtime_mt.rs new file mode 100644 index 000000000000..728caeae99eb --- /dev/null +++ b/third_party/rust/tokio/src/runtime/context/runtime_mt.rs @@ -0,0 +1,36 @@ +use super::{EnterRuntime, CONTEXT}; + +/// Returns true if in a runtime context. +pub(crate) fn current_enter_context() -> EnterRuntime { + CONTEXT.with(|c| c.runtime.get()) +} + +/// Forces the current "entered" state to be cleared while the closure +/// is executed. +pub(crate) fn exit_runtime R, R>(f: F) -> R { + // Reset in case the closure panics + struct Reset(EnterRuntime); + + impl Drop for Reset { + fn drop(&mut self) { + CONTEXT.with(|c| { + assert!( + !c.runtime.get().is_entered(), + "closure claimed permanent executor" + ); + c.runtime.set(self.0); + }); + } + } + + let was = CONTEXT.with(|c| { + let e = c.runtime.get(); + assert!(e.is_entered(), "asked to exit when not entered"); + c.runtime.set(EnterRuntime::NotEntered); + e + }); + + let _reset = Reset(was); + // dropping _reset after f() will reset ENTERED + f() +} diff --git a/third_party/rust/tokio/src/runtime/context/scoped.rs b/third_party/rust/tokio/src/runtime/context/scoped.rs new file mode 100644 index 000000000000..7b202a16c075 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/context/scoped.rs @@ -0,0 +1,56 @@ +use std::cell::Cell; +use std::ptr; + +/// Scoped thread-local storage +pub(super) struct Scoped { + pub(super) inner: Cell<*const T>, +} + +impl Scoped { + pub(super) const fn new() -> Scoped { + Scoped { + inner: Cell::new(ptr::null()), + } + } + + /// Inserts a value into the scoped cell for the duration of the closure + pub(super) fn set(&self, t: &T, f: F) -> R + where + F: FnOnce() -> R, + { + struct Reset<'a, T> { + cell: &'a Cell<*const T>, + prev: *const T, + } + + impl Drop for Reset<'_, T> { + fn drop(&mut self) { + self.cell.set(self.prev); + } + } + + let prev = self.inner.get(); + self.inner.set(t as *const _); + + let _reset = Reset { + cell: &self.inner, + prev, + }; + + f() + } + + /// Gets the value out of the scoped cell; + pub(super) fn with(&self, f: F) -> R + where + F: FnOnce(Option<&T>) -> R, + { + let val = self.inner.get(); + + if val.is_null() { + f(None) + } else { + unsafe { f(Some(&*val)) } + } + } +} diff --git a/third_party/rust/tokio/src/coop.rs b/third_party/rust/tokio/src/runtime/coop.rs similarity index 78% rename from third_party/rust/tokio/src/coop.rs rename to third_party/rust/tokio/src/runtime/coop.rs index 96905319b656..2dba24615925 100644 --- a/third_party/rust/tokio/src/coop.rs +++ b/third_party/rust/tokio/src/runtime/coop.rs @@ -29,17 +29,18 @@ // other futures. By doing this, you avoid double-counting each iteration of // the outer future against the cooperating budget. -use std::cell::Cell; - -thread_local! { - static CURRENT: Cell = Cell::new(Budget::unconstrained()); -} +use crate::runtime::context; /// Opaque type tracking the amount of "work" a task may still do before /// yielding back to the scheduler. #[derive(Debug, Copy, Clone)] pub(crate) struct Budget(Option); +pub(crate) struct BudgetDecrement { + success: bool, + hit_zero: bool, +} + impl Budget { /// Budget assigned to a task on each poll. /// @@ -56,7 +57,7 @@ impl Budget { } /// Returns an unconstrained budget. Operations will not be limited. - const fn unconstrained() -> Budget { + pub(super) const fn unconstrained() -> Budget { Budget(None) } @@ -81,37 +82,42 @@ pub(crate) fn with_unconstrained(f: impl FnOnce() -> R) -> R { #[inline(always)] fn with_budget(budget: Budget, f: impl FnOnce() -> R) -> R { - struct ResetGuard<'a> { - cell: &'a Cell, + struct ResetGuard { prev: Budget, } - impl<'a> Drop for ResetGuard<'a> { + impl Drop for ResetGuard { fn drop(&mut self) { - self.cell.set(self.prev); + let _ = context::budget(|cell| { + cell.set(self.prev); + }); } } - CURRENT.with(move |cell| { + #[allow(unused_variables)] + let maybe_guard = context::budget(|cell| { let prev = cell.get(); - cell.set(budget); - let _guard = ResetGuard { cell, prev }; + ResetGuard { prev } + }); - f() - }) + // The function is called regardless even if the budget is not successfully + // set due to the thread-local being destroyed. + f() } #[inline(always)] pub(crate) fn has_budget_remaining() -> bool { - CURRENT.with(|cell| cell.get().has_remaining()) + // If the current budget cannot be accessed due to the thread-local being + // shutdown, then we assume there is budget remaining. + context::budget(|cell| cell.get().has_remaining()).unwrap_or(true) } cfg_rt_multi_thread! { /// Sets the current task's budget. pub(crate) fn set(budget: Budget) { - CURRENT.with(|cell| cell.set(budget)) + let _ = context::budget(|cell| cell.set(budget)); } } @@ -120,15 +126,16 @@ cfg_rt! { /// /// Returns the remaining budget pub(crate) fn stop() -> Budget { - CURRENT.with(|cell| { + context::budget(|cell| { let prev = cell.get(); cell.set(Budget::unconstrained()); prev - }) + }).unwrap_or(Budget::unconstrained()) } } cfg_coop! { + use std::cell::Cell; use std::task::{Context, Poll}; #[must_use] @@ -146,7 +153,7 @@ cfg_coop! { // They are both represented as the remembered budget being unconstrained. let budget = self.0.get(); if !budget.is_unconstrained() { - CURRENT.with(|cell| { + let _ = context::budget(|cell| { cell.set(budget); }); } @@ -167,33 +174,65 @@ cfg_coop! { /// that progress was made. #[inline] pub(crate) fn poll_proceed(cx: &mut Context<'_>) -> Poll { - CURRENT.with(|cell| { + context::budget(|cell| { let mut budget = cell.get(); - if budget.decrement() { + let decrement = budget.decrement(); + + if decrement.success { let restore = RestoreOnPending(Cell::new(cell.get())); cell.set(budget); + + // avoid double counting + if decrement.hit_zero { + inc_budget_forced_yield_count(); + } + Poll::Ready(restore) } else { cx.waker().wake_by_ref(); Poll::Pending } - }) + }).unwrap_or(Poll::Ready(RestoreOnPending(Cell::new(Budget::unconstrained())))) + } + + cfg_rt! { + cfg_metrics! { + #[inline(always)] + fn inc_budget_forced_yield_count() { + let _ = context::with_current(|handle| { + handle.scheduler_metrics().inc_budget_forced_yield_count(); + }); + } + } + + cfg_not_metrics! { + #[inline(always)] + fn inc_budget_forced_yield_count() {} + } + } + + cfg_not_rt! { + #[inline(always)] + fn inc_budget_forced_yield_count() {} } impl Budget { /// Decrements the budget. Returns `true` if successful. Decrementing fails /// when there is not enough remaining budget. - fn decrement(&mut self) -> bool { + fn decrement(&mut self) -> BudgetDecrement { if let Some(num) = &mut self.0 { if *num > 0 { *num -= 1; - true + + let hit_zero = *num == 0; + + BudgetDecrement { success: true, hit_zero } } else { - false + BudgetDecrement { success: false, hit_zero: false } } } else { - true + BudgetDecrement { success: true, hit_zero: false } } } @@ -207,15 +246,15 @@ cfg_coop! { mod test { use super::*; - #[cfg(target_arch = "wasm32")] + #[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; fn get() -> Budget { - CURRENT.with(|cell| cell.get()) + context::budget(|cell| cell.get()).unwrap_or(Budget::unconstrained()) } #[test] - fn bugeting() { + fn budgeting() { use futures::future::poll_fn; use tokio_test::*; diff --git a/third_party/rust/tokio/src/runtime/driver.rs b/third_party/rust/tokio/src/runtime/driver.rs index 7e459779bbd0..572fdefb0dac 100644 --- a/third_party/rust/tokio/src/runtime/driver.rs +++ b/third_party/rust/tokio/src/runtime/driver.rs @@ -1,63 +1,233 @@ //! Abstracts out the entire chain of runtime sub-drivers into common types. -use crate::park::thread::ParkThread; -use crate::park::Park; + +// Eventually, this file will see significant refactoring / cleanup. For now, we +// don't need to worry much about dead code with certain feature permutations. +#![cfg_attr(not(feature = "full"), allow(dead_code))] + +use crate::runtime::park::{ParkThread, UnparkThread}; use std::io; use std::time::Duration; +#[derive(Debug)] +pub(crate) struct Driver { + inner: TimeDriver, +} + +#[derive(Debug)] +pub(crate) struct Handle { + /// IO driver handle + pub(crate) io: IoHandle, + + /// Signal driver handle + #[cfg_attr(any(not(unix), loom), allow(dead_code))] + pub(crate) signal: SignalHandle, + + /// Time driver handle + pub(crate) time: TimeHandle, + + /// Source of `Instant::now()` + #[cfg_attr(not(all(feature = "time", feature = "test-util")), allow(dead_code))] + pub(crate) clock: Clock, +} + +pub(crate) struct Cfg { + pub(crate) enable_io: bool, + pub(crate) enable_time: bool, + pub(crate) enable_pause_time: bool, + pub(crate) start_paused: bool, + pub(crate) nevents: usize, +} + +impl Driver { + pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Handle)> { + let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io, cfg.nevents)?; + + let clock = create_clock(cfg.enable_pause_time, cfg.start_paused); + + let (time_driver, time_handle) = create_time_driver(cfg.enable_time, io_stack, &clock); + + Ok(( + Self { inner: time_driver }, + Handle { + io: io_handle, + signal: signal_handle, + time: time_handle, + clock, + }, + )) + } + + pub(crate) fn park(&mut self, handle: &Handle) { + self.inner.park(handle) + } + + pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) { + self.inner.park_timeout(handle, duration) + } + + pub(crate) fn shutdown(&mut self, handle: &Handle) { + self.inner.shutdown(handle) + } +} + +impl Handle { + pub(crate) fn unpark(&self) { + #[cfg(feature = "time")] + if let Some(handle) = &self.time { + handle.unpark(); + } + + self.io.unpark(); + } + + cfg_io_driver! { + #[track_caller] + pub(crate) fn io(&self) -> &crate::runtime::io::Handle { + self.io + .as_ref() + .expect("A Tokio 1.x context was found, but IO is disabled. Call `enable_io` on the runtime builder to enable IO.") + } + } + + cfg_signal_internal_and_unix! { + #[track_caller] + pub(crate) fn signal(&self) -> &crate::runtime::signal::Handle { + self.signal + .as_ref() + .expect("there is no signal driver running, must be called from the context of Tokio runtime") + } + } + + cfg_time! { + /// Returns a reference to the time driver handle. + /// + /// Panics if no time driver is present. + #[track_caller] + pub(crate) fn time(&self) -> &crate::runtime::time::Handle { + self.time + .as_ref() + .expect("A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers.") + } + + pub(crate) fn clock(&self) -> &Clock { + &self.clock + } + } +} + // ===== io driver ===== cfg_io_driver! { - type IoDriver = crate::io::driver::Driver; - type IoStack = crate::park::either::Either; - pub(crate) type IoHandle = Option; + pub(crate) type IoDriver = crate::runtime::io::Driver; - fn create_io_stack(enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> { - use crate::park::either::Either; + #[derive(Debug)] + pub(crate) enum IoStack { + Enabled(ProcessDriver), + Disabled(ParkThread), + } + #[derive(Debug)] + pub(crate) enum IoHandle { + Enabled(crate::runtime::io::Handle), + Disabled(UnparkThread), + } + + fn create_io_stack(enabled: bool, nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> { #[cfg(loom)] assert!(!enabled); let ret = if enabled { - let io_driver = crate::io::driver::Driver::new()?; - let io_handle = io_driver.handle(); + let (io_driver, io_handle) = crate::runtime::io::Driver::new(nevents)?; - let (signal_driver, signal_handle) = create_signal_driver(io_driver)?; + let (signal_driver, signal_handle) = create_signal_driver(io_driver, &io_handle)?; let process_driver = create_process_driver(signal_driver); - (Either::A(process_driver), Some(io_handle), signal_handle) + (IoStack::Enabled(process_driver), IoHandle::Enabled(io_handle), signal_handle) } else { - (Either::B(ParkThread::new()), Default::default(), Default::default()) + let park_thread = ParkThread::new(); + let unpark_thread = park_thread.unpark(); + (IoStack::Disabled(park_thread), IoHandle::Disabled(unpark_thread), Default::default()) }; Ok(ret) } + + impl IoStack { + pub(crate) fn park(&mut self, handle: &Handle) { + match self { + IoStack::Enabled(v) => v.park(handle), + IoStack::Disabled(v) => v.park(), + } + } + + pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) { + match self { + IoStack::Enabled(v) => v.park_timeout(handle, duration), + IoStack::Disabled(v) => v.park_timeout(duration), + } + } + + pub(crate) fn shutdown(&mut self, handle: &Handle) { + match self { + IoStack::Enabled(v) => v.shutdown(handle), + IoStack::Disabled(v) => v.shutdown(), + } + } + } + + impl IoHandle { + pub(crate) fn unpark(&self) { + match self { + IoHandle::Enabled(handle) => handle.unpark(), + IoHandle::Disabled(handle) => handle.unpark(), + } + } + + pub(crate) fn as_ref(&self) -> Option<&crate::runtime::io::Handle> { + match self { + IoHandle::Enabled(v) => Some(v), + IoHandle::Disabled(..) => None, + } + } + } } cfg_not_io_driver! { - pub(crate) type IoHandle = (); - type IoStack = ParkThread; + pub(crate) type IoHandle = UnparkThread; - fn create_io_stack(_enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> { - Ok((ParkThread::new(), Default::default(), Default::default())) + #[derive(Debug)] + pub(crate) struct IoStack(ParkThread); + + fn create_io_stack(_enabled: bool, _nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> { + let park_thread = ParkThread::new(); + let unpark_thread = park_thread.unpark(); + Ok((IoStack(park_thread), unpark_thread, Default::default())) + } + + impl IoStack { + pub(crate) fn park(&mut self, _handle: &Handle) { + self.0.park(); + } + + pub(crate) fn park_timeout(&mut self, _handle: &Handle, duration: Duration) { + self.0.park_timeout(duration); + } + + pub(crate) fn shutdown(&mut self, _handle: &Handle) { + self.0.shutdown(); + } } } // ===== signal driver ===== -macro_rules! cfg_signal_internal_and_unix { - ($($item:item)*) => { - #[cfg(unix)] - cfg_signal_internal! { $($item)* } - } -} - cfg_signal_internal_and_unix! { - type SignalDriver = crate::signal::unix::driver::Driver; - pub(crate) type SignalHandle = Option; + type SignalDriver = crate::runtime::signal::Driver; + pub(crate) type SignalHandle = Option; - fn create_signal_driver(io_driver: IoDriver) -> io::Result<(SignalDriver, SignalHandle)> { - let driver = crate::signal::unix::driver::Driver::new(io_driver)?; + fn create_signal_driver(io_driver: IoDriver, io_handle: &crate::runtime::io::Handle) -> io::Result<(SignalDriver, SignalHandle)> { + let driver = crate::runtime::signal::Driver::new(io_driver, io_handle)?; let handle = driver.handle(); Ok((driver, Some(handle))) } @@ -69,7 +239,7 @@ cfg_not_signal_internal! { cfg_io_driver! { type SignalDriver = IoDriver; - fn create_signal_driver(io_driver: IoDriver) -> io::Result<(SignalDriver, SignalHandle)> { + fn create_signal_driver(io_driver: IoDriver, _io_handle: &crate::runtime::io::Handle) -> io::Result<(SignalDriver, SignalHandle)> { Ok((io_driver, ())) } } @@ -78,10 +248,10 @@ cfg_not_signal_internal! { // ===== process driver ===== cfg_process_driver! { - type ProcessDriver = crate::process::unix::driver::Driver; + type ProcessDriver = crate::runtime::process::Driver; fn create_process_driver(signal_driver: SignalDriver) -> ProcessDriver { - crate::process::unix::driver::Driver::new(signal_driver) + ProcessDriver::new(signal_driver) } } @@ -98,10 +268,16 @@ cfg_not_process_driver! { // ===== time driver ===== cfg_time! { - type TimeDriver = crate::park::either::Either, IoStack>; + #[derive(Debug)] + pub(crate) enum TimeDriver { + Enabled { + driver: crate::runtime::time::Driver, + }, + Disabled(IoStack), + } pub(crate) type Clock = crate::time::Clock; - pub(crate) type TimeHandle = Option; + pub(crate) type TimeHandle = Option; fn create_clock(enable_pausing: bool, start_paused: bool) -> Clock { crate::time::Clock::new(enable_pausing, start_paused) @@ -110,17 +286,37 @@ cfg_time! { fn create_time_driver( enable: bool, io_stack: IoStack, - clock: Clock, + clock: &Clock, ) -> (TimeDriver, TimeHandle) { - use crate::park::either::Either; - if enable { - let driver = crate::time::driver::Driver::new(io_stack, clock); - let handle = driver.handle(); + let (driver, handle) = crate::runtime::time::Driver::new(io_stack, clock); - (Either::A(driver), Some(handle)) + (TimeDriver::Enabled { driver }, Some(handle)) } else { - (Either::B(io_stack), None) + (TimeDriver::Disabled(io_stack), None) + } + } + + impl TimeDriver { + pub(crate) fn park(&mut self, handle: &Handle) { + match self { + TimeDriver::Enabled { driver, .. } => driver.park(handle), + TimeDriver::Disabled(v) => v.park(handle), + } + } + + pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) { + match self { + TimeDriver::Enabled { driver } => driver.park_timeout(handle, duration), + TimeDriver::Disabled(v) => v.park_timeout(handle, duration), + } + } + + pub(crate) fn shutdown(&mut self, handle: &Handle) { + match self { + TimeDriver::Enabled { driver } => driver.shutdown(handle), + TimeDriver::Disabled(v) => v.shutdown(handle), + } } } } @@ -138,71 +334,8 @@ cfg_not_time! { fn create_time_driver( _enable: bool, io_stack: IoStack, - _clock: Clock, + _clock: &Clock, ) -> (TimeDriver, TimeHandle) { (io_stack, ()) } } - -// ===== runtime driver ===== - -#[derive(Debug)] -pub(crate) struct Driver { - inner: TimeDriver, -} - -pub(crate) struct Resources { - pub(crate) io_handle: IoHandle, - pub(crate) signal_handle: SignalHandle, - pub(crate) time_handle: TimeHandle, - pub(crate) clock: Clock, -} - -pub(crate) struct Cfg { - pub(crate) enable_io: bool, - pub(crate) enable_time: bool, - pub(crate) enable_pause_time: bool, - pub(crate) start_paused: bool, -} - -impl Driver { - pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Resources)> { - let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io)?; - - let clock = create_clock(cfg.enable_pause_time, cfg.start_paused); - - let (time_driver, time_handle) = - create_time_driver(cfg.enable_time, io_stack, clock.clone()); - - Ok(( - Self { inner: time_driver }, - Resources { - io_handle, - signal_handle, - time_handle, - clock, - }, - )) - } -} - -impl Park for Driver { - type Unpark = ::Unpark; - type Error = ::Error; - - fn unpark(&self) -> Self::Unpark { - self.inner.unpark() - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.inner.park() - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.inner.park_timeout(duration) - } - - fn shutdown(&mut self) { - self.inner.shutdown() - } -} diff --git a/third_party/rust/tokio/src/runtime/dump.rs b/third_party/rust/tokio/src/runtime/dump.rs new file mode 100644 index 000000000000..994b7f9c0153 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/dump.rs @@ -0,0 +1,76 @@ +//! Snapshots of runtime state. +//! +//! See [Handle::dump][crate::runtime::Handle::dump]. + +use std::fmt; + +/// A snapshot of a runtime's state. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. +#[derive(Debug)] +pub struct Dump { + tasks: Tasks, +} + +/// Snapshots of tasks. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. +#[derive(Debug)] +pub struct Tasks { + tasks: Vec, +} + +/// A snapshot of a task. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. +#[derive(Debug)] +pub struct Task { + trace: Trace, +} + +/// An execution trace of a task's last poll. +/// +/// See [Handle::dump][crate::runtime::Handle::dump]. +#[derive(Debug)] +pub struct Trace { + inner: super::task::trace::Trace, +} + +impl Dump { + pub(crate) fn new(tasks: Vec) -> Self { + Self { + tasks: Tasks { tasks }, + } + } + + /// Tasks in this snapshot. + pub fn tasks(&self) -> &Tasks { + &self.tasks + } +} + +impl Tasks { + /// Iterate over tasks. + pub fn iter(&self) -> impl Iterator { + self.tasks.iter() + } +} + +impl Task { + pub(crate) fn new(trace: super::task::trace::Trace) -> Self { + Self { + trace: Trace { inner: trace }, + } + } + + /// A trace of this task's state. + pub fn trace(&self) -> &Trace { + &self.trace + } +} + +impl fmt::Display for Trace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(f) + } +} diff --git a/third_party/rust/tokio/src/runtime/enter.rs b/third_party/rust/tokio/src/runtime/enter.rs deleted file mode 100644 index 3f14cb58782f..000000000000 --- a/third_party/rust/tokio/src/runtime/enter.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::cell::{Cell, RefCell}; -use std::fmt; -use std::marker::PhantomData; - -#[derive(Debug, Clone, Copy)] -pub(crate) enum EnterContext { - #[cfg_attr(not(feature = "rt"), allow(dead_code))] - Entered { - allow_blocking: bool, - }, - NotEntered, -} - -impl EnterContext { - pub(crate) fn is_entered(self) -> bool { - matches!(self, EnterContext::Entered { .. }) - } -} - -thread_local!(static ENTERED: Cell = Cell::new(EnterContext::NotEntered)); - -/// Represents an executor context. -pub(crate) struct Enter { - _p: PhantomData>, -} - -cfg_rt! { - use crate::park::thread::ParkError; - - use std::time::Duration; - - /// Marks the current thread as being within the dynamic extent of an - /// executor. - pub(crate) fn enter(allow_blocking: bool) -> Enter { - if let Some(enter) = try_enter(allow_blocking) { - return enter; - } - - panic!( - "Cannot start a runtime from within a runtime. This happens \ - because a function (like `block_on`) attempted to block the \ - current thread while the thread is being used to drive \ - asynchronous tasks." - ); - } - - /// Tries to enter a runtime context, returns `None` if already in a runtime - /// context. - pub(crate) fn try_enter(allow_blocking: bool) -> Option { - ENTERED.with(|c| { - if c.get().is_entered() { - None - } else { - c.set(EnterContext::Entered { allow_blocking }); - Some(Enter { _p: PhantomData }) - } - }) - } -} - -// Forces the current "entered" state to be cleared while the closure -// is executed. -// -// # Warning -// -// This is hidden for a reason. Do not use without fully understanding -// executors. Misusing can easily cause your program to deadlock. -cfg_rt_multi_thread! { - pub(crate) fn exit R, R>(f: F) -> R { - // Reset in case the closure panics - struct Reset(EnterContext); - impl Drop for Reset { - fn drop(&mut self) { - ENTERED.with(|c| { - assert!(!c.get().is_entered(), "closure claimed permanent executor"); - c.set(self.0); - }); - } - } - - let was = ENTERED.with(|c| { - let e = c.get(); - assert!(e.is_entered(), "asked to exit when not entered"); - c.set(EnterContext::NotEntered); - e - }); - - let _reset = Reset(was); - // dropping _reset after f() will reset ENTERED - f() - } -} - -cfg_rt! { - /// Disallows blocking in the current runtime context until the guard is dropped. - pub(crate) fn disallow_blocking() -> DisallowBlockingGuard { - let reset = ENTERED.with(|c| { - if let EnterContext::Entered { - allow_blocking: true, - } = c.get() - { - c.set(EnterContext::Entered { - allow_blocking: false, - }); - true - } else { - false - } - }); - DisallowBlockingGuard(reset) - } - - pub(crate) struct DisallowBlockingGuard(bool); - impl Drop for DisallowBlockingGuard { - fn drop(&mut self) { - if self.0 { - // XXX: Do we want some kind of assertion here, or is "best effort" okay? - ENTERED.with(|c| { - if let EnterContext::Entered { - allow_blocking: false, - } = c.get() - { - c.set(EnterContext::Entered { - allow_blocking: true, - }); - } - }) - } - } - } -} - -cfg_rt_multi_thread! { - /// Returns true if in a runtime context. - pub(crate) fn context() -> EnterContext { - ENTERED.with(|c| c.get()) - } -} - -cfg_rt! { - impl Enter { - /// Blocks the thread on the specified future, returning the value with - /// which that future completes. - pub(crate) fn block_on(&mut self, f: F) -> Result - where - F: std::future::Future, - { - use crate::park::thread::CachedParkThread; - - let mut park = CachedParkThread::new(); - park.block_on(f) - } - - /// Blocks the thread on the specified future for **at most** `timeout` - /// - /// If the future completes before `timeout`, the result is returned. If - /// `timeout` elapses, then `Err` is returned. - pub(crate) fn block_on_timeout(&mut self, f: F, timeout: Duration) -> Result - where - F: std::future::Future, - { - use crate::park::Park; - use crate::park::thread::CachedParkThread; - use std::task::Context; - use std::task::Poll::Ready; - use std::time::Instant; - - let mut park = CachedParkThread::new(); - let waker = park.get_unpark()?.into_waker(); - let mut cx = Context::from_waker(&waker); - - pin!(f); - let when = Instant::now() + timeout; - - loop { - if let Ready(v) = crate::coop::budget(|| f.as_mut().poll(&mut cx)) { - return Ok(v); - } - - let now = Instant::now(); - - if now >= when { - return Err(()); - } - - park.park_timeout(when - now)?; - } - } - } -} - -impl fmt::Debug for Enter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Enter").finish() - } -} - -impl Drop for Enter { - fn drop(&mut self) { - ENTERED.with(|c| { - assert!(c.get().is_entered()); - c.set(EnterContext::NotEntered); - }); - } -} diff --git a/third_party/rust/tokio/src/runtime/handle.rs b/third_party/rust/tokio/src/runtime/handle.rs index 9dbe6774dd04..be4743d4775e 100644 --- a/third_party/rust/tokio/src/runtime/handle.rs +++ b/third_party/rust/tokio/src/runtime/handle.rs @@ -1,11 +1,4 @@ -use crate::runtime::blocking::{BlockingTask, NoopSchedule}; -use crate::runtime::task::{self, JoinHandle}; -use crate::runtime::{blocking, context, driver, Spawner}; -use crate::util::error::{CONTEXT_MISSING_ERROR, THREAD_LOCAL_DESTROYED_ERROR}; - -use std::future::Future; -use std::marker::PhantomData; -use std::{error, fmt}; +use crate::runtime::{context, scheduler, RuntimeFlavor}; /// Handle to the runtime. /// @@ -14,39 +7,19 @@ use std::{error, fmt}; /// /// [`Runtime::handle`]: crate::runtime::Runtime::handle() #[derive(Debug, Clone)] +// When the `rt` feature is *not* enabled, this type is still defined, but not +// included in the public API. pub struct Handle { - pub(super) spawner: Spawner, - - /// Handles to the I/O drivers - #[cfg_attr( - not(any(feature = "net", feature = "process", all(unix, feature = "signal"))), - allow(dead_code) - )] - pub(super) io_handle: driver::IoHandle, - - /// Handles to the signal drivers - #[cfg_attr( - any( - loom, - not(all(unix, feature = "signal")), - not(all(unix, feature = "process")), - ), - allow(dead_code) - )] - pub(super) signal_handle: driver::SignalHandle, - - /// Handles to the time drivers - #[cfg_attr(not(feature = "time"), allow(dead_code))] - pub(super) time_handle: driver::TimeHandle, - - /// Source of `Instant::now()` - #[cfg_attr(not(all(feature = "time", feature = "test-util")), allow(dead_code))] - pub(super) clock: driver::Clock, - - /// Blocking pool spawner - pub(super) blocking_spawner: blocking::Spawner, + pub(crate) inner: scheduler::Handle, } +use crate::runtime::task::JoinHandle; +use crate::util::error::{CONTEXT_MISSING_ERROR, THREAD_LOCAL_DESTROYED_ERROR}; + +use std::future::Future; +use std::marker::PhantomData; +use std::{error, fmt}; + /// Runtime context guard. /// /// Returned by [`Runtime::enter`] and [`Handle::enter`], the context guard exits @@ -56,32 +29,72 @@ pub struct Handle { #[derive(Debug)] #[must_use = "Creating and dropping a guard does nothing"] pub struct EnterGuard<'a> { - _guard: context::EnterGuard, + _guard: context::SetCurrentGuard, _handle_lifetime: PhantomData<&'a Handle>, } impl Handle { /// Enters the runtime context. This allows you to construct types that must - /// have an executor available on creation such as [`Sleep`] or [`TcpStream`]. - /// It will also allow you to call methods such as [`tokio::spawn`]. + /// have an executor available on creation such as [`Sleep`] or + /// [`TcpStream`]. It will also allow you to call methods such as + /// [`tokio::spawn`] and [`Handle::current`] without panicking. + /// + /// # Panics + /// + /// When calling `Handle::enter` multiple times, the returned guards + /// **must** be dropped in the reverse order that they were acquired. + /// Failure to do so will result in a panic and possible memory leaks. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// let rt = Runtime::new().unwrap(); + /// + /// let _guard = rt.enter(); + /// tokio::spawn(async { + /// println!("Hello world!"); + /// }); + /// ``` + /// + /// Do **not** do the following, this shows a scenario that will result in a + /// panic and possible memory leak. + /// + /// ```should_panic + /// use tokio::runtime::Runtime; + /// + /// let rt1 = Runtime::new().unwrap(); + /// let rt2 = Runtime::new().unwrap(); + /// + /// let enter1 = rt1.enter(); + /// let enter2 = rt2.enter(); + /// + /// drop(enter1); + /// drop(enter2); + /// ``` /// /// [`Sleep`]: struct@crate::time::Sleep /// [`TcpStream`]: struct@crate::net::TcpStream /// [`tokio::spawn`]: fn@crate::spawn pub fn enter(&self) -> EnterGuard<'_> { EnterGuard { - _guard: context::enter(self.clone()), + _guard: match context::try_set_current(&self.inner) { + Some(guard) => guard, + None => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + }, _handle_lifetime: PhantomData, } } /// Returns a `Handle` view over the currently running `Runtime`. /// - /// # Panic + /// # Panics /// /// This will panic if called outside the context of a Tokio runtime. That means that you must - /// call this on one of the threads **being run by the runtime**. Calling this from within a - /// thread created by `std::thread::spawn` (for example) will cause a panic. + /// call this on one of the threads **being run by the runtime**, or from a thread with an active + /// `EnterGuard`. Calling this from within a thread created by `std::thread::spawn` (for example) + /// will cause a panic unless that thread has an active `EnterGuard`. /// /// # Examples /// @@ -105,16 +118,24 @@ impl Handle { /// # let handle = /// thread::spawn(move || { /// // Notice that the handle is created outside of this thread and then moved in - /// handle.spawn(async { /* ... */ }) - /// // This next line would cause a panic - /// // let handle2 = Handle::current(); + /// handle.spawn(async { /* ... */ }); + /// // This next line would cause a panic because we haven't entered the runtime + /// // and created an EnterGuard + /// // let handle2 = Handle::current(); // panic + /// // So we create a guard here with Handle::enter(); + /// let _guard = handle.enter(); + /// // Now we can call Handle::current(); + /// let handle2 = Handle::current(); /// }); /// # handle.join().unwrap(); /// # }); /// # } /// ``` + #[track_caller] pub fn current() -> Self { - context::current() + Handle { + inner: scheduler::Handle::current(), + } } /// Returns a Handle view over the currently running Runtime @@ -123,7 +144,9 @@ impl Handle { /// /// Contrary to `current`, this never panics pub fn try_current() -> Result { - context::try_current() + context::with_current(|inner| Handle { + inner: inner.clone(), + }) } /// Spawns a future onto the Tokio runtime. @@ -132,6 +155,10 @@ impl Handle { /// thread pool. The thread pool is then responsible for polling the future /// until it completes. /// + /// The provided future will start running in the background immediately + /// when `spawn` is called, even if you don't await the returned + /// `JoinHandle`. + /// /// See [module level][mod] documentation for more details. /// /// [mod]: index.html @@ -159,12 +186,10 @@ impl Handle { F: Future + Send + 'static, F::Output: Send + 'static, { - #[cfg(all(tokio_unstable, feature = "tracing"))] - let future = crate::util::trace::task(future, "task", None); - self.spawner.spawn(future) + self.spawn_named(future, None) } - /// Runs the provided function on an executor dedicated to blocking. + /// Runs the provided function on an executor dedicated to blocking /// operations. /// /// # Examples @@ -189,85 +214,7 @@ impl Handle { F: FnOnce() -> R + Send + 'static, R: Send + 'static, { - let (join_handle, _was_spawned) = - if cfg!(debug_assertions) && std::mem::size_of::() > 2048 { - self.spawn_blocking_inner(Box::new(func), blocking::Mandatory::NonMandatory, None) - } else { - self.spawn_blocking_inner(func, blocking::Mandatory::NonMandatory, None) - }; - - join_handle - } - - cfg_fs! { - #[track_caller] - #[cfg_attr(any( - all(loom, not(test)), // the function is covered by loom tests - test - ), allow(dead_code))] - pub(crate) fn spawn_mandatory_blocking(&self, func: F) -> Option> - where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, - { - let (join_handle, was_spawned) = if cfg!(debug_assertions) && std::mem::size_of::() > 2048 { - self.spawn_blocking_inner( - Box::new(func), - blocking::Mandatory::Mandatory, - None - ) - } else { - self.spawn_blocking_inner( - func, - blocking::Mandatory::Mandatory, - None - ) - }; - - if was_spawned { - Some(join_handle) - } else { - None - } - } - } - - #[track_caller] - pub(crate) fn spawn_blocking_inner( - &self, - func: F, - is_mandatory: blocking::Mandatory, - name: Option<&str>, - ) -> (JoinHandle, bool) - where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, - { - let fut = BlockingTask::new(func); - - #[cfg(all(tokio_unstable, feature = "tracing"))] - let fut = { - use tracing::Instrument; - let location = std::panic::Location::caller(); - let span = tracing::trace_span!( - target: "tokio::task::blocking", - "runtime.spawn", - kind = %"blocking", - task.name = %name.unwrap_or_default(), - "fn" = %std::any::type_name::(), - spawn.location = %format_args!("{}:{}:{}", location.file(), location.line(), location.column()), - ); - fut.instrument(span) - }; - - #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let _ = name; - - let (task, handle) = task::unowned(fut, NoopSchedule); - let spawned = self - .blocking_spawner - .spawn(blocking::Task::new(task, is_mandatory), self); - (handle, spawned.is_ok()) + self.inner.blocking_spawner().spawn_blocking(self, func) } /// Runs a future to completion on this `Handle`'s associated `Runtime`. @@ -342,23 +289,73 @@ impl Handle { /// [`tokio::time`]: crate::time #[track_caller] pub fn block_on(&self, future: F) -> F::Output { + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + ))] + let future = super::task::trace::Trace::root(future); + #[cfg(all(tokio_unstable, feature = "tracing"))] - let future = crate::util::trace::task(future, "block_on", None); + let future = + crate::util::trace::task(future, "block_on", None, super::task::Id::next().as_u64()); - // Enter the **runtime** context. This configures spawning, the current I/O driver, ... - let _rt_enter = self.enter(); - - // Enter a **blocking** context. This prevents blocking from a runtime. - let mut blocking_enter = crate::runtime::enter(true); - - // Block on the future - blocking_enter - .block_on(future) - .expect("failed to park thread") + // Enter the runtime context. This sets the current driver handles and + // prevents blocking an existing runtime. + context::enter_runtime(&self.inner, true, |blocking| { + blocking.block_on(future).expect("failed to park thread") + }) } - pub(crate) fn shutdown(mut self) { - self.spawner.shutdown(); + #[track_caller] + pub(crate) fn spawn_named(&self, future: F, _name: Option<&str>) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + let id = crate::runtime::task::Id::next(); + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + ))] + let future = super::task::trace::Trace::root(future); + #[cfg(all(tokio_unstable, feature = "tracing"))] + let future = crate::util::trace::task(future, "task", _name, id.as_u64()); + self.inner.spawn(future, id) + } + + /// Returns the flavor of the current `Runtime`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{Handle, RuntimeFlavor}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() { + /// assert_eq!(RuntimeFlavor::CurrentThread, Handle::current().runtime_flavor()); + /// } + /// ``` + /// + /// ``` + /// use tokio::runtime::{Handle, RuntimeFlavor}; + /// + /// #[tokio::main(flavor = "multi_thread", worker_threads = 4)] + /// async fn main() { + /// assert_eq!(RuntimeFlavor::MultiThread, Handle::current().runtime_flavor()); + /// } + /// ``` + pub fn runtime_flavor(&self) -> RuntimeFlavor { + match self.inner { + scheduler::Handle::CurrentThread(_) => RuntimeFlavor::CurrentThread, + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Handle::MultiThread(_) => RuntimeFlavor::MultiThread, + } } } @@ -374,6 +371,161 @@ cfg_metrics! { } } +cfg_taskdump! { + impl Handle { + /// Captures a snapshot of the runtime's state. + /// + /// This functionality is experimental, and comes with a number of + /// requirements and limitations. + /// + /// # Examples + /// + /// This can be used to get call traces of each task in the runtime. + /// Calls to `Handle::dump` should usually be enclosed in a + /// [timeout][crate::time::timeout], so that dumping does not escalate a + /// single blocked runtime thread into an entirely blocked runtime. + /// + /// ``` + /// # use tokio::runtime::Runtime; + /// # fn dox() { + /// # let rt = Runtime::new().unwrap(); + /// # rt.spawn(async { + /// use tokio::runtime::Handle; + /// use tokio::time::{timeout, Duration}; + /// + /// // Inside an async block or function. + /// let handle = Handle::current(); + /// if let Ok(dump) = timeout(Duration::from_secs(2), handle.dump()).await { + /// for (i, task) in dump.tasks().iter().enumerate() { + /// let trace = task.trace(); + /// println!("TASK {i}:"); + /// println!("{trace}\n"); + /// } + /// } + /// # }); + /// # } + /// ``` + /// + /// This produces highly detailed traces of tasks; e.g.: + /// + /// ```plain + /// TASK 0: + /// ╼ dump::main::{{closure}}::a::{{closure}} at /tokio/examples/dump.rs:18:20 + /// └╼ dump::main::{{closure}}::b::{{closure}} at /tokio/examples/dump.rs:23:20 + /// └╼ dump::main::{{closure}}::c::{{closure}} at /tokio/examples/dump.rs:28:24 + /// └╼ tokio::sync::barrier::Barrier::wait::{{closure}} at /tokio/tokio/src/sync/barrier.rs:129:10 + /// └╼ as core::future::future::Future>::poll at /tokio/tokio/src/util/trace.rs:77:46 + /// └╼ tokio::sync::barrier::Barrier::wait_internal::{{closure}} at /tokio/tokio/src/sync/barrier.rs:183:36 + /// └╼ tokio::sync::watch::Receiver::changed::{{closure}} at /tokio/tokio/src/sync/watch.rs:604:55 + /// └╼ tokio::sync::watch::changed_impl::{{closure}} at /tokio/tokio/src/sync/watch.rs:755:18 + /// └╼ ::poll at /tokio/tokio/src/sync/notify.rs:1103:9 + /// └╼ tokio::sync::notify::Notified::poll_notified at /tokio/tokio/src/sync/notify.rs:996:32 + /// ``` + /// + /// # Requirements + /// + /// ## Debug Info Must Be Available + /// + /// To produce task traces, the application must **not** be compiled + /// with split debuginfo. On Linux, including debuginfo within the + /// application binary is the (correct) default. You can further ensure + /// this behavior with the following directive in your `Cargo.toml`: + /// + /// ```toml + /// [profile.*] + /// split-debuginfo = "off" + /// ``` + /// + /// ## Unstable Features + /// + /// This functionality is **unstable**, and requires both the + /// `tokio_unstable` and `tokio_taskdump` cfg flags to be set. + /// + /// You can do this by setting the `RUSTFLAGS` environment variable + /// before invoking `cargo`; e.g.: + /// ```bash + /// RUSTFLAGS="--cfg tokio_unstable --cfg tokio_taskdump" cargo run --example dump + /// ``` + /// + /// Or by [configuring][cargo-config] `rustflags` in + /// `.cargo/config.toml`: + /// ```text + /// [build] + /// rustflags = ["--cfg tokio_unstable", "--cfg tokio_taskdump"] + /// ``` + /// + /// [cargo-config]: + /// https://doc.rust-lang.org/cargo/reference/config.html + /// + /// ## Platform Requirements + /// + /// Task dumps are supported on Linux atop aarch64, x86 and x86_64. + /// + /// ## Current Thread Runtime Requirements + /// + /// On the `current_thread` runtime, task dumps may only be requested + /// from *within* the context of the runtime being dumped. Do not, for + /// example, await `Handle::dump()` on a different runtime. + /// + /// # Limitations + /// + /// ## Performance + /// + /// Although enabling the `tokio_taskdump` feature imposes virtually no + /// additional runtime overhead, actually calling `Handle::dump` is + /// expensive. The runtime must synchronize and pause its workers, then + /// re-poll every task in a special tracing mode. Avoid requesting dumps + /// often. + /// + /// ## Local Executors + /// + /// Tasks managed by local executors (e.g., `FuturesUnordered` and + /// [`LocalSet`][crate::task::LocalSet]) may not appear in task dumps. + /// + /// ## Non-Termination When Workers Are Blocked + /// + /// The future produced by `Handle::dump` may never produce `Ready` if + /// another runtime worker is blocked for more than 250ms. This may + /// occur if a dump is requested during shutdown, or if another runtime + /// worker is infinite looping or synchronously deadlocked. For these + /// reasons, task dumping should usually be paired with an explicit + /// [timeout][crate::time::timeout]. + pub async fn dump(&self) -> crate::runtime::Dump { + match &self.inner { + scheduler::Handle::CurrentThread(handle) => handle.dump(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Handle::MultiThread(handle) => { + // perform the trace in a separate thread so that the + // trace itself does not appear in the taskdump. + let handle = handle.clone(); + spawn_thread(async { + let handle = handle; + handle.dump().await + }).await + }, + } + } + } + + cfg_rt_multi_thread! { + /// Spawn a new thread and asynchronously await on its result. + async fn spawn_thread(f: F) -> ::Output + where + F: Future + Send + 'static, + ::Output: Send + 'static + { + let (tx, rx) = crate::sync::oneshot::channel(); + crate::loom::thread::spawn(|| { + let rt = crate::runtime::Builder::new_current_thread().build().unwrap(); + rt.block_on(async { + let _ = tx.send(f.await); + }); + }); + rx.await.unwrap() + } + } +} + /// Error returned by `try_current` when no Runtime has been started #[derive(Debug)] pub struct TryCurrentError { diff --git a/third_party/rust/tokio/src/runtime/io/metrics.rs b/third_party/rust/tokio/src/runtime/io/metrics.rs new file mode 100644 index 000000000000..ec341efe680a --- /dev/null +++ b/third_party/rust/tokio/src/runtime/io/metrics.rs @@ -0,0 +1,24 @@ +//! This file contains mocks of the metrics types used in the I/O driver. +//! +//! The reason these mocks don't live in `src/runtime/mock.rs` is because +//! these need to be available in the case when `net` is enabled but +//! `rt` is not. + +cfg_not_rt_and_metrics_and_net! { + #[derive(Default)] + pub(crate) struct IoDriverMetrics {} + + impl IoDriverMetrics { + pub(crate) fn incr_fd_count(&self) {} + pub(crate) fn dec_fd_count(&self) {} + pub(crate) fn incr_ready_count_by(&self, _amt: u64) {} + } +} + +cfg_net! { + cfg_rt! { + cfg_metrics! { + pub(crate) use crate::runtime::IoDriverMetrics; + } + } +} diff --git a/third_party/rust/tokio/src/io/driver/mod.rs b/third_party/rust/tokio/src/runtime/io/mod.rs similarity index 51% rename from third_party/rust/tokio/src/io/driver/mod.rs rename to third_party/rust/tokio/src/runtime/io/mod.rs index 19f67a24e7f1..2dd426f111f4 100644 --- a/third_party/rust/tokio/src/io/driver/mod.rs +++ b/third_party/rust/tokio/src/runtime/io/mod.rs @@ -1,12 +1,4 @@ -#![cfg_attr(not(feature = "rt"), allow(dead_code))] - -mod interest; -#[allow(unreachable_pub)] -pub use interest::Interest; - -mod ready; -#[allow(unreachable_pub)] -pub use ready::Ready; +#![cfg_attr(not(all(feature = "rt", feature = "net")), allow(dead_code))] mod registration; pub(crate) use registration::Registration; @@ -14,13 +6,18 @@ pub(crate) use registration::Registration; mod scheduled_io; use scheduled_io::ScheduledIo; -use crate::park::{Park, Unpark}; +mod metrics; + +use crate::io::interest::Interest; +use crate::io::ready::Ready; +use crate::runtime::driver; use crate::util::slab::{self, Slab}; -use crate::{loom::sync::Mutex, util::bit}; +use crate::{loom::sync::RwLock, util::bit}; + +use metrics::IoDriverMetrics; use std::fmt; use std::io; -use std::sync::{Arc, Weak}; use std::time::Duration; /// I/O driver, backed by Mio. @@ -29,51 +26,58 @@ pub(crate) struct Driver { /// as it is mostly used to determine when to call `compact()`. tick: u8, + /// True when an event with the signal token is received + signal_ready: bool, + /// Reuse the `mio::Events` value across calls to poll. - events: Option, + events: mio::Events, /// Primary slab handle containing the state for each resource registered - /// with this driver. During Drop this is moved into the Inner structure, so - /// this is an Option to allow it to be vacated (until Drop this is always - /// Some). - resources: Option>, + /// with this driver. + resources: Slab, /// The system event queue. poll: mio::Poll, - - /// State shared between the reactor and the handles. - inner: Arc, } /// A reference to an I/O driver. -#[derive(Clone)] pub(crate) struct Handle { - inner: Weak, + /// Registers I/O resources. + registry: mio::Registry, + + /// Allocates `ScheduledIo` handles when creating new resources. + io_dispatch: RwLock, + + /// Used to wake up the reactor from a call to `turn`. + /// Not supported on Wasi due to lack of threading support. + #[cfg(not(tokio_wasi))] + waker: mio::Waker, + + pub(crate) metrics: IoDriverMetrics, } #[derive(Debug)] pub(crate) struct ReadyEvent { tick: u8, pub(crate) ready: Ready, + is_shutdown: bool, } -pub(super) struct Inner { - /// Primary slab handle containing the state for each resource registered - /// with this driver. - /// - /// The ownership of this slab is moved into this structure during - /// `Driver::drop`, so that `Inner::drop` can notify all outstanding handles - /// without risking new ones being registered in the meantime. - resources: Mutex>>, +cfg_net_unix!( + impl ReadyEvent { + pub(crate) fn with_ready(&self, ready: Ready) -> Self { + Self { + ready, + tick: self.tick, + is_shutdown: self.is_shutdown, + } + } + } +); - /// Registers I/O resources. - registry: mio::Registry, - - /// Allocates `ScheduledIo` handles when creating new resources. - pub(super) io_dispatch: slab::Allocator, - - /// Used to wake up the reactor from a call to `turn`. - waker: mio::Waker, +struct IoDispatcher { + allocator: slab::Allocator, + is_shutdown: bool, } #[derive(Debug, Eq, PartialEq, Clone, Copy)] @@ -90,6 +94,7 @@ enum Tick { // TODO: Don't use a fake token. Instead, reserve a slot entry for the wakeup // token. const TOKEN_WAKEUP: mio::Token = mio::Token(1 << 31); +const TOKEN_SIGNAL: mio::Token = mio::Token(1 + (1 << 31)); const ADDRESS: bit::Pack = bit::Pack::least_significant(24); @@ -112,85 +117,113 @@ fn _assert_kinds() { impl Driver { /// Creates a new event loop, returning any error that happened during the /// creation. - pub(crate) fn new() -> io::Result { + pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> { let poll = mio::Poll::new()?; + #[cfg(not(tokio_wasi))] let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?; let registry = poll.registry().try_clone()?; let slab = Slab::new(); let allocator = slab.allocator(); - Ok(Driver { + let driver = Driver { tick: 0, - events: Some(mio::Events::with_capacity(1024)), + signal_ready: false, + events: mio::Events::with_capacity(nevents), poll, - resources: Some(slab), - inner: Arc::new(Inner { - resources: Mutex::new(None), - registry, - io_dispatch: allocator, - waker, - }), - }) + resources: slab, + }; + + let handle = Handle { + registry, + io_dispatch: RwLock::new(IoDispatcher::new(allocator)), + #[cfg(not(tokio_wasi))] + waker, + metrics: IoDriverMetrics::default(), + }; + + Ok((driver, handle)) } - /// Returns a handle to this event loop which can be sent across threads - /// and can be used as a proxy to the event loop itself. - /// - /// Handles are cloneable and clones always refer to the same event loop. - /// This handle is typically passed into functions that create I/O objects - /// to bind them to this event loop. - pub(crate) fn handle(&self) -> Handle { - Handle { - inner: Arc::downgrade(&self.inner), + pub(crate) fn park(&mut self, rt_handle: &driver::Handle) { + let handle = rt_handle.io(); + self.turn(handle, None); + } + + pub(crate) fn park_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) { + let handle = rt_handle.io(); + self.turn(handle, Some(duration)); + } + + pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) { + let handle = rt_handle.io(); + + if handle.shutdown() { + self.resources.for_each(|io| { + // If a task is waiting on the I/O resource, notify it that the + // runtime is being shutdown. And shutdown will clear all wakers. + io.shutdown(); + }); } } - fn turn(&mut self, max_wait: Option) -> io::Result<()> { + fn turn(&mut self, handle: &Handle, max_wait: Option) { // How often to call `compact()` on the resource slab const COMPACT_INTERVAL: u8 = 255; self.tick = self.tick.wrapping_add(1); if self.tick == COMPACT_INTERVAL { - self.resources.as_mut().unwrap().compact() + self.resources.compact() } - let mut events = self.events.take().expect("i/o driver event store missing"); + let events = &mut self.events; // Block waiting for an event to happen, peeling out how many events // happened. - match self.poll.poll(&mut events, max_wait) { + match self.poll.poll(events, max_wait) { Ok(_) => {} Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), + #[cfg(tokio_wasi)] + Err(e) if e.kind() == io::ErrorKind::InvalidInput => { + // In case of wasm32_wasi this error happens, when trying to poll without subscriptions + // just return from the park, as there would be nothing, which wakes us up. + } + Err(e) => panic!("unexpected error when polling the I/O driver: {:?}", e), } // Process all the events that came in, dispatching appropriately + let mut ready_count = 0; for event in events.iter() { let token = event.token(); - if token != TOKEN_WAKEUP { - self.dispatch(token, Ready::from_mio(event)); + if token == TOKEN_WAKEUP { + // Nothing to do, the event is used to unblock the I/O driver + } else if token == TOKEN_SIGNAL { + self.signal_ready = true; + } else { + Self::dispatch( + &mut self.resources, + self.tick, + token, + Ready::from_mio(event), + ); + ready_count += 1; } } - self.events = Some(events); - - Ok(()) + handle.metrics.incr_ready_count_by(ready_count); } - fn dispatch(&mut self, token: mio::Token, ready: Ready) { + fn dispatch(resources: &mut Slab, tick: u8, token: mio::Token, ready: Ready) { let addr = slab::Address::from_usize(ADDRESS.unpack(token.0)); - let resources = self.resources.as_mut().unwrap(); - let io = match resources.get(addr) { Some(io) => io, None => return, }; - let res = io.set_readiness(Some(token.0), Tick::Set(self.tick), |curr| curr | ready); + let res = io.set_readiness(Some(token.0), Tick::Set(tick), |curr| curr | ready); if res.is_err() { // token no longer valid! @@ -201,84 +234,12 @@ impl Driver { } } -impl Drop for Driver { - fn drop(&mut self) { - (*self.inner.resources.lock()) = self.resources.take(); - } -} - -impl Drop for Inner { - fn drop(&mut self) { - let resources = self.resources.lock().take(); - - if let Some(mut slab) = resources { - slab.for_each(|io| { - // If a task is waiting on the I/O resource, notify it. The task - // will then attempt to use the I/O resource and fail due to the - // driver being shutdown. - io.shutdown(); - }); - } - } -} - -impl Park for Driver { - type Unpark = Handle; - type Error = io::Error; - - fn unpark(&self) -> Self::Unpark { - self.handle() - } - - fn park(&mut self) -> io::Result<()> { - self.turn(None)?; - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> io::Result<()> { - self.turn(Some(duration))?; - Ok(()) - } - - fn shutdown(&mut self) {} -} - impl fmt::Debug for Driver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Driver") } } -// ===== impl Handle ===== - -cfg_rt! { - impl Handle { - /// Returns a handle to the current reactor. - /// - /// # Panics - /// - /// This function panics if there is no current reactor set and `rt` feature - /// flag is not enabled. - pub(super) fn current() -> Self { - crate::runtime::context::io_handle().expect("A Tokio 1.x context was found, but IO is disabled. Call `enable_io` on the runtime builder to enable IO.") - } - } -} - -cfg_not_rt! { - impl Handle { - /// Returns a handle to the current reactor. - /// - /// # Panics - /// - /// This function panics if there is no current reactor set, or if the `rt` - /// feature flag is not enabled. - pub(super) fn current() -> Self { - panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) - } - } -} - impl Handle { /// Forces a reactor blocked in a call to `turn` to wakeup, or otherwise /// makes the next call to `turn` return immediately. @@ -289,20 +250,64 @@ impl Handle { /// after this method has been called. If the reactor is not currently /// blocked in `turn`, then the next call to `turn` will not block and /// return immediately. - fn wakeup(&self) { - if let Some(inner) = self.inner() { - inner.waker.wake().expect("failed to wake I/O driver"); + pub(crate) fn unpark(&self) { + #[cfg(not(tokio_wasi))] + self.waker.wake().expect("failed to wake I/O driver"); + } + + /// Registers an I/O resource with the reactor for a given `mio::Ready` state. + /// + /// The registration token is returned. + pub(super) fn add_source( + &self, + source: &mut impl mio::event::Source, + interest: Interest, + ) -> io::Result> { + let (address, shared) = self.allocate()?; + + let token = GENERATION.pack(shared.generation(), ADDRESS.pack(address.as_usize(), 0)); + + self.registry + .register(source, mio::Token(token), interest.to_mio())?; + + self.metrics.incr_fd_count(); + + Ok(shared) + } + + /// Deregisters an I/O resource from the reactor. + pub(super) fn deregister_source(&self, source: &mut impl mio::event::Source) -> io::Result<()> { + self.registry.deregister(source)?; + + self.metrics.dec_fd_count(); + + Ok(()) + } + + /// shutdown the dispatcher. + fn shutdown(&self) -> bool { + let mut io = self.io_dispatch.write().unwrap(); + if io.is_shutdown { + return false; } + io.is_shutdown = true; + true } - pub(super) fn inner(&self) -> Option> { - self.inner.upgrade() - } -} - -impl Unpark for Handle { - fn unpark(&self) { - self.wakeup(); + fn allocate(&self) -> io::Result<(slab::Address, slab::Ref)> { + let io = self.io_dispatch.read().unwrap(); + if io.is_shutdown { + return Err(io::Error::new( + io::ErrorKind::Other, + crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR, + )); + } + io.allocator.allocate().ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "reactor at max registered I/O resources", + ) + }) } } @@ -312,35 +317,14 @@ impl fmt::Debug for Handle { } } -// ===== impl Inner ===== +// ===== impl IoDispatcher ===== -impl Inner { - /// Registers an I/O resource with the reactor for a given `mio::Ready` state. - /// - /// The registration token is returned. - pub(super) fn add_source( - &self, - source: &mut impl mio::event::Source, - interest: Interest, - ) -> io::Result> { - let (address, shared) = self.io_dispatch.allocate().ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "reactor at max registered I/O resources", - ) - })?; - - let token = GENERATION.pack(shared.generation(), ADDRESS.pack(address.as_usize(), 0)); - - self.registry - .register(source, mio::Token(token), interest.to_mio())?; - - Ok(shared) - } - - /// Deregisters an I/O resource from the reactor. - pub(super) fn deregister_source(&self, source: &mut impl mio::event::Source) -> io::Result<()> { - self.registry.deregister(source) +impl IoDispatcher { + fn new(allocator: slab::Allocator) -> Self { + Self { + allocator, + is_shutdown: false, + } } } @@ -352,3 +336,21 @@ impl Direction { } } } + +// Signal handling +cfg_signal_internal_and_unix! { + impl Handle { + pub(crate) fn register_signal_receiver(&self, receiver: &mut mio::net::UnixStream) -> io::Result<()> { + self.registry.register(receiver, TOKEN_SIGNAL, mio::Interest::READABLE)?; + Ok(()) + } + } + + impl Driver { + pub(crate) fn consume_signal_ready(&mut self) -> bool { + let ret = self.signal_ready; + self.signal_ready = false; + ret + } + } +} diff --git a/third_party/rust/tokio/src/io/driver/registration.rs b/third_party/rust/tokio/src/runtime/io/registration.rs similarity index 82% rename from third_party/rust/tokio/src/io/driver/registration.rs rename to third_party/rust/tokio/src/runtime/io/registration.rs index 7350be6345de..341fa0539a1e 100644 --- a/third_party/rust/tokio/src/io/driver/registration.rs +++ b/third_party/rust/tokio/src/runtime/io/registration.rs @@ -1,6 +1,8 @@ #![cfg_attr(not(feature = "net"), allow(dead_code))] -use crate::io::driver::{Direction, Handle, Interest, ReadyEvent, ScheduledIo}; +use crate::io::interest::Interest; +use crate::runtime::io::{Direction, Handle, ReadyEvent, ScheduledIo}; +use crate::runtime::scheduler; use crate::util::slab; use mio::event::Source; @@ -42,8 +44,8 @@ cfg_io_driver! { /// [`poll_write_ready`]: method@Self::poll_write_ready` #[derive(Debug)] pub(crate) struct Registration { - /// Handle to the associated driver. - handle: Handle, + /// Handle to the associated runtime. + handle: scheduler::Handle, /// Reference to state stored by the driver. shared: slab::Ref, @@ -56,10 +58,8 @@ unsafe impl Sync for Registration {} // ===== impl Registration ===== impl Registration { - /// Registers the I/O resource with the default reactor, for a specific - /// `Interest`. `new_with_interest` should be used over `new` when you need - /// control over the readiness state, such as when a file descriptor only - /// allows reads. This does not add `hup` or `error` so if you are + /// Registers the I/O resource with the reactor for the provided handle, for + /// a specific `Interest`. This does not add `hup` or `error` so if you are /// interested in those states, you will need to add them to the readiness /// state passed to this function. /// @@ -67,19 +67,13 @@ impl Registration { /// /// - `Ok` if the registration happened successfully /// - `Err` if an error was encountered during registration + #[track_caller] pub(crate) fn new_with_interest_and_handle( io: &mut impl Source, interest: Interest, - handle: Handle, + handle: scheduler::Handle, ) -> io::Result { - let shared = if let Some(inner) = handle.inner() { - inner.add_source(io, interest)? - } else { - return Err(io::Error::new( - io::ErrorKind::Other, - "failed to find event loop", - )); - }; + let shared = handle.driver().io().add_source(io, interest)?; Ok(Registration { handle, shared }) } @@ -101,11 +95,7 @@ impl Registration { /// /// `Err` is returned if an error is encountered. pub(crate) fn deregister(&mut self, io: &mut impl Source) -> io::Result<()> { - let inner = match self.handle.inner() { - Some(inner) => inner, - None => return Err(io::Error::new(io::ErrorKind::Other, "reactor gone")), - }; - inner.deregister_source(io) + self.handle().deregister_source(io) } pub(crate) fn clear_readiness(&self, event: ReadyEvent) { @@ -126,6 +116,7 @@ impl Registration { // Uses the poll path, requiring the caller to ensure mutual exclusion for // correctness. Only the last task to call this function is notified. + #[cfg(not(tokio_wasi))] pub(crate) fn poll_read_io( &self, cx: &mut Context<'_>, @@ -153,11 +144,12 @@ impl Registration { cx: &mut Context<'_>, direction: Direction, ) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let ev = ready!(self.shared.poll_readiness(cx, direction)); - if self.handle.inner().is_none() { + if ev.is_shutdown { return Poll::Ready(Err(gone())); } @@ -206,6 +198,10 @@ impl Registration { res => res, } } + + fn handle(&self) -> &Handle { + self.handle.driver().io() + } } impl Drop for Registration { @@ -222,28 +218,22 @@ impl Drop for Registration { } fn gone() -> io::Error { - io::Error::new(io::ErrorKind::Other, "IO driver has terminated") + io::Error::new( + io::ErrorKind::Other, + crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR, + ) } cfg_io_readiness! { impl Registration { pub(crate) async fn readiness(&self, interest: Interest) -> io::Result { - use std::future::Future; - use std::pin::Pin; + let ev = self.shared.readiness(interest).await; - let fut = self.shared.readiness(interest); - pin!(fut); + if ev.is_shutdown { + return Err(gone()) + } - crate::future::poll_fn(|cx| { - if self.handle.inner().is_none() { - return Poll::Ready(Err(io::Error::new( - io::ErrorKind::Other, - crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR - ))); - } - - Pin::new(&mut fut).poll(cx).map(Ok) - }).await + Ok(ev) } pub(crate) async fn async_io(&self, interest: Interest, mut f: impl FnMut() -> io::Result) -> io::Result { diff --git a/third_party/rust/tokio/src/io/driver/scheduled_io.rs b/third_party/rust/tokio/src/runtime/io/scheduled_io.rs similarity index 88% rename from third_party/rust/tokio/src/io/driver/scheduled_io.rs rename to third_party/rust/tokio/src/runtime/io/scheduled_io.rs index 76f93431ba22..197a4e0e211c 100644 --- a/third_party/rust/tokio/src/io/driver/scheduled_io.rs +++ b/third_party/rust/tokio/src/runtime/io/scheduled_io.rs @@ -1,4 +1,6 @@ -use super::{Interest, Ready, ReadyEvent, Tick}; +use super::{ReadyEvent, Tick}; +use crate::io::interest::Interest; +use crate::io::ready::Ready; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::Mutex; use crate::util::bit; @@ -44,9 +46,6 @@ struct Waiters { /// Waker used for AsyncWrite. writer: Option, - - /// True if this ScheduledIo has been killed due to IO driver shutdown. - is_shutdown: bool, } cfg_io_readiness! { @@ -66,6 +65,14 @@ cfg_io_readiness! { _p: PhantomPinned, } + generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull) -> NonNull> { + &self.pointers + } + } + } + /// Future returned by `readiness()`. struct Readiness<'a> { scheduled_io: &'a ScheduledIo, @@ -85,7 +92,7 @@ cfg_io_readiness! { // The `ScheduledIo::readiness` (`AtomicUsize`) is packed full of goodness. // -// | reserved | generation | driver tick | readiness | +// | shutdown | generation | driver tick | readiness | // |----------+------------+--------------+-----------| // | 1 bit | 7 bits + 8 bits + 16 bits | @@ -95,6 +102,8 @@ const TICK: bit::Pack = READINESS.then(8); const GENERATION: bit::Pack = TICK.then(7); +const SHUTDOWN: bit::Pack = GENERATION.then(1); + #[test] fn test_generations_assert_same() { assert_eq!(super::GENERATION, GENERATION); @@ -128,9 +137,11 @@ impl ScheduledIo { } /// Invoked when the IO driver is shut down; forces this ScheduledIo into a - /// permanently ready state. + /// permanently shutdown state. pub(super) fn shutdown(&self) { - self.wake0(Ready::ALL, true) + let mask = SHUTDOWN.pack(1, 0); + self.readiness.fetch_or(mask, AcqRel); + self.wake(Ready::ALL); } /// Sets the readiness on this `ScheduledIo` by invoking the given closure on @@ -209,16 +220,10 @@ impl ScheduledIo { /// than 32 wakers to notify, if the stack array fills up, the lock is /// released, the array is cleared, and the iteration continues. pub(super) fn wake(&self, ready: Ready) { - self.wake0(ready, false); - } - - fn wake0(&self, ready: Ready, shutdown: bool) { let mut wakers = WakeList::new(); let mut waiters = self.waiters.lock(); - waiters.is_shutdown |= shutdown; - // check for AsyncRead slot if ready.is_readable() { if let Some(waker) = waiters.reader.take() { @@ -273,6 +278,7 @@ impl ScheduledIo { ReadyEvent { tick: TICK.unpack(curr) as u8, ready: interest.mask() & Ready::from_usize(READINESS.unpack(curr)), + is_shutdown: SHUTDOWN.unpack(curr) != 0, } } @@ -289,8 +295,9 @@ impl ScheduledIo { let curr = self.readiness.load(Acquire); let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr)); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; - if ready.is_empty() { + if ready.is_empty() && !is_shutdown { // Update the task info let mut waiters = self.waiters.lock(); let slot = match direction { @@ -315,10 +322,12 @@ impl ScheduledIo { // taking the waiters lock let curr = self.readiness.load(Acquire); let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr)); - if waiters.is_shutdown { + let is_shutdown = SHUTDOWN.unpack(curr) != 0; + if is_shutdown { Poll::Ready(ReadyEvent { tick: TICK.unpack(curr) as u8, ready: direction.mask(), + is_shutdown, }) } else if ready.is_empty() { Poll::Pending @@ -326,12 +335,14 @@ impl ScheduledIo { Poll::Ready(ReadyEvent { tick: TICK.unpack(curr) as u8, ready, + is_shutdown, }) } } else { Poll::Ready(ReadyEvent { tick: TICK.unpack(curr) as u8, ready, + is_shutdown, }) } } @@ -399,8 +410,8 @@ cfg_io_readiness! { ptr } - unsafe fn pointers(mut target: NonNull) -> NonNull> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull) -> NonNull> { + Waiter::addr_of_pointers(target) } } @@ -423,16 +434,17 @@ cfg_io_readiness! { // Optimistically check existing readiness let curr = scheduled_io.readiness.load(SeqCst); let ready = Ready::from_usize(READINESS.unpack(curr)); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; // Safety: `waiter.interest` never changes let interest = unsafe { (*waiter.get()).interest }; let ready = ready.intersection(interest); - if !ready.is_empty() { + if !ready.is_empty() || is_shutdown { // Currently ready! let tick = TICK.unpack(curr) as u8; *state = State::Done; - return Poll::Ready(ReadyEvent { tick, ready }); + return Poll::Ready(ReadyEvent { tick, ready, is_shutdown }); } // Wasn't ready, take the lock (and check again while locked). @@ -440,18 +452,19 @@ cfg_io_readiness! { let curr = scheduled_io.readiness.load(SeqCst); let mut ready = Ready::from_usize(READINESS.unpack(curr)); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; - if waiters.is_shutdown { + if is_shutdown { ready = Ready::ALL; } let ready = ready.intersection(interest); - if !ready.is_empty() { + if !ready.is_empty() || is_shutdown { // Currently ready! let tick = TICK.unpack(curr) as u8; *state = State::Done; - return Poll::Ready(ReadyEvent { tick, ready }); + return Poll::Ready(ReadyEvent { tick, ready, is_shutdown }); } // Not ready even after locked, insert into list... @@ -500,14 +513,26 @@ cfg_io_readiness! { drop(waiters); } State::Done => { - let tick = TICK.unpack(scheduled_io.readiness.load(Acquire)) as u8; - // Safety: State::Done means it is no longer shared let w = unsafe { &mut *waiter.get() }; + let curr = scheduled_io.readiness.load(Acquire); + let is_shutdown = SHUTDOWN.unpack(curr) != 0; + + // The returned tick might be newer than the event + // which notified our waker. This is ok because the future + // still didn't return `Poll::Ready`. + let tick = TICK.unpack(curr) as u8; + + // The readiness state could have been cleared in the meantime, + // but we allow the returned ready set to be empty. + let curr_ready = Ready::from_usize(READINESS.unpack(curr)); + let ready = curr_ready.intersection(w.interest); + return Poll::Ready(ReadyEvent { tick, - ready: Ready::from_interest(w.interest), + ready, + is_shutdown, }); } } diff --git a/third_party/rust/tokio/src/runtime/metrics/batch.rs b/third_party/rust/tokio/src/runtime/metrics/batch.rs index f1c3fa6b7474..1bb4e261f733 100644 --- a/third_party/rust/tokio/src/runtime/metrics/batch.rs +++ b/third_party/rust/tokio/src/runtime/metrics/batch.rs @@ -1,8 +1,7 @@ -use crate::runtime::WorkerMetrics; +use crate::runtime::metrics::{HistogramBatch, WorkerMetrics}; -use std::convert::TryFrom; use std::sync::atomic::Ordering::Relaxed; -use std::time::Instant; +use std::time::{Duration, Instant}; pub(crate) struct MetricsBatch { /// Number of times the worker parked. @@ -11,9 +10,12 @@ pub(crate) struct MetricsBatch { /// Number of times the worker woke w/o doing work. noop_count: u64, - /// Number of times stolen. + /// Number of tasks stolen. steal_count: u64, + /// Number of times tasks where stolen. + steal_operations: u64, + /// Number of tasks that were polled by the worker. poll_count: u64, @@ -30,21 +32,44 @@ pub(crate) struct MetricsBatch { /// The total busy duration in nanoseconds. busy_duration_total: u64, - last_resume_time: Instant, + + /// Instant at which work last resumed (continued after park). + processing_scheduled_tasks_started_at: Instant, + + /// If `Some`, tracks poll times in nanoseconds + poll_timer: Option, +} + +struct PollTimer { + /// Histogram of poll counts within each band. + poll_counts: HistogramBatch, + + /// Instant when the most recent task started polling. + poll_started_at: Instant, } impl MetricsBatch { - pub(crate) fn new() -> MetricsBatch { + pub(crate) fn new(worker_metrics: &WorkerMetrics) -> MetricsBatch { + let now = Instant::now(); + MetricsBatch { park_count: 0, noop_count: 0, steal_count: 0, + steal_operations: 0, poll_count: 0, poll_count_on_last_park: 0, local_schedule_count: 0, overflow_count: 0, busy_duration_total: 0, - last_resume_time: Instant::now(), + processing_scheduled_tasks_started_at: now, + poll_timer: worker_metrics + .poll_count_histogram + .as_ref() + .map(|worker_poll_counts| PollTimer { + poll_counts: HistogramBatch::from_histogram(worker_poll_counts), + poll_started_at: now, + }), } } @@ -52,6 +77,9 @@ impl MetricsBatch { worker.park_count.store(self.park_count, Relaxed); worker.noop_count.store(self.noop_count, Relaxed); worker.steal_count.store(self.steal_count, Relaxed); + worker + .steal_operations + .store(self.steal_operations, Relaxed); worker.poll_count.store(self.poll_count, Relaxed); worker @@ -62,6 +90,11 @@ impl MetricsBatch { .local_schedule_count .store(self.local_schedule_count, Relaxed); worker.overflow_count.store(self.overflow_count, Relaxed); + + if let Some(poll_timer) = &self.poll_timer { + let dst = worker.poll_count_histogram.as_ref().unwrap(); + poll_timer.poll_counts.submit(dst); + } } /// The worker is about to park. @@ -73,23 +106,39 @@ impl MetricsBatch { } else { self.poll_count_on_last_park = self.poll_count; } - - let busy_duration = self.last_resume_time.elapsed(); - let busy_duration = u64::try_from(busy_duration.as_nanos()).unwrap_or(u64::MAX); - self.busy_duration_total += busy_duration; } - pub(crate) fn returned_from_park(&mut self) { - self.last_resume_time = Instant::now(); + /// Start processing a batch of tasks + pub(crate) fn start_processing_scheduled_tasks(&mut self) { + self.processing_scheduled_tasks_started_at = Instant::now(); + } + + /// Stop processing a batch of tasks + pub(crate) fn end_processing_scheduled_tasks(&mut self) { + let busy_duration = self.processing_scheduled_tasks_started_at.elapsed(); + self.busy_duration_total += duration_as_u64(busy_duration); + } + + /// Start polling an individual task + pub(crate) fn start_poll(&mut self) { + self.poll_count += 1; + + if let Some(poll_timer) = &mut self.poll_timer { + poll_timer.poll_started_at = Instant::now(); + } + } + + /// Stop polling an individual task + pub(crate) fn end_poll(&mut self) { + if let Some(poll_timer) = &mut self.poll_timer { + let elapsed = duration_as_u64(poll_timer.poll_started_at.elapsed()); + poll_timer.poll_counts.measure(elapsed, 1); + } } pub(crate) fn inc_local_schedule_count(&mut self) { self.local_schedule_count += 1; } - - pub(crate) fn incr_poll_count(&mut self) { - self.poll_count += 1; - } } cfg_rt_multi_thread! { @@ -98,8 +147,16 @@ cfg_rt_multi_thread! { self.steal_count += by as u64; } + pub(crate) fn incr_steal_operations(&mut self) { + self.steal_operations += 1; + } + pub(crate) fn incr_overflow_count(&mut self) { self.overflow_count += 1; } } } + +fn duration_as_u64(dur: Duration) -> u64 { + u64::try_from(dur.as_nanos()).unwrap_or(u64::MAX) +} diff --git a/third_party/rust/tokio/src/runtime/metrics/histogram.rs b/third_party/rust/tokio/src/runtime/metrics/histogram.rs new file mode 100644 index 000000000000..976f54fe8525 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/metrics/histogram.rs @@ -0,0 +1,502 @@ +use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +use std::cmp; +use std::ops::Range; + +#[derive(Debug)] +pub(crate) struct Histogram { + /// The histogram buckets + buckets: Box<[AtomicU64]>, + + /// Bucket scale, linear or log + scale: HistogramScale, + + /// Minimum resolution + resolution: u64, +} + +#[derive(Debug, Clone)] +pub(crate) struct HistogramBuilder { + /// Histogram scale + pub(crate) scale: HistogramScale, + + /// Must be a power of 2 + pub(crate) resolution: u64, + + /// Number of buckets + pub(crate) num_buckets: usize, +} + +#[derive(Debug)] +pub(crate) struct HistogramBatch { + buckets: Box<[u64]>, + scale: HistogramScale, + resolution: u64, +} + +cfg_unstable! { + /// Whether the histogram used to aggregate a metric uses a linear or + /// logarithmic scale. + #[derive(Debug, Copy, Clone, Eq, PartialEq)] + #[non_exhaustive] + pub enum HistogramScale { + /// Linear bucket scale + Linear, + + /// Logarithmic bucket scale + Log, + } +} + +impl Histogram { + pub(crate) fn num_buckets(&self) -> usize { + self.buckets.len() + } + + pub(crate) fn get(&self, bucket: usize) -> u64 { + self.buckets[bucket].load(Relaxed) + } + + pub(crate) fn bucket_range(&self, bucket: usize) -> Range { + match self.scale { + HistogramScale::Log => Range { + start: if bucket == 0 { + 0 + } else { + self.resolution << (bucket - 1) + }, + end: if bucket == self.buckets.len() - 1 { + u64::MAX + } else { + self.resolution << bucket + }, + }, + HistogramScale::Linear => Range { + start: self.resolution * bucket as u64, + end: if bucket == self.buckets.len() - 1 { + u64::MAX + } else { + self.resolution * (bucket as u64 + 1) + }, + }, + } + } +} + +impl HistogramBatch { + pub(crate) fn from_histogram(histogram: &Histogram) -> HistogramBatch { + let buckets = vec![0; histogram.buckets.len()].into_boxed_slice(); + + HistogramBatch { + buckets, + scale: histogram.scale, + resolution: histogram.resolution, + } + } + + pub(crate) fn measure(&mut self, value: u64, count: u64) { + self.buckets[self.value_to_bucket(value)] += count; + } + + pub(crate) fn submit(&self, histogram: &Histogram) { + debug_assert_eq!(self.scale, histogram.scale); + debug_assert_eq!(self.resolution, histogram.resolution); + debug_assert_eq!(self.buckets.len(), histogram.buckets.len()); + + for i in 0..self.buckets.len() { + histogram.buckets[i].store(self.buckets[i], Relaxed); + } + } + + fn value_to_bucket(&self, value: u64) -> usize { + match self.scale { + HistogramScale::Linear => { + let max = self.buckets.len() - 1; + cmp::min(value / self.resolution, max as u64) as usize + } + HistogramScale::Log => { + let max = self.buckets.len() - 1; + + if value < self.resolution { + 0 + } else { + let significant_digits = 64 - value.leading_zeros(); + let bucket_digits = 64 - (self.resolution - 1).leading_zeros(); + cmp::min(significant_digits as usize - bucket_digits as usize, max) + } + } + } + } +} + +impl HistogramBuilder { + pub(crate) fn new() -> HistogramBuilder { + HistogramBuilder { + scale: HistogramScale::Linear, + // Resolution is in nanoseconds. + resolution: 100_000, + num_buckets: 10, + } + } + + pub(crate) fn build(&self) -> Histogram { + let mut resolution = self.resolution; + + assert!(resolution > 0); + + if matches!(self.scale, HistogramScale::Log) { + resolution = resolution.next_power_of_two(); + } + + Histogram { + buckets: (0..self.num_buckets) + .map(|_| AtomicU64::new(0)) + .collect::>() + .into_boxed_slice(), + resolution, + scale: self.scale, + } + } +} + +impl Default for HistogramBuilder { + fn default() -> HistogramBuilder { + HistogramBuilder::new() + } +} + +#[cfg(test)] +mod test { + use super::*; + + macro_rules! assert_bucket_eq { + ($h:expr, $bucket:expr, $val:expr) => {{ + assert_eq!($h.buckets[$bucket], $val); + }}; + } + + #[test] + fn log_scale_resolution_1() { + let h = HistogramBuilder { + scale: HistogramScale::Log, + resolution: 1, + num_buckets: 10, + } + .build(); + + assert_eq!(h.bucket_range(0), 0..1); + assert_eq!(h.bucket_range(1), 1..2); + assert_eq!(h.bucket_range(2), 2..4); + assert_eq!(h.bucket_range(3), 4..8); + assert_eq!(h.bucket_range(9), 256..u64::MAX); + + let mut b = HistogramBatch::from_histogram(&h); + + b.measure(0, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 0); + + b.measure(1, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 0); + + b.measure(2, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 1); + + b.measure(3, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 2); + + b.measure(4, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 2); + assert_bucket_eq!(b, 3, 1); + + b.measure(100, 1); + assert_bucket_eq!(b, 7, 1); + + b.measure(128, 1); + assert_bucket_eq!(b, 8, 1); + + b.measure(4096, 1); + assert_bucket_eq!(b, 9, 1); + } + + #[test] + fn log_scale_resolution_2() { + let h = HistogramBuilder { + scale: HistogramScale::Log, + resolution: 2, + num_buckets: 10, + } + .build(); + + assert_eq!(h.bucket_range(0), 0..2); + assert_eq!(h.bucket_range(1), 2..4); + assert_eq!(h.bucket_range(2), 4..8); + assert_eq!(h.bucket_range(3), 8..16); + assert_eq!(h.bucket_range(9), 512..u64::MAX); + + let mut b = HistogramBatch::from_histogram(&h); + + b.measure(0, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 0); + + b.measure(1, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 0); + + b.measure(2, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 0); + + b.measure(3, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 0); + + b.measure(4, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 1); + + b.measure(5, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 2); + + b.measure(6, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 3); + + b.measure(7, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 4); + + b.measure(8, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 4); + assert_bucket_eq!(b, 3, 1); + + b.measure(100, 1); + assert_bucket_eq!(b, 6, 1); + + b.measure(128, 1); + assert_bucket_eq!(b, 7, 1); + + b.measure(4096, 1); + assert_bucket_eq!(b, 9, 1); + + for bucket in h.buckets.iter() { + assert_eq!(bucket.load(Relaxed), 0); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + } + + #[test] + fn linear_scale_resolution_1() { + let h = HistogramBuilder { + scale: HistogramScale::Linear, + resolution: 1, + num_buckets: 10, + } + .build(); + + assert_eq!(h.bucket_range(0), 0..1); + assert_eq!(h.bucket_range(1), 1..2); + assert_eq!(h.bucket_range(2), 2..3); + assert_eq!(h.bucket_range(3), 3..4); + assert_eq!(h.bucket_range(9), 9..u64::MAX); + + let mut b = HistogramBatch::from_histogram(&h); + + b.measure(0, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 0); + + b.measure(1, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 0); + + b.measure(2, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 1); + assert_bucket_eq!(b, 3, 0); + + b.measure(3, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 1); + assert_bucket_eq!(b, 3, 1); + + b.measure(5, 1); + assert_bucket_eq!(b, 5, 1); + + b.measure(4096, 1); + assert_bucket_eq!(b, 9, 1); + + for bucket in h.buckets.iter() { + assert_eq!(bucket.load(Relaxed), 0); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + } + + #[test] + fn linear_scale_resolution_100() { + let h = HistogramBuilder { + scale: HistogramScale::Linear, + resolution: 100, + num_buckets: 10, + } + .build(); + + assert_eq!(h.bucket_range(0), 0..100); + assert_eq!(h.bucket_range(1), 100..200); + assert_eq!(h.bucket_range(2), 200..300); + assert_eq!(h.bucket_range(3), 300..400); + assert_eq!(h.bucket_range(9), 900..u64::MAX); + + let mut b = HistogramBatch::from_histogram(&h); + + b.measure(0, 1); + assert_bucket_eq!(b, 0, 1); + assert_bucket_eq!(b, 1, 0); + + b.measure(50, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 0); + + b.measure(100, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 1); + assert_bucket_eq!(b, 2, 0); + + b.measure(101, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 0); + + b.measure(200, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 1); + + b.measure(299, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 2); + + b.measure(222, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 3); + + b.measure(300, 1); + assert_bucket_eq!(b, 0, 2); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 3); + assert_bucket_eq!(b, 3, 1); + + b.measure(888, 1); + assert_bucket_eq!(b, 8, 1); + + b.measure(4096, 1); + assert_bucket_eq!(b, 9, 1); + + for bucket in h.buckets.iter() { + assert_eq!(bucket.load(Relaxed), 0); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + } + + #[test] + fn inc_by_more_than_one() { + let h = HistogramBuilder { + scale: HistogramScale::Linear, + resolution: 100, + num_buckets: 10, + } + .build(); + + let mut b = HistogramBatch::from_histogram(&h); + + b.measure(0, 3); + assert_bucket_eq!(b, 0, 3); + assert_bucket_eq!(b, 1, 0); + + b.measure(50, 5); + assert_bucket_eq!(b, 0, 8); + assert_bucket_eq!(b, 1, 0); + + b.measure(100, 2); + assert_bucket_eq!(b, 0, 8); + assert_bucket_eq!(b, 1, 2); + assert_bucket_eq!(b, 2, 0); + + b.measure(101, 19); + assert_bucket_eq!(b, 0, 8); + assert_bucket_eq!(b, 1, 21); + assert_bucket_eq!(b, 2, 0); + + for bucket in h.buckets.iter() { + assert_eq!(bucket.load(Relaxed), 0); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + + b.submit(&h); + + for i in 0..h.buckets.len() { + assert_eq!(h.buckets[i].load(Relaxed), b.buckets[i]); + } + } +} diff --git a/third_party/rust/tokio/src/runtime/metrics/io.rs b/third_party/rust/tokio/src/runtime/metrics/io.rs new file mode 100644 index 000000000000..06efdd42d721 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/metrics/io.rs @@ -0,0 +1,24 @@ +#![cfg_attr(not(feature = "net"), allow(dead_code))] + +use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +#[derive(Default)] +pub(crate) struct IoDriverMetrics { + pub(super) fd_registered_count: AtomicU64, + pub(super) fd_deregistered_count: AtomicU64, + pub(super) ready_count: AtomicU64, +} + +impl IoDriverMetrics { + pub(crate) fn incr_fd_count(&self) { + self.fd_registered_count.fetch_add(1, Relaxed); + } + + pub(crate) fn dec_fd_count(&self) { + self.fd_deregistered_count.fetch_add(1, Relaxed); + } + + pub(crate) fn incr_ready_count_by(&self, amt: u64) { + self.ready_count.fetch_add(amt, Relaxed); + } +} diff --git a/third_party/rust/tokio/src/runtime/metrics/mock.rs b/third_party/rust/tokio/src/runtime/metrics/mock.rs index 6b9cf704f422..8f8345c08b43 100644 --- a/third_party/rust/tokio/src/runtime/metrics/mock.rs +++ b/third_party/rust/tokio/src/runtime/metrics/mock.rs @@ -6,6 +6,9 @@ pub(crate) struct WorkerMetrics {} pub(crate) struct MetricsBatch {} +#[derive(Clone, Default)] +pub(crate) struct HistogramBuilder {} + impl SchedulerMetrics { pub(crate) fn new() -> Self { Self {} @@ -20,24 +23,33 @@ impl WorkerMetrics { Self {} } + pub(crate) fn from_config(config: &crate::runtime::Config) -> Self { + // Prevent the dead-code warning from being triggered + let _ = &config.metrics_poll_count_histogram; + Self::new() + } + pub(crate) fn set_queue_depth(&self, _len: usize) {} } impl MetricsBatch { - pub(crate) fn new() -> Self { + pub(crate) fn new(_: &WorkerMetrics) -> Self { Self {} } pub(crate) fn submit(&mut self, _to: &WorkerMetrics) {} pub(crate) fn about_to_park(&mut self) {} - pub(crate) fn returned_from_park(&mut self) {} - pub(crate) fn incr_poll_count(&mut self) {} pub(crate) fn inc_local_schedule_count(&mut self) {} + pub(crate) fn start_processing_scheduled_tasks(&mut self) {} + pub(crate) fn end_processing_scheduled_tasks(&mut self) {} + pub(crate) fn start_poll(&mut self) {} + pub(crate) fn end_poll(&mut self) {} } cfg_rt_multi_thread! { impl MetricsBatch { pub(crate) fn incr_steal_count(&mut self, _by: u16) {} + pub(crate) fn incr_steal_operations(&mut self) {} pub(crate) fn incr_overflow_count(&mut self) {} } } diff --git a/third_party/rust/tokio/src/runtime/metrics/mod.rs b/third_party/rust/tokio/src/runtime/metrics/mod.rs index ca643a59047e..88be4a5211fd 100644 --- a/third_party/rust/tokio/src/runtime/metrics/mod.rs +++ b/third_party/rust/tokio/src/runtime/metrics/mod.rs @@ -12,6 +12,11 @@ cfg_metrics! { mod batch; pub(crate) use batch::MetricsBatch; + mod histogram; + pub(crate) use histogram::{Histogram, HistogramBatch, HistogramBuilder}; + #[allow(unreachable_pub)] // rust-lang/rust#57411 + pub use histogram::HistogramScale; + mod runtime; #[allow(unreachable_pub)] // rust-lang/rust#57411 pub use runtime::RuntimeMetrics; @@ -21,10 +26,15 @@ cfg_metrics! { mod worker; pub(crate) use worker::WorkerMetrics; + + cfg_net! { + mod io; + pub(crate) use io::IoDriverMetrics; + } } cfg_not_metrics! { mod mock; - pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch}; + pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch, HistogramBuilder}; } diff --git a/third_party/rust/tokio/src/runtime/metrics/runtime.rs b/third_party/rust/tokio/src/runtime/metrics/runtime.rs index 0f8055907f56..1f990a1f8527 100644 --- a/third_party/rust/tokio/src/runtime/metrics/runtime.rs +++ b/third_party/rust/tokio/src/runtime/metrics/runtime.rs @@ -1,5 +1,6 @@ use crate::runtime::Handle; +use std::ops::Range; use std::sync::atomic::Ordering::Relaxed; use std::time::Duration; @@ -39,7 +40,76 @@ impl RuntimeMetrics { /// } /// ``` pub fn num_workers(&self) -> usize { - self.handle.spawner.num_workers() + self.handle.inner.num_workers() + } + + /// Returns the number of additional threads spawned by the runtime. + /// + /// The number of workers is set by configuring `max_blocking_threads` on + /// `runtime::Builder`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let _ = tokio::task::spawn_blocking(move || { + /// // Stand-in for compute-heavy work or using synchronous APIs + /// 1 + 1 + /// }).await; + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.num_blocking_threads(); + /// println!("Runtime has created {} threads", n); + /// } + /// ``` + pub fn num_blocking_threads(&self) -> usize { + self.handle.inner.num_blocking_threads() + } + + /// Returns the number of active tasks in the runtime. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.active_tasks_count(); + /// println!("Runtime has {} active tasks", n); + /// } + /// ``` + pub fn active_tasks_count(&self) -> usize { + self.handle.inner.active_tasks_count() + } + + /// Returns the number of idle threads, which have spawned by the runtime + /// for `spawn_blocking` calls. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let _ = tokio::task::spawn_blocking(move || { + /// // Stand-in for compute-heavy work or using synchronous APIs + /// 1 + 1 + /// }).await; + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.num_idle_blocking_threads(); + /// println!("Runtime has {} idle blocking thread pool threads", n); + /// } + /// ``` + pub fn num_idle_blocking_threads(&self) -> usize { + self.handle.inner.num_idle_blocking_threads() } /// Returns the number of tasks scheduled from **outside** of the runtime. @@ -68,12 +138,27 @@ impl RuntimeMetrics { /// ``` pub fn remote_schedule_count(&self) -> u64 { self.handle - .spawner + .inner .scheduler_metrics() .remote_schedule_count .load(Relaxed) } + /// Returns the number of times that tasks have been forced to yield back to the scheduler + /// after exhausting their task budgets. + /// + /// This count starts at zero when the runtime is created and increases by one each time a task yields due to exhausting its budget. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + pub fn budget_forced_yield_count(&self) -> u64 { + self.handle + .inner + .scheduler_metrics() + .budget_forced_yield_count + .load(Relaxed) + } + /// Returns the total number of times the given worker thread has parked. /// /// The worker park count starts at zero when the runtime is created and @@ -88,7 +173,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -111,7 +196,7 @@ impl RuntimeMetrics { /// ``` pub fn worker_park_count(&self, worker: usize) -> u64 { self.handle - .spawner + .inner .worker_metrics(worker) .park_count .load(Relaxed) @@ -131,7 +216,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -154,20 +239,21 @@ impl RuntimeMetrics { /// ``` pub fn worker_noop_count(&self, worker: usize) -> u64 { self.handle - .spawner + .inner .worker_metrics(worker) .noop_count .load(Relaxed) } - /// Returns the number of times the given worker thread stole tasks from + /// Returns the number of tasks the given worker thread stole from /// another worker thread. /// - /// This metric only applies to the **multi-threaded** runtime and will always return `0` when using the current thread runtime. + /// This metric only applies to the **multi-threaded** runtime and will + /// always return `0` when using the current thread runtime. /// /// The worker steal count starts at zero when the runtime is created and - /// increases by one each time the worker has processed its scheduled queue - /// and successfully steals more pending tasks from another worker. + /// increases by `N` each time the worker has processed its scheduled queue + /// and successfully steals `N` more pending tasks from another worker. /// /// The counter is monotonically increasing. It is never decremented or /// reset to zero. @@ -176,7 +262,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -193,18 +279,64 @@ impl RuntimeMetrics { /// async fn main() { /// let metrics = Handle::current().metrics(); /// - /// let n = metrics.worker_noop_count(0); - /// println!("worker 0 has stolen tasks {} times", n); + /// let n = metrics.worker_steal_count(0); + /// println!("worker 0 has stolen {} tasks", n); /// } /// ``` pub fn worker_steal_count(&self, worker: usize) -> u64 { self.handle - .spawner + .inner .worker_metrics(worker) .steal_count .load(Relaxed) } + /// Returns the number of times the given worker thread stole tasks from + /// another worker thread. + /// + /// This metric only applies to the **multi-threaded** runtime and will + /// always return `0` when using the current thread runtime. + /// + /// The worker steal count starts at zero when the runtime is created and + /// increases by one each time the worker has processed its scheduled queue + /// and successfully steals more pending tasks from another worker. + /// + /// The counter is monotonically increasing. It is never decremented or + /// reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.worker_steal_operations(0); + /// println!("worker 0 has stolen tasks {} times", n); + /// } + /// ``` + pub fn worker_steal_operations(&self, worker: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .steal_operations + .load(Relaxed) + } + /// Returns the number of tasks the given worker thread has polled. /// /// The worker poll count starts at zero when the runtime is created and @@ -217,7 +349,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -240,7 +372,7 @@ impl RuntimeMetrics { /// ``` pub fn worker_poll_count(&self, worker: usize) -> u64 { self.handle - .spawner + .inner .worker_metrics(worker) .poll_count .load(Relaxed) @@ -261,7 +393,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -278,14 +410,14 @@ impl RuntimeMetrics { /// async fn main() { /// let metrics = Handle::current().metrics(); /// - /// let n = metrics.worker_poll_count(0); - /// println!("worker 0 has polled {} tasks", n); + /// let n = metrics.worker_total_busy_duration(0); + /// println!("worker 0 was busy for a total of {:?}", n); /// } /// ``` pub fn worker_total_busy_duration(&self, worker: usize) -> Duration { let nanos = self .handle - .spawner + .inner .worker_metrics(worker) .busy_duration_total .load(Relaxed); @@ -308,7 +440,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -331,7 +463,7 @@ impl RuntimeMetrics { /// ``` pub fn worker_local_schedule_count(&self, worker: usize) -> u64 { self.handle - .spawner + .inner .worker_metrics(worker) .local_schedule_count .load(Relaxed) @@ -354,7 +486,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -377,7 +509,7 @@ impl RuntimeMetrics { /// ``` pub fn worker_overflow_count(&self, worker: usize) -> u64 { self.handle - .spawner + .inner .worker_metrics(worker) .overflow_count .load(Relaxed) @@ -386,7 +518,7 @@ impl RuntimeMetrics { /// Returns the number of tasks currently scheduled in the runtime's /// injection queue. /// - /// Tasks that are spanwed or notified from a non-runtime thread are + /// Tasks that are spawned or notified from a non-runtime thread are /// scheduled using the runtime's injection queue. This metric returns the /// **current** number of tasks pending in the injection queue. As such, the /// returned value may increase or decrease as new tasks are scheduled and @@ -406,7 +538,7 @@ impl RuntimeMetrics { /// } /// ``` pub fn injection_queue_depth(&self) -> usize { - self.handle.spawner.injection_queue_depth() + self.handle.inner.injection_queue_depth() } /// Returns the number of tasks currently scheduled in the given worker's @@ -422,7 +554,7 @@ impl RuntimeMetrics { /// /// `worker` is the index of the worker being queried. The given value must /// be between 0 and `num_workers()`. The index uniquely identifies a single - /// worker and will continue to indentify the worker throughout the lifetime + /// worker and will continue to identify the worker throughout the lifetime /// of the runtime instance. /// /// # Panics @@ -444,6 +576,308 @@ impl RuntimeMetrics { /// } /// ``` pub fn worker_local_queue_depth(&self, worker: usize) -> usize { - self.handle.spawner.worker_local_queue_depth(worker) + self.handle.inner.worker_local_queue_depth(worker) + } + + /// Returns `true` if the runtime is tracking the distribution of task poll + /// times. + /// + /// Task poll times are not instrumented by default as doing so requires + /// calling [`Instant::now()`] twice per task poll. The feature is enabled + /// by calling [`enable_metrics_poll_count_histogram()`] when building the + /// runtime. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{self, Handle}; + /// + /// fn main() { + /// runtime::Builder::new_current_thread() + /// .enable_metrics_poll_count_histogram() + /// .build() + /// .unwrap() + /// .block_on(async { + /// let metrics = Handle::current().metrics(); + /// let enabled = metrics.poll_count_histogram_enabled(); + /// + /// println!("Tracking task poll time distribution: {:?}", enabled); + /// }); + /// } + /// ``` + /// + /// [`enable_metrics_poll_count_histogram()`]: crate::runtime::Builder::enable_metrics_poll_count_histogram + /// [`Instant::now()`]: std::time::Instant::now + pub fn poll_count_histogram_enabled(&self) -> bool { + self.handle + .inner + .worker_metrics(0) + .poll_count_histogram + .is_some() + } + + /// Returns the number of histogram buckets tracking the distribution of + /// task poll times. + /// + /// This value is configured by calling + /// [`metrics_poll_count_histogram_buckets()`] when building the runtime. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{self, Handle}; + /// + /// fn main() { + /// runtime::Builder::new_current_thread() + /// .enable_metrics_poll_count_histogram() + /// .build() + /// .unwrap() + /// .block_on(async { + /// let metrics = Handle::current().metrics(); + /// let buckets = metrics.poll_count_histogram_num_buckets(); + /// + /// println!("Histogram buckets: {:?}", buckets); + /// }); + /// } + /// ``` + /// + /// [`metrics_poll_count_histogram_buckets()`]: + /// crate::runtime::Builder::metrics_poll_count_histogram_buckets + pub fn poll_count_histogram_num_buckets(&self) -> usize { + self.handle + .inner + .worker_metrics(0) + .poll_count_histogram + .as_ref() + .map(|histogram| histogram.num_buckets()) + .unwrap_or_default() + } + + /// Returns the range of task poll times tracked by the given bucket. + /// + /// This value is configured by calling + /// [`metrics_poll_count_histogram_resolution()`] when building the runtime. + /// + /// # Panics + /// + /// The method panics if `bucket` represents an invalid bucket index, i.e. + /// is greater than or equal to `poll_count_histogram_num_buckets()`. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{self, Handle}; + /// + /// fn main() { + /// runtime::Builder::new_current_thread() + /// .enable_metrics_poll_count_histogram() + /// .build() + /// .unwrap() + /// .block_on(async { + /// let metrics = Handle::current().metrics(); + /// let buckets = metrics.poll_count_histogram_num_buckets(); + /// + /// for i in 0..buckets { + /// let range = metrics.poll_count_histogram_bucket_range(i); + /// println!("Histogram bucket {} range: {:?}", i, range); + /// } + /// }); + /// } + /// ``` + /// + /// [`metrics_poll_count_histogram_resolution()`]: + /// crate::runtime::Builder::metrics_poll_count_histogram_resolution + #[track_caller] + pub fn poll_count_histogram_bucket_range(&self, bucket: usize) -> Range { + self.handle + .inner + .worker_metrics(0) + .poll_count_histogram + .as_ref() + .map(|histogram| { + let range = histogram.bucket_range(bucket); + std::ops::Range { + start: Duration::from_nanos(range.start), + end: Duration::from_nanos(range.end), + } + }) + .unwrap_or_default() + } + + /// Returns the number of times the given worker polled tasks with a poll + /// duration within the given bucket's range. + /// + /// Each worker maintains its own histogram and the counts for each bucket + /// starts at zero when the runtime is created. Each time the worker polls a + /// task, it tracks the duration the task poll time took and increments the + /// associated bucket by 1. + /// + /// Each bucket is a monotonically increasing counter. It is never + /// decremented or reset to zero. + /// + /// # Arguments + /// + /// `worker` is the index of the worker being queried. The given value must + /// be between 0 and `num_workers()`. The index uniquely identifies a single + /// worker and will continue to identify the worker throughout the lifetime + /// of the runtime instance. + /// + /// `bucket` is the index of the bucket being queried. The bucket is scoped + /// to the worker. The range represented by the bucket can be queried by + /// calling [`poll_count_histogram_bucket_range()`]. Each worker maintains + /// identical bucket ranges. + /// + /// # Panics + /// + /// The method panics when `worker` represents an invalid worker, i.e. is + /// greater than or equal to `num_workers()` or if `bucket` represents an + /// invalid bucket. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::{self, Handle}; + /// + /// fn main() { + /// runtime::Builder::new_current_thread() + /// .enable_metrics_poll_count_histogram() + /// .build() + /// .unwrap() + /// .block_on(async { + /// let metrics = Handle::current().metrics(); + /// let buckets = metrics.poll_count_histogram_num_buckets(); + /// + /// for worker in 0..metrics.num_workers() { + /// for i in 0..buckets { + /// let count = metrics.poll_count_histogram_bucket_count(worker, i); + /// println!("Poll count {}", count); + /// } + /// } + /// }); + /// } + /// ``` + /// + /// [`poll_count_histogram_bucket_range()`]: crate::runtime::RuntimeMetrics::poll_count_histogram_bucket_range + #[track_caller] + pub fn poll_count_histogram_bucket_count(&self, worker: usize, bucket: usize) -> u64 { + self.handle + .inner + .worker_metrics(worker) + .poll_count_histogram + .as_ref() + .map(|histogram| histogram.get(bucket)) + .unwrap_or_default() + } + + /// Returns the number of tasks currently scheduled in the blocking + /// thread pool, spawned using `spawn_blocking`. + /// + /// This metric returns the **current** number of tasks pending in + /// blocking thread pool. As such, the returned value may increase + /// or decrease as new tasks are scheduled and processed. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.blocking_queue_depth(); + /// println!("{} tasks currently pending in the blocking thread pool", n); + /// } + /// ``` + pub fn blocking_queue_depth(&self) -> usize { + self.handle.inner.blocking_queue_depth() + } +} + +cfg_net! { + impl RuntimeMetrics { + /// Returns the number of file descriptors that have been registered with the + /// runtime's I/O driver. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let registered_fds = metrics.io_driver_fd_registered_count(); + /// println!("{} fds have been registered with the runtime's I/O driver.", registered_fds); + /// + /// let deregistered_fds = metrics.io_driver_fd_deregistered_count(); + /// + /// let current_fd_count = registered_fds - deregistered_fds; + /// println!("{} fds are currently registered by the runtime's I/O driver.", current_fd_count); + /// } + /// ``` + pub fn io_driver_fd_registered_count(&self) -> u64 { + self.with_io_driver_metrics(|m| { + m.fd_registered_count.load(Relaxed) + }) + } + + /// Returns the number of file descriptors that have been deregistered by the + /// runtime's I/O driver. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.io_driver_fd_deregistered_count(); + /// println!("{} fds have been deregistered by the runtime's I/O driver.", n); + /// } + /// ``` + pub fn io_driver_fd_deregistered_count(&self) -> u64 { + self.with_io_driver_metrics(|m| { + m.fd_deregistered_count.load(Relaxed) + }) + } + + /// Returns the number of ready events processed by the runtime's + /// I/O driver. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Handle; + /// + /// #[tokio::main] + /// async fn main() { + /// let metrics = Handle::current().metrics(); + /// + /// let n = metrics.io_driver_ready_count(); + /// println!("{} ready events processed by the runtime's I/O driver.", n); + /// } + /// ``` + pub fn io_driver_ready_count(&self) -> u64 { + self.with_io_driver_metrics(|m| m.ready_count.load(Relaxed)) + } + + fn with_io_driver_metrics(&self, f: F) -> u64 + where + F: Fn(&super::IoDriverMetrics) -> u64, + { + // TODO: Investigate if this should return 0, most of our metrics always increase + // thus this breaks that guarantee. + self.handle + .inner + .driver() + .io + .as_ref() + .map(|h| f(&h.metrics)) + .unwrap_or(0) + } } } diff --git a/third_party/rust/tokio/src/runtime/metrics/scheduler.rs b/third_party/rust/tokio/src/runtime/metrics/scheduler.rs index d1ba3b644201..d9f8edfaabcb 100644 --- a/third_party/rust/tokio/src/runtime/metrics/scheduler.rs +++ b/third_party/rust/tokio/src/runtime/metrics/scheduler.rs @@ -11,12 +11,14 @@ use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; pub(crate) struct SchedulerMetrics { /// Number of tasks that are scheduled from outside the runtime. pub(super) remote_schedule_count: AtomicU64, + pub(super) budget_forced_yield_count: AtomicU64, } impl SchedulerMetrics { pub(crate) fn new() -> SchedulerMetrics { SchedulerMetrics { remote_schedule_count: AtomicU64::new(0), + budget_forced_yield_count: AtomicU64::new(0), } } @@ -24,4 +26,9 @@ impl SchedulerMetrics { pub(crate) fn inc_remote_schedule_count(&self) { self.remote_schedule_count.fetch_add(1, Relaxed); } + + /// Increment the number of tasks forced to yield due to budget exhaustion + pub(crate) fn inc_budget_forced_yield_count(&self) { + self.budget_forced_yield_count.fetch_add(1, Relaxed); + } } diff --git a/third_party/rust/tokio/src/runtime/metrics/worker.rs b/third_party/rust/tokio/src/runtime/metrics/worker.rs index c9b85e48e4c0..e0f23e6a08ed 100644 --- a/third_party/rust/tokio/src/runtime/metrics/worker.rs +++ b/third_party/rust/tokio/src/runtime/metrics/worker.rs @@ -1,7 +1,9 @@ use crate::loom::sync::atomic::Ordering::Relaxed; use crate::loom::sync::atomic::{AtomicU64, AtomicUsize}; +use crate::runtime::metrics::Histogram; +use crate::runtime::Config; -/// Retreive runtime worker metrics. +/// Retrieve runtime worker metrics. /// /// **Note**: This is an [unstable API][unstable]. The public API of this type /// may break in 1.x releases. See [the documentation on unstable @@ -17,9 +19,12 @@ pub(crate) struct WorkerMetrics { /// Number of times the worker woke then parked again without doing work. pub(crate) noop_count: AtomicU64, - /// Number of times the worker attempted to steal. + /// Number of tasks the worker stole. pub(crate) steal_count: AtomicU64, + /// Number of times the worker stole + pub(crate) steal_operations: AtomicU64, + /// Number of tasks the worker polled. pub(crate) poll_count: AtomicU64, @@ -35,19 +40,33 @@ pub(crate) struct WorkerMetrics { /// Number of tasks currently in the local queue. Used only by the /// current-thread scheduler. pub(crate) queue_depth: AtomicUsize, + + /// If `Some`, tracks the the number of polls by duration range. + pub(super) poll_count_histogram: Option, } impl WorkerMetrics { + pub(crate) fn from_config(config: &Config) -> WorkerMetrics { + let mut worker_metrics = WorkerMetrics::new(); + worker_metrics.poll_count_histogram = config + .metrics_poll_count_histogram + .as_ref() + .map(|histogram_builder| histogram_builder.build()); + worker_metrics + } + pub(crate) fn new() -> WorkerMetrics { WorkerMetrics { park_count: AtomicU64::new(0), noop_count: AtomicU64::new(0), steal_count: AtomicU64::new(0), + steal_operations: AtomicU64::new(0), poll_count: AtomicU64::new(0), overflow_count: AtomicU64::new(0), busy_duration_total: AtomicU64::new(0), local_schedule_count: AtomicU64::new(0), queue_depth: AtomicUsize::new(0), + poll_count_histogram: None, } } diff --git a/third_party/rust/tokio/src/runtime/mod.rs b/third_party/rust/tokio/src/runtime/mod.rs index 7c381b0bbd0f..cb198f51f0d2 100644 --- a/third_party/rust/tokio/src/runtime/mod.rs +++ b/third_party/rust/tokio/src/runtime/mod.rs @@ -137,7 +137,7 @@ //! use tokio::runtime; //! //! # fn main() -> Result<(), Box> { -//! let basic_rt = runtime::Builder::new_current_thread() +//! let rt = runtime::Builder::new_current_thread() //! .build()?; //! # Ok(()) } //! ``` @@ -156,9 +156,11 @@ //! multi-thread scheduler spawns threads to schedule tasks and for `spawn_blocking` //! calls. //! -//! While the `Runtime` is active, threads may shutdown after periods of being -//! idle. Once `Runtime` is dropped, all runtime threads are forcibly shutdown. -//! Any tasks that have not yet completed will be dropped. +//! While the `Runtime` is active, threads may shut down after periods of being +//! idle. Once `Runtime` is dropped, all runtime threads have usually been +//! terminated, but in the presence of unstoppable spawned work are not +//! guaranteed to have been terminated. See the +//! [struct level documentation](Runtime#shutdown) for more details. //! //! [tasks]: crate::task //! [`Runtime`]: Runtime @@ -166,7 +168,6 @@ //! [`tokio::main`]: ../attr.main.html //! [runtime builder]: crate::runtime::Builder //! [`Runtime::new`]: crate::runtime::Runtime::new -//! [`Builder::basic_scheduler`]: crate::runtime::Builder::basic_scheduler //! [`Builder::threaded_scheduler`]: crate::runtime::Builder::threaded_scheduler //! [`Builder::enable_io`]: crate::runtime::Builder::enable_io //! [`Builder::enable_time`]: crate::runtime::Builder::enable_time @@ -174,32 +175,44 @@ // At the top due to macros #[cfg(test)] -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm))] #[macro_use] mod tests; -pub(crate) mod enter; +pub(crate) mod context; -pub(crate) mod task; +pub(crate) mod coop; -cfg_metrics! { - mod metrics; - pub use metrics::RuntimeMetrics; +pub(crate) mod park; - pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics}; +mod driver; + +pub(crate) mod scheduler; + +cfg_io_driver_impl! { + pub(crate) mod io; } -cfg_not_metrics! { - pub(crate) mod metrics; - pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch}; +cfg_process_driver! { + mod process; +} + +cfg_time! { + pub(crate) mod time; +} + +cfg_signal_internal_and_unix! { + pub(crate) mod signal; } cfg_rt! { - mod basic_scheduler; - use basic_scheduler::BasicScheduler; + pub(crate) mod task; + + mod config; + use config::Config; mod blocking; - use blocking::BlockingPool; + #[cfg_attr(tokio_wasi, allow(unused_imports))] pub(crate) use blocking::spawn_blocking; cfg_trace! { @@ -212,412 +225,41 @@ cfg_rt! { mod builder; pub use self::builder::Builder; + cfg_unstable! { + pub use self::builder::UnhandledPanic; + pub use crate::util::rand::RngSeed; + } - pub(crate) mod context; - pub(crate) mod driver; - - use self::enter::enter; + cfg_taskdump! { + pub mod dump; + pub use dump::Dump; + } mod handle; pub use handle::{EnterGuard, Handle, TryCurrentError}; - mod spawner; - use self::spawner::Spawner; -} + mod runtime; + pub use runtime::{Runtime, RuntimeFlavor}; -cfg_rt_multi_thread! { - mod park; - use park::Parker; -} + mod thread_id; + pub(crate) use thread_id::ThreadId; -cfg_rt_multi_thread! { - mod queue; + cfg_metrics! { + mod metrics; + pub use metrics::{RuntimeMetrics, HistogramScale}; - pub(crate) mod thread_pool; - use self::thread_pool::ThreadPool; -} + pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics, HistogramBuilder}; -cfg_rt! { - use crate::task::JoinHandle; - - use std::future::Future; - use std::time::Duration; - - /// The Tokio runtime. - /// - /// The runtime provides an I/O driver, task scheduler, [timer], and - /// blocking pool, necessary for running asynchronous tasks. - /// - /// Instances of `Runtime` can be created using [`new`], or [`Builder`]. - /// However, most users will use the `#[tokio::main]` annotation on their - /// entry point instead. - /// - /// See [module level][mod] documentation for more details. - /// - /// # Shutdown - /// - /// Shutting down the runtime is done by dropping the value. The current - /// thread will block until the shut down operation has completed. - /// - /// * Drain any scheduled work queues. - /// * Drop any futures that have not yet completed. - /// * Drop the reactor. - /// - /// Once the reactor has dropped, any outstanding I/O resources bound to - /// that reactor will no longer function. Calling any method on them will - /// result in an error. - /// - /// # Sharing - /// - /// The Tokio runtime implements `Sync` and `Send` to allow you to wrap it - /// in a `Arc`. Most fn take `&self` to allow you to call them concurrently - /// across multiple threads. - /// - /// Calls to `shutdown` and `shutdown_timeout` require exclusive ownership of - /// the runtime type and this can be achieved via `Arc::try_unwrap` when only - /// one strong count reference is left over. - /// - /// [timer]: crate::time - /// [mod]: index.html - /// [`new`]: method@Self::new - /// [`Builder`]: struct@Builder - #[derive(Debug)] - pub struct Runtime { - /// Task executor - kind: Kind, - - /// Handle to runtime, also contains driver handles - handle: Handle, - - /// Blocking pool handle, used to signal shutdown - blocking_pool: BlockingPool, + cfg_net! { + pub(crate) use metrics::IoDriverMetrics; + } } - /// The runtime executor is either a thread-pool or a current-thread executor. - #[derive(Debug)] - enum Kind { - /// Execute all tasks on the current-thread. - CurrentThread(BasicScheduler), - - /// Execute tasks across multiple threads. - #[cfg(feature = "rt-multi-thread")] - ThreadPool(ThreadPool), + cfg_not_metrics! { + pub(crate) mod metrics; + pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch, HistogramBuilder}; } /// After thread starts / before thread stops type Callback = std::sync::Arc; - - impl Runtime { - /// Creates a new runtime instance with default configuration values. - /// - /// This results in the multi threaded scheduler, I/O driver, and time driver being - /// initialized. - /// - /// Most applications will not need to call this function directly. Instead, - /// they will use the [`#[tokio::main]` attribute][main]. When a more complex - /// configuration is necessary, the [runtime builder] may be used. - /// - /// See [module level][mod] documentation for more details. - /// - /// # Examples - /// - /// Creating a new `Runtime` with default configuration values. - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// let rt = Runtime::new() - /// .unwrap(); - /// - /// // Use the runtime... - /// ``` - /// - /// [mod]: index.html - /// [main]: ../attr.main.html - /// [threaded scheduler]: index.html#threaded-scheduler - /// [basic scheduler]: index.html#basic-scheduler - /// [runtime builder]: crate::runtime::Builder - #[cfg(feature = "rt-multi-thread")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] - pub fn new() -> std::io::Result { - Builder::new_multi_thread().enable_all().build() - } - - /// Returns a handle to the runtime's spawner. - /// - /// The returned handle can be used to spawn tasks that run on this runtime, and can - /// be cloned to allow moving the `Handle` to other threads. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// let rt = Runtime::new() - /// .unwrap(); - /// - /// let handle = rt.handle(); - /// - /// // Use the handle... - /// ``` - pub fn handle(&self) -> &Handle { - &self.handle - } - - /// Spawns a future onto the Tokio runtime. - /// - /// This spawns the given future onto the runtime's executor, usually a - /// thread pool. The thread pool is then responsible for polling the future - /// until it completes. - /// - /// See [module level][mod] documentation for more details. - /// - /// [mod]: index.html - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// # fn dox() { - /// // Create the runtime - /// let rt = Runtime::new().unwrap(); - /// - /// // Spawn a future onto the runtime - /// rt.spawn(async { - /// println!("now running on a worker thread"); - /// }); - /// # } - /// ``` - #[track_caller] - pub fn spawn(&self, future: F) -> JoinHandle - where - F: Future + Send + 'static, - F::Output: Send + 'static, - { - self.handle.spawn(future) - } - - /// Runs the provided function on an executor dedicated to blocking operations. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// # fn dox() { - /// // Create the runtime - /// let rt = Runtime::new().unwrap(); - /// - /// // Spawn a blocking function onto the runtime - /// rt.spawn_blocking(|| { - /// println!("now running on a worker thread"); - /// }); - /// # } - #[track_caller] - pub fn spawn_blocking(&self, func: F) -> JoinHandle - where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, - { - self.handle.spawn_blocking(func) - } - - /// Runs a future to completion on the Tokio runtime. This is the - /// runtime's entry point. - /// - /// This runs the given future on the current thread, blocking until it is - /// complete, and yielding its resolved result. Any tasks or timers - /// which the future spawns internally will be executed on the runtime. - /// - /// # Multi thread scheduler - /// - /// When the multi thread scheduler is used this will allow futures - /// to run within the io driver and timer context of the overall runtime. - /// - /// # Current thread scheduler - /// - /// When the current thread scheduler is enabled `block_on` - /// can be called concurrently from multiple threads. The first call - /// will take ownership of the io and timer drivers. This means - /// other threads which do not own the drivers will hook into that one. - /// When the first `block_on` completes, other threads will be able to - /// "steal" the driver to allow continued execution of their futures. - /// - /// # Panics - /// - /// This function panics if the provided future panics, or if called within an - /// asynchronous execution context. - /// - /// # Examples - /// - /// ```no_run - /// use tokio::runtime::Runtime; - /// - /// // Create the runtime - /// let rt = Runtime::new().unwrap(); - /// - /// // Execute the future, blocking the current thread until completion - /// rt.block_on(async { - /// println!("hello"); - /// }); - /// ``` - /// - /// [handle]: fn@Handle::block_on - #[track_caller] - pub fn block_on(&self, future: F) -> F::Output { - #[cfg(all(tokio_unstable, feature = "tracing"))] - let future = crate::util::trace::task(future, "block_on", None); - - let _enter = self.enter(); - - match &self.kind { - Kind::CurrentThread(exec) => exec.block_on(future), - #[cfg(feature = "rt-multi-thread")] - Kind::ThreadPool(exec) => exec.block_on(future), - } - } - - /// Enters the runtime context. - /// - /// This allows you to construct types that must have an executor - /// available on creation such as [`Sleep`] or [`TcpStream`]. It will - /// also allow you to call methods such as [`tokio::spawn`]. - /// - /// [`Sleep`]: struct@crate::time::Sleep - /// [`TcpStream`]: struct@crate::net::TcpStream - /// [`tokio::spawn`]: fn@crate::spawn - /// - /// # Example - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// fn function_that_spawns(msg: String) { - /// // Had we not used `rt.enter` below, this would panic. - /// tokio::spawn(async move { - /// println!("{}", msg); - /// }); - /// } - /// - /// fn main() { - /// let rt = Runtime::new().unwrap(); - /// - /// let s = "Hello World!".to_string(); - /// - /// // By entering the context, we tie `tokio::spawn` to this executor. - /// let _guard = rt.enter(); - /// function_that_spawns(s); - /// } - /// ``` - pub fn enter(&self) -> EnterGuard<'_> { - self.handle.enter() - } - - /// Shuts down the runtime, waiting for at most `duration` for all spawned - /// task to shutdown. - /// - /// Usually, dropping a `Runtime` handle is sufficient as tasks are able to - /// shutdown in a timely fashion. However, dropping a `Runtime` will wait - /// indefinitely for all tasks to terminate, and there are cases where a long - /// blocking task has been spawned, which can block dropping `Runtime`. - /// - /// In this case, calling `shutdown_timeout` with an explicit wait timeout - /// can work. The `shutdown_timeout` will signal all tasks to shutdown and - /// will wait for at most `duration` for all spawned tasks to terminate. If - /// `timeout` elapses before all tasks are dropped, the function returns and - /// outstanding tasks are potentially leaked. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime::Runtime; - /// use tokio::task; - /// - /// use std::thread; - /// use std::time::Duration; - /// - /// fn main() { - /// let runtime = Runtime::new().unwrap(); - /// - /// runtime.block_on(async move { - /// task::spawn_blocking(move || { - /// thread::sleep(Duration::from_secs(10_000)); - /// }); - /// }); - /// - /// runtime.shutdown_timeout(Duration::from_millis(100)); - /// } - /// ``` - pub fn shutdown_timeout(mut self, duration: Duration) { - // Wakeup and shutdown all the worker threads - self.handle.clone().shutdown(); - self.blocking_pool.shutdown(Some(duration)); - } - - /// Shuts down the runtime, without waiting for any spawned tasks to shutdown. - /// - /// This can be useful if you want to drop a runtime from within another runtime. - /// Normally, dropping a runtime will block indefinitely for spawned blocking tasks - /// to complete, which would normally not be permitted within an asynchronous context. - /// By calling `shutdown_background()`, you can drop the runtime from such a context. - /// - /// Note however, that because we do not wait for any blocking tasks to complete, this - /// may result in a resource leak (in that any blocking tasks are still running until they - /// return. - /// - /// This function is equivalent to calling `shutdown_timeout(Duration::of_nanos(0))`. - /// - /// ``` - /// use tokio::runtime::Runtime; - /// - /// fn main() { - /// let runtime = Runtime::new().unwrap(); - /// - /// runtime.block_on(async move { - /// let inner_runtime = Runtime::new().unwrap(); - /// // ... - /// inner_runtime.shutdown_background(); - /// }); - /// } - /// ``` - pub fn shutdown_background(self) { - self.shutdown_timeout(Duration::from_nanos(0)) - } - } - - #[allow(clippy::single_match)] // there are comments in the error branch, so we don't want if-let - impl Drop for Runtime { - fn drop(&mut self) { - match &mut self.kind { - Kind::CurrentThread(basic) => { - // This ensures that tasks spawned on the basic runtime are dropped inside the - // runtime's context. - match self::context::try_enter(self.handle.clone()) { - Some(guard) => basic.set_context_guard(guard), - None => { - // The context thread-local has already been destroyed. - // - // We don't set the guard in this case. Calls to tokio::spawn in task - // destructors would fail regardless if this happens. - }, - } - }, - #[cfg(feature = "rt-multi-thread")] - Kind::ThreadPool(_) => { - // The threaded scheduler drops its tasks on its worker threads, which is - // already in the runtime's context. - }, - } - } - } - - cfg_metrics! { - impl Runtime { - /// TODO - pub fn metrics(&self) -> RuntimeMetrics { - self.handle.metrics() - } - } - } } diff --git a/third_party/rust/tokio/src/runtime/park.rs b/third_party/rust/tokio/src/runtime/park.rs index 033b9f20beeb..2392846abed7 100644 --- a/third_party/rust/tokio/src/runtime/park.rs +++ b/third_party/rust/tokio/src/runtime/park.rs @@ -1,153 +1,102 @@ -//! Parks the runtime. -//! -//! A combination of the various resource driver park handles. +#![cfg_attr(not(feature = "full"), allow(dead_code))] use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::{Arc, Condvar, Mutex}; -use crate::loom::thread; -use crate::park::{Park, Unpark}; -use crate::runtime::driver::Driver; -use crate::util::TryLock; use std::sync::atomic::Ordering::SeqCst; use std::time::Duration; -pub(crate) struct Parker { +#[derive(Debug)] +pub(crate) struct ParkThread { inner: Arc, } -pub(crate) struct Unparker { +/// Unblocks a thread that was blocked by `ParkThread`. +#[derive(Clone, Debug)] +pub(crate) struct UnparkThread { inner: Arc, } +#[derive(Debug)] struct Inner { - /// Avoids entering the park if possible state: AtomicUsize, - - /// Used to coordinate access to the driver / condvar mutex: Mutex<()>, - - /// Condvar to block on if the driver is unavailable. condvar: Condvar, - - /// Resource (I/O, time, ...) driver - shared: Arc, } const EMPTY: usize = 0; -const PARKED_CONDVAR: usize = 1; -const PARKED_DRIVER: usize = 2; -const NOTIFIED: usize = 3; +const PARKED: usize = 1; +const NOTIFIED: usize = 2; -/// Shared across multiple Parker handles -struct Shared { - /// Shared driver. Only one thread at a time can use this - driver: TryLock, - - /// Unpark handle - handle: ::Unpark, +tokio_thread_local! { + static CURRENT_PARKER: ParkThread = ParkThread::new(); } -impl Parker { - pub(crate) fn new(driver: Driver) -> Parker { - let handle = driver.unpark(); +// Bit of a hack, but it is only for loom +#[cfg(loom)] +tokio_thread_local! { + static CURRENT_THREAD_PARK_COUNT: AtomicUsize = AtomicUsize::new(0); +} - Parker { +// ==== impl ParkThread ==== + +impl ParkThread { + pub(crate) fn new() -> Self { + Self { inner: Arc::new(Inner { state: AtomicUsize::new(EMPTY), mutex: Mutex::new(()), condvar: Condvar::new(), - shared: Arc::new(Shared { - driver: TryLock::new(driver), - handle, - }), }), } } -} -impl Clone for Parker { - fn clone(&self) -> Parker { - Parker { - inner: Arc::new(Inner { - state: AtomicUsize::new(EMPTY), - mutex: Mutex::new(()), - condvar: Condvar::new(), - shared: self.inner.shared.clone(), - }), - } - } -} - -impl Park for Parker { - type Unpark = Unparker; - type Error = (); - - fn unpark(&self) -> Unparker { - Unparker { - inner: self.inner.clone(), - } + pub(crate) fn unpark(&self) -> UnparkThread { + let inner = self.inner.clone(); + UnparkThread { inner } } - fn park(&mut self) -> Result<(), Self::Error> { + pub(crate) fn park(&mut self) { + #[cfg(loom)] + CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst)); self.inner.park(); - Ok(()) } - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - // Only parking with zero is supported... - assert_eq!(duration, Duration::from_millis(0)); + pub(crate) fn park_timeout(&mut self, duration: Duration) { + #[cfg(loom)] + CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst)); - if let Some(mut driver) = self.inner.shared.driver.try_lock() { - driver.park_timeout(duration).map_err(|_| ()) - } else { - Ok(()) - } + // Wasm doesn't have threads, so just sleep. + #[cfg(not(tokio_wasm))] + self.inner.park_timeout(duration); + #[cfg(tokio_wasm)] + std::thread::sleep(duration); } - fn shutdown(&mut self) { + pub(crate) fn shutdown(&mut self) { self.inner.shutdown(); } } -impl Unpark for Unparker { - fn unpark(&self) { - self.inner.unpark(); - } -} +// ==== impl Inner ==== impl Inner { /// Parks the current thread for at most `dur`. fn park(&self) { - for _ in 0..3 { - // If we were previously notified then we consume this notification and - // return quickly. - if self - .state - .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) - .is_ok() - { - return; - } - - thread::yield_now(); + // If we were previously notified then we consume this notification and + // return quickly. + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + return; } - if let Some(mut driver) = self.shared.driver.try_lock() { - self.park_driver(&mut driver); - } else { - self.park_condvar(); - } - } - - fn park_condvar(&self) { // Otherwise we need to coordinate going to sleep let mut m = self.mutex.lock(); - match self - .state - .compare_exchange(EMPTY, PARKED_CONDVAR, SeqCst, SeqCst) - { + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { Ok(_) => {} Err(NOTIFIED) => { // We must read here, even though we know it will be `NOTIFIED`. @@ -180,33 +129,44 @@ impl Inner { } } - fn park_driver(&self, driver: &mut Driver) { - match self + fn park_timeout(&self, dur: Duration) { + // Like `park` above we have a fast path for an already-notified thread, + // and afterwards we start coordinating for a sleep. Return quickly. + if self .state - .compare_exchange(EMPTY, PARKED_DRIVER, SeqCst, SeqCst) + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() { + return; + } + + if dur == Duration::from_millis(0) { + return; + } + + let m = self.mutex.lock(); + + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { Ok(_) => {} Err(NOTIFIED) => { - // We must read here, even though we know it will be `NOTIFIED`. - // This is because `unpark` may have been called again since we read - // `NOTIFIED` in the `compare_exchange` above. We must perform an - // acquire operation that synchronizes with that `unpark` to observe - // any writes it made before the call to unpark. To do that we must - // read from the write it made to `state`. + // We must read again here, see `park`. let old = self.state.swap(EMPTY, SeqCst); debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); return; } - Err(actual) => panic!("inconsistent park state; actual = {}", actual), + Err(actual) => panic!("inconsistent park_timeout state; actual = {}", actual), } - // TODO: don't unwrap - driver.park().unwrap(); + // Wait with a timeout, and if we spuriously wake up or otherwise wake up + // from a notification, we just want to unconditionally set the state back to + // empty, either consuming a notification or un-flagging ourselves as + // parked. + let (_m, _result) = self.condvar.wait_timeout(m, dur).unwrap(); match self.state.swap(EMPTY, SeqCst) { - NOTIFIED => {} // got a notification, hurray! - PARKED_DRIVER => {} // no notification, alas + NOTIFIED => {} // got a notification, hurray! + PARKED => {} // no notification, alas n => panic!("inconsistent park_timeout state: {}", n), } } @@ -218,15 +178,12 @@ impl Inner { // is already `NOTIFIED`. That is why this must be a swap rather than a // compare-and-swap that returns if it reads `NOTIFIED` on failure. match self.state.swap(NOTIFIED, SeqCst) { - EMPTY => {} // no one was waiting - NOTIFIED => {} // already unparked - PARKED_CONDVAR => self.unpark_condvar(), - PARKED_DRIVER => self.unpark_driver(), - actual => panic!("inconsistent state in unpark; actual = {}", actual), + EMPTY => return, // no one was waiting + NOTIFIED => return, // already unparked + PARKED => {} // gotta go wake someone up + _ => panic!("inconsistent state in unpark"), } - } - fn unpark_condvar(&self) { // There is a period between when the parked thread sets `state` to // `PARKED` (or last checked `state` in the case of a spurious wake // up) and when it actually waits on `cvar`. If we were to notify @@ -243,15 +200,149 @@ impl Inner { self.condvar.notify_one() } - fn unpark_driver(&self) { - self.shared.handle.unpark(); - } - fn shutdown(&self) { - if let Some(mut driver) = self.shared.driver.try_lock() { - driver.shutdown(); - } - self.condvar.notify_all(); } } + +impl Default for ParkThread { + fn default() -> Self { + Self::new() + } +} + +// ===== impl UnparkThread ===== + +impl UnparkThread { + pub(crate) fn unpark(&self) { + self.inner.unpark(); + } +} + +use crate::loom::thread::AccessError; +use std::future::Future; +use std::marker::PhantomData; +use std::mem; +use std::rc::Rc; +use std::task::{RawWaker, RawWakerVTable, Waker}; + +/// Blocks the current thread using a condition variable. +#[derive(Debug)] +pub(crate) struct CachedParkThread { + _anchor: PhantomData>, +} + +impl CachedParkThread { + /// Creates a new `ParkThread` handle for the current thread. + /// + /// This type cannot be moved to other threads, so it should be created on + /// the thread that the caller intends to park. + pub(crate) fn new() -> CachedParkThread { + CachedParkThread { + _anchor: PhantomData, + } + } + + pub(crate) fn waker(&self) -> Result { + self.unpark().map(|unpark| unpark.into_waker()) + } + + fn unpark(&self) -> Result { + self.with_current(|park_thread| park_thread.unpark()) + } + + pub(crate) fn park(&mut self) { + self.with_current(|park_thread| park_thread.inner.park()) + .unwrap(); + } + + pub(crate) fn park_timeout(&mut self, duration: Duration) { + self.with_current(|park_thread| park_thread.inner.park_timeout(duration)) + .unwrap(); + } + + /// Gets a reference to the `ParkThread` handle for this thread. + fn with_current(&self, f: F) -> Result + where + F: FnOnce(&ParkThread) -> R, + { + CURRENT_PARKER.try_with(|inner| f(inner)) + } + + pub(crate) fn block_on(&mut self, f: F) -> Result { + use std::task::Context; + use std::task::Poll::Ready; + + // `get_unpark()` should not return a Result + let waker = self.waker()?; + let mut cx = Context::from_waker(&waker); + + pin!(f); + + loop { + if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) { + return Ok(v); + } + + self.park(); + } + } +} + +impl UnparkThread { + pub(crate) fn into_waker(self) -> Waker { + unsafe { + let raw = unparker_to_raw_waker(self.inner); + Waker::from_raw(raw) + } + } +} + +impl Inner { + #[allow(clippy::wrong_self_convention)] + fn into_raw(this: Arc) -> *const () { + Arc::into_raw(this) as *const () + } + + unsafe fn from_raw(ptr: *const ()) -> Arc { + Arc::from_raw(ptr as *const Inner) + } +} + +unsafe fn unparker_to_raw_waker(unparker: Arc) -> RawWaker { + RawWaker::new( + Inner::into_raw(unparker), + &RawWakerVTable::new(clone, wake, wake_by_ref, drop_waker), + ) +} + +unsafe fn clone(raw: *const ()) -> RawWaker { + let unparker = Inner::from_raw(raw); + + // Increment the ref count + mem::forget(unparker.clone()); + + unparker_to_raw_waker(unparker) +} + +unsafe fn drop_waker(raw: *const ()) { + let _ = Inner::from_raw(raw); +} + +unsafe fn wake(raw: *const ()) { + let unparker = Inner::from_raw(raw); + unparker.unpark(); +} + +unsafe fn wake_by_ref(raw: *const ()) { + let unparker = Inner::from_raw(raw); + unparker.unpark(); + + // We don't actually own a reference to the unparker + mem::forget(unparker); +} + +#[cfg(loom)] +pub(crate) fn current_thread_park_count() -> usize { + CURRENT_THREAD_PARK_COUNT.with(|count| count.load(SeqCst)) +} diff --git a/third_party/rust/tokio/src/process/unix/driver.rs b/third_party/rust/tokio/src/runtime/process.rs similarity index 55% rename from third_party/rust/tokio/src/process/unix/driver.rs rename to third_party/rust/tokio/src/runtime/process.rs index 84dc8fbd027a..df339b0e729f 100644 --- a/third_party/rust/tokio/src/process/unix/driver.rs +++ b/third_party/rust/tokio/src/runtime/process.rs @@ -2,11 +2,10 @@ //! Process driver. -use crate::park::Park; use crate::process::unix::GlobalOrphanQueue; -use crate::signal::unix::driver::{Driver as SignalDriver, Handle as SignalHandle}; +use crate::runtime::driver; +use crate::runtime::signal::{Driver as SignalDriver, Handle as SignalHandle}; -use std::io; use std::time::Duration; /// Responsible for cleaning up orphaned child processes on Unix platforms. @@ -28,31 +27,18 @@ impl Driver { signal_handle, } } -} -// ===== impl Park for Driver ===== - -impl Park for Driver { - type Unpark = ::Unpark; - type Error = io::Error; - - fn unpark(&self) -> Self::Unpark { - self.park.unpark() - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.park.park()?; + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.park.park(handle); GlobalOrphanQueue::reap_orphans(&self.signal_handle); - Ok(()) } - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.park.park_timeout(duration)?; + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + self.park.park_timeout(handle, duration); GlobalOrphanQueue::reap_orphans(&self.signal_handle); - Ok(()) } - fn shutdown(&mut self) { - self.park.shutdown() + pub(crate) fn shutdown(&mut self, handle: &driver::Handle) { + self.park.shutdown(handle) } } diff --git a/third_party/rust/tokio/src/runtime/runtime.rs b/third_party/rust/tokio/src/runtime/runtime.rs new file mode 100644 index 000000000000..3f3499975839 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/runtime.rs @@ -0,0 +1,445 @@ +use crate::runtime::blocking::BlockingPool; +use crate::runtime::scheduler::CurrentThread; +use crate::runtime::{context, EnterGuard, Handle}; +use crate::task::JoinHandle; + +use std::future::Future; +use std::time::Duration; + +cfg_rt_multi_thread! { + use crate::runtime::Builder; + use crate::runtime::scheduler::MultiThread; +} + +/// The Tokio runtime. +/// +/// The runtime provides an I/O driver, task scheduler, [timer], and +/// blocking pool, necessary for running asynchronous tasks. +/// +/// Instances of `Runtime` can be created using [`new`], or [`Builder`]. +/// However, most users will use the `#[tokio::main]` annotation on their +/// entry point instead. +/// +/// See [module level][mod] documentation for more details. +/// +/// # Shutdown +/// +/// Shutting down the runtime is done by dropping the value, or calling +/// [`Runtime::shutdown_background`] or [`Runtime::shutdown_timeout`]. +/// +/// Tasks spawned through [`Runtime::spawn`] keep running until they yield. +/// Then they are dropped. They are not *guaranteed* to run to completion, but +/// *might* do so if they do not yield until completion. +/// +/// Blocking functions spawned through [`Runtime::spawn_blocking`] keep running +/// until they return. +/// +/// The thread initiating the shutdown blocks until all spawned work has been +/// stopped. This can take an indefinite amount of time. The `Drop` +/// implementation waits forever for this. +/// +/// `shutdown_background` and `shutdown_timeout` can be used if waiting forever +/// is undesired. When the timeout is reached, spawned work that did not stop +/// in time and threads running it are leaked. The work continues to run until +/// one of the stopping conditions is fulfilled, but the thread initiating the +/// shutdown is unblocked. +/// +/// Once the runtime has been dropped, any outstanding I/O resources bound to +/// it will no longer function. Calling any method on them will result in an +/// error. +/// +/// # Sharing +/// +/// The Tokio runtime implements `Sync` and `Send` to allow you to wrap it +/// in a `Arc`. Most fn take `&self` to allow you to call them concurrently +/// across multiple threads. +/// +/// Calls to `shutdown` and `shutdown_timeout` require exclusive ownership of +/// the runtime type and this can be achieved via `Arc::try_unwrap` when only +/// one strong count reference is left over. +/// +/// [timer]: crate::time +/// [mod]: index.html +/// [`new`]: method@Self::new +/// [`Builder`]: struct@Builder +#[derive(Debug)] +pub struct Runtime { + /// Task scheduler + scheduler: Scheduler, + + /// Handle to runtime, also contains driver handles + handle: Handle, + + /// Blocking pool handle, used to signal shutdown + blocking_pool: BlockingPool, +} + +/// The flavor of a `Runtime`. +/// +/// This is the return type for [`Handle::runtime_flavor`](crate::runtime::Handle::runtime_flavor()). +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum RuntimeFlavor { + /// The flavor that executes all tasks on the current thread. + CurrentThread, + /// The flavor that executes tasks across multiple threads. + MultiThread, +} + +/// The runtime scheduler is either a multi-thread or a current-thread executor. +#[derive(Debug)] +pub(super) enum Scheduler { + /// Execute all tasks on the current-thread. + CurrentThread(CurrentThread), + + /// Execute tasks across multiple threads. + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + MultiThread(MultiThread), +} + +impl Runtime { + pub(super) fn from_parts( + scheduler: Scheduler, + handle: Handle, + blocking_pool: BlockingPool, + ) -> Runtime { + Runtime { + scheduler, + handle, + blocking_pool, + } + } + + cfg_not_wasi! { + /// Creates a new runtime instance with default configuration values. + /// + /// This results in the multi threaded scheduler, I/O driver, and time driver being + /// initialized. + /// + /// Most applications will not need to call this function directly. Instead, + /// they will use the [`#[tokio::main]` attribute][main]. When a more complex + /// configuration is necessary, the [runtime builder] may be used. + /// + /// See [module level][mod] documentation for more details. + /// + /// # Examples + /// + /// Creating a new `Runtime` with default configuration values. + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// let rt = Runtime::new() + /// .unwrap(); + /// + /// // Use the runtime... + /// ``` + /// + /// [mod]: index.html + /// [main]: ../attr.main.html + /// [threaded scheduler]: index.html#threaded-scheduler + /// [runtime builder]: crate::runtime::Builder + #[cfg(feature = "rt-multi-thread")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))] + pub fn new() -> std::io::Result { + Builder::new_multi_thread().enable_all().build() + } + } + + /// Returns a handle to the runtime's spawner. + /// + /// The returned handle can be used to spawn tasks that run on this runtime, and can + /// be cloned to allow moving the `Handle` to other threads. + /// + /// Calling [`Handle::block_on`] on a handle to a `current_thread` runtime is error-prone. + /// Refer to the documentation of [`Handle::block_on`] for more. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// let rt = Runtime::new() + /// .unwrap(); + /// + /// let handle = rt.handle(); + /// + /// // Use the handle... + /// ``` + pub fn handle(&self) -> &Handle { + &self.handle + } + + /// Spawns a future onto the Tokio runtime. + /// + /// This spawns the given future onto the runtime's executor, usually a + /// thread pool. The thread pool is then responsible for polling the future + /// until it completes. + /// + /// The provided future will start running in the background immediately + /// when `spawn` is called, even if you don't await the returned + /// `JoinHandle`. + /// + /// See [module level][mod] documentation for more details. + /// + /// [mod]: index.html + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// # fn dox() { + /// // Create the runtime + /// let rt = Runtime::new().unwrap(); + /// + /// // Spawn a future onto the runtime + /// rt.spawn(async { + /// println!("now running on a worker thread"); + /// }); + /// # } + /// ``` + #[track_caller] + pub fn spawn(&self, future: F) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + self.handle.spawn(future) + } + + /// Runs the provided function on an executor dedicated to blocking operations. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// # fn dox() { + /// // Create the runtime + /// let rt = Runtime::new().unwrap(); + /// + /// // Spawn a blocking function onto the runtime + /// rt.spawn_blocking(|| { + /// println!("now running on a worker thread"); + /// }); + /// # } + #[track_caller] + pub fn spawn_blocking(&self, func: F) -> JoinHandle + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + self.handle.spawn_blocking(func) + } + + /// Runs a future to completion on the Tokio runtime. This is the + /// runtime's entry point. + /// + /// This runs the given future on the current thread, blocking until it is + /// complete, and yielding its resolved result. Any tasks or timers + /// which the future spawns internally will be executed on the runtime. + /// + /// # Non-worker future + /// + /// Note that the future required by this function does not run as a + /// worker. The expectation is that other tasks are spawned by the future here. + /// Awaiting on other futures from the future provided here will not + /// perform as fast as those spawned as workers. + /// + /// # Multi thread scheduler + /// + /// When the multi thread scheduler is used this will allow futures + /// to run within the io driver and timer context of the overall runtime. + /// + /// Any spawned tasks will continue running after `block_on` returns. + /// + /// # Current thread scheduler + /// + /// When the current thread scheduler is enabled `block_on` + /// can be called concurrently from multiple threads. The first call + /// will take ownership of the io and timer drivers. This means + /// other threads which do not own the drivers will hook into that one. + /// When the first `block_on` completes, other threads will be able to + /// "steal" the driver to allow continued execution of their futures. + /// + /// Any spawned tasks will be suspended after `block_on` returns. Calling + /// `block_on` again will resume previously spawned tasks. + /// + /// # Panics + /// + /// This function panics if the provided future panics, or if called within an + /// asynchronous execution context. + /// + /// # Examples + /// + /// ```no_run + /// use tokio::runtime::Runtime; + /// + /// // Create the runtime + /// let rt = Runtime::new().unwrap(); + /// + /// // Execute the future, blocking the current thread until completion + /// rt.block_on(async { + /// println!("hello"); + /// }); + /// ``` + /// + /// [handle]: fn@Handle::block_on + #[track_caller] + pub fn block_on(&self, future: F) -> F::Output { + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + ))] + let future = super::task::trace::Trace::root(future); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let future = crate::util::trace::task( + future, + "block_on", + None, + crate::runtime::task::Id::next().as_u64(), + ); + + let _enter = self.enter(); + + match &self.scheduler { + Scheduler::CurrentThread(exec) => exec.block_on(&self.handle.inner, future), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Scheduler::MultiThread(exec) => exec.block_on(&self.handle.inner, future), + } + } + + /// Enters the runtime context. + /// + /// This allows you to construct types that must have an executor + /// available on creation such as [`Sleep`] or [`TcpStream`]. It will + /// also allow you to call methods such as [`tokio::spawn`]. + /// + /// [`Sleep`]: struct@crate::time::Sleep + /// [`TcpStream`]: struct@crate::net::TcpStream + /// [`tokio::spawn`]: fn@crate::spawn + /// + /// # Example + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// fn function_that_spawns(msg: String) { + /// // Had we not used `rt.enter` below, this would panic. + /// tokio::spawn(async move { + /// println!("{}", msg); + /// }); + /// } + /// + /// fn main() { + /// let rt = Runtime::new().unwrap(); + /// + /// let s = "Hello World!".to_string(); + /// + /// // By entering the context, we tie `tokio::spawn` to this executor. + /// let _guard = rt.enter(); + /// function_that_spawns(s); + /// } + /// ``` + pub fn enter(&self) -> EnterGuard<'_> { + self.handle.enter() + } + + /// Shuts down the runtime, waiting for at most `duration` for all spawned + /// work to stop. + /// + /// See the [struct level documentation](Runtime#shutdown) for more details. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime::Runtime; + /// use tokio::task; + /// + /// use std::thread; + /// use std::time::Duration; + /// + /// fn main() { + /// let runtime = Runtime::new().unwrap(); + /// + /// runtime.block_on(async move { + /// task::spawn_blocking(move || { + /// thread::sleep(Duration::from_secs(10_000)); + /// }); + /// }); + /// + /// runtime.shutdown_timeout(Duration::from_millis(100)); + /// } + /// ``` + pub fn shutdown_timeout(mut self, duration: Duration) { + // Wakeup and shutdown all the worker threads + self.handle.inner.shutdown(); + self.blocking_pool.shutdown(Some(duration)); + } + + /// Shuts down the runtime, without waiting for any spawned work to stop. + /// + /// This can be useful if you want to drop a runtime from within another runtime. + /// Normally, dropping a runtime will block indefinitely for spawned blocking tasks + /// to complete, which would normally not be permitted within an asynchronous context. + /// By calling `shutdown_background()`, you can drop the runtime from such a context. + /// + /// Note however, that because we do not wait for any blocking tasks to complete, this + /// may result in a resource leak (in that any blocking tasks are still running until they + /// return. + /// + /// See the [struct level documentation](Runtime#shutdown) for more details. + /// + /// This function is equivalent to calling `shutdown_timeout(Duration::from_nanos(0))`. + /// + /// ``` + /// use tokio::runtime::Runtime; + /// + /// fn main() { + /// let runtime = Runtime::new().unwrap(); + /// + /// runtime.block_on(async move { + /// let inner_runtime = Runtime::new().unwrap(); + /// // ... + /// inner_runtime.shutdown_background(); + /// }); + /// } + /// ``` + pub fn shutdown_background(self) { + self.shutdown_timeout(Duration::from_nanos(0)) + } +} + +#[allow(clippy::single_match)] // there are comments in the error branch, so we don't want if-let +impl Drop for Runtime { + fn drop(&mut self) { + match &mut self.scheduler { + Scheduler::CurrentThread(current_thread) => { + // This ensures that tasks spawned on the current-thread + // runtime are dropped inside the runtime's context. + let _guard = context::try_set_current(&self.handle.inner); + current_thread.shutdown(&self.handle.inner); + } + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Scheduler::MultiThread(multi_thread) => { + // The threaded scheduler drops its tasks on its worker threads, which is + // already in the runtime's context. + multi_thread.shutdown(&self.handle.inner); + } + } + } +} + +cfg_metrics! { + impl Runtime { + /// TODO + pub fn metrics(&self) -> crate::runtime::RuntimeMetrics { + self.handle.metrics() + } + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/current_thread.rs b/third_party/rust/tokio/src/runtime/scheduler/current_thread.rs new file mode 100644 index 000000000000..ac4a8d6fac14 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/current_thread.rs @@ -0,0 +1,750 @@ +use crate::future::poll_fn; +use crate::loom::sync::atomic::AtomicBool; +use crate::loom::sync::Arc; +use crate::runtime::driver::{self, Driver}; +use crate::runtime::scheduler::{self, Defer, Inject}; +use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task}; +use crate::runtime::{blocking, context, Config, MetricsBatch, SchedulerMetrics, WorkerMetrics}; +use crate::sync::notify::Notify; +use crate::util::atomic_cell::AtomicCell; +use crate::util::{waker_ref, RngSeedGenerator, Wake, WakerRef}; + +use std::cell::RefCell; +use std::collections::VecDeque; +use std::fmt; +use std::future::Future; +use std::sync::atomic::Ordering::{AcqRel, Release}; +use std::task::Poll::{Pending, Ready}; +use std::task::Waker; +use std::time::Duration; + +/// Executes tasks on the current thread +pub(crate) struct CurrentThread { + /// Core scheduler data is acquired by a thread entering `block_on`. + core: AtomicCell, + + /// Notifier for waking up other threads to steal the + /// driver. + notify: Notify, +} + +/// Handle to the current thread scheduler +pub(crate) struct Handle { + /// Scheduler state shared across threads + shared: Shared, + + /// Resource driver handles + pub(crate) driver: driver::Handle, + + /// Blocking pool spawner + pub(crate) blocking_spawner: blocking::Spawner, + + /// Current random number generator seed + pub(crate) seed_generator: RngSeedGenerator, +} + +/// Data required for executing the scheduler. The struct is passed around to +/// a function that will perform the scheduling work and acts as a capability token. +struct Core { + /// Scheduler run queue + tasks: VecDeque, + + /// Current tick + tick: u32, + + /// Runtime driver + /// + /// The driver is removed before starting to park the thread + driver: Option, + + /// Metrics batch + metrics: MetricsBatch, + + /// How often to check the global queue + global_queue_interval: u32, + + /// True if a task panicked without being handled and the runtime is + /// configured to shutdown on unhandled panic. + unhandled_panic: bool, +} + +/// Scheduler state shared between threads. +struct Shared { + /// Remote run queue + inject: Inject>, + + /// Collection of all active tasks spawned onto this executor. + owned: OwnedTasks>, + + /// Indicates whether the blocked on thread was woken. + woken: AtomicBool, + + /// Scheduler configuration options + config: Config, + + /// Keeps track of various runtime metrics. + scheduler_metrics: SchedulerMetrics, + + /// This scheduler only has one worker. + worker_metrics: WorkerMetrics, +} + +/// Thread-local context. +/// +/// pub(crate) to store in `runtime::context`. +pub(crate) struct Context { + /// Scheduler handle + handle: Arc, + + /// Scheduler core, enabling the holder of `Context` to execute the + /// scheduler. + core: RefCell>>, + + /// Deferred tasks, usually ones that called `task::yield_now()`. + pub(crate) defer: Defer, +} + +type Notified = task::Notified>; + +/// Initial queue capacity. +const INITIAL_CAPACITY: usize = 64; + +/// Used if none is specified. This is a temporary constant and will be removed +/// as we unify tuning logic between the multi-thread and current-thread +/// schedulers. +const DEFAULT_GLOBAL_QUEUE_INTERVAL: u32 = 31; + +impl CurrentThread { + pub(crate) fn new( + driver: Driver, + driver_handle: driver::Handle, + blocking_spawner: blocking::Spawner, + seed_generator: RngSeedGenerator, + config: Config, + ) -> (CurrentThread, Arc) { + let worker_metrics = WorkerMetrics::from_config(&config); + + // Get the configured global queue interval, or use the default. + let global_queue_interval = config + .global_queue_interval + .unwrap_or(DEFAULT_GLOBAL_QUEUE_INTERVAL); + + let handle = Arc::new(Handle { + shared: Shared { + inject: Inject::new(), + owned: OwnedTasks::new(), + woken: AtomicBool::new(false), + config, + scheduler_metrics: SchedulerMetrics::new(), + worker_metrics, + }, + driver: driver_handle, + blocking_spawner, + seed_generator, + }); + + let core = AtomicCell::new(Some(Box::new(Core { + tasks: VecDeque::with_capacity(INITIAL_CAPACITY), + tick: 0, + driver: Some(driver), + metrics: MetricsBatch::new(&handle.shared.worker_metrics), + global_queue_interval, + unhandled_panic: false, + }))); + + let scheduler = CurrentThread { + core, + notify: Notify::new(), + }; + + (scheduler, handle) + } + + #[track_caller] + pub(crate) fn block_on(&self, handle: &scheduler::Handle, future: F) -> F::Output { + pin!(future); + + crate::runtime::context::enter_runtime(handle, false, |blocking| { + let handle = handle.as_current_thread(); + + // Attempt to steal the scheduler core and block_on the future if we can + // there, otherwise, lets select on a notification that the core is + // available or the future is complete. + loop { + if let Some(core) = self.take_core(handle) { + return core.block_on(future); + } else { + let notified = self.notify.notified(); + pin!(notified); + + if let Some(out) = blocking + .block_on(poll_fn(|cx| { + if notified.as_mut().poll(cx).is_ready() { + return Ready(None); + } + + if let Ready(out) = future.as_mut().poll(cx) { + return Ready(Some(out)); + } + + Pending + })) + .expect("Failed to `Enter::block_on`") + { + return out; + } + } + } + }) + } + + fn take_core(&self, handle: &Arc) -> Option> { + let core = self.core.take()?; + + Some(CoreGuard { + context: scheduler::Context::CurrentThread(Context { + handle: handle.clone(), + core: RefCell::new(Some(core)), + defer: Defer::new(), + }), + scheduler: self, + }) + } + + pub(crate) fn shutdown(&mut self, handle: &scheduler::Handle) { + let handle = handle.as_current_thread(); + + // Avoid a double panic if we are currently panicking and + // the lock may be poisoned. + + let core = match self.take_core(handle) { + Some(core) => core, + None if std::thread::panicking() => return, + None => panic!("Oh no! We never placed the Core back, this is a bug!"), + }; + + // Check that the thread-local is not being destroyed + let tls_available = context::with_current(|_| ()).is_ok(); + + if tls_available { + core.enter(|core, _context| { + let core = shutdown2(core, handle); + (core, ()) + }); + } else { + // Shutdown without setting the context. `tokio::spawn` calls will + // fail, but those will fail either way because the thread-local is + // not available anymore. + let context = core.context.expect_current_thread(); + let core = context.core.borrow_mut().take().unwrap(); + + let core = shutdown2(core, handle); + *context.core.borrow_mut() = Some(core); + } + } +} + +fn shutdown2(mut core: Box, handle: &Handle) -> Box { + // Drain the OwnedTasks collection. This call also closes the + // collection, ensuring that no tasks are ever pushed after this + // call returns. + handle.shared.owned.close_and_shutdown_all(); + + // Drain local queue + // We already shut down every task, so we just need to drop the task. + while let Some(task) = core.next_local_task(handle) { + drop(task); + } + + // Close the injection queue + handle.shared.inject.close(); + + // Drain remote queue + while let Some(task) = handle.shared.inject.pop() { + drop(task); + } + + assert!(handle.shared.owned.is_empty()); + + // Submit metrics + core.submit_metrics(handle); + + // Shutdown the resource drivers + if let Some(driver) = core.driver.as_mut() { + driver.shutdown(&handle.driver); + } + + core +} + +impl fmt::Debug for CurrentThread { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("CurrentThread").finish() + } +} + +// ===== impl Core ===== + +impl Core { + /// Get and increment the current tick + fn tick(&mut self) { + self.tick = self.tick.wrapping_add(1); + } + + fn next_task(&mut self, handle: &Handle) -> Option { + if self.tick % self.global_queue_interval == 0 { + handle + .next_remote_task() + .or_else(|| self.next_local_task(handle)) + } else { + self.next_local_task(handle) + .or_else(|| handle.next_remote_task()) + } + } + + fn next_local_task(&mut self, handle: &Handle) -> Option { + let ret = self.tasks.pop_front(); + handle + .shared + .worker_metrics + .set_queue_depth(self.tasks.len()); + ret + } + + fn push_task(&mut self, handle: &Handle, task: Notified) { + self.tasks.push_back(task); + self.metrics.inc_local_schedule_count(); + handle + .shared + .worker_metrics + .set_queue_depth(self.tasks.len()); + } + + fn submit_metrics(&mut self, handle: &Handle) { + self.metrics.submit(&handle.shared.worker_metrics); + } +} + +#[cfg(tokio_taskdump)] +fn wake_deferred_tasks_and_free(context: &Context) { + let wakers = context.defer.take_deferred(); + for waker in wakers { + waker.wake(); + } +} + +// ===== impl Context ===== + +impl Context { + /// Execute the closure with the given scheduler core stored in the + /// thread-local context. + fn run_task(&self, mut core: Box, f: impl FnOnce() -> R) -> (Box, R) { + core.metrics.start_poll(); + let mut ret = self.enter(core, || crate::runtime::coop::budget(f)); + ret.0.metrics.end_poll(); + ret + } + + /// Blocks the current thread until an event is received by the driver, + /// including I/O events, timer events, ... + fn park(&self, mut core: Box, handle: &Handle) -> Box { + let mut driver = core.driver.take().expect("driver missing"); + + if let Some(f) = &handle.shared.config.before_park { + // Incorrect lint, the closures are actually different types so `f` + // cannot be passed as an argument to `enter`. + #[allow(clippy::redundant_closure)] + let (c, _) = self.enter(core, || f()); + core = c; + } + + // This check will fail if `before_park` spawns a task for us to run + // instead of parking the thread + if core.tasks.is_empty() { + // Park until the thread is signaled + core.metrics.about_to_park(); + core.submit_metrics(handle); + + let (c, _) = self.enter(core, || { + driver.park(&handle.driver); + self.defer.wake(); + }); + + core = c; + } + + if let Some(f) = &handle.shared.config.after_unpark { + // Incorrect lint, the closures are actually different types so `f` + // cannot be passed as an argument to `enter`. + #[allow(clippy::redundant_closure)] + let (c, _) = self.enter(core, || f()); + core = c; + } + + core.driver = Some(driver); + core + } + + /// Checks the driver for new events without blocking the thread. + fn park_yield(&self, mut core: Box, handle: &Handle) -> Box { + let mut driver = core.driver.take().expect("driver missing"); + + core.submit_metrics(handle); + + let (mut core, _) = self.enter(core, || { + driver.park_timeout(&handle.driver, Duration::from_millis(0)); + self.defer.wake(); + }); + + core.driver = Some(driver); + core + } + + fn enter(&self, core: Box, f: impl FnOnce() -> R) -> (Box, R) { + // Store the scheduler core in the thread-local context + // + // A drop-guard is employed at a higher level. + *self.core.borrow_mut() = Some(core); + + // Execute the closure while tracking the execution budget + let ret = f(); + + // Take the scheduler core back + let core = self.core.borrow_mut().take().expect("core missing"); + (core, ret) + } + + pub(crate) fn defer(&self, waker: &Waker) { + self.defer.defer(waker); + } +} + +// ===== impl Handle ===== + +impl Handle { + /// Spawns a future onto the `CurrentThread` scheduler + pub(crate) fn spawn( + me: &Arc, + future: F, + id: crate::runtime::task::Id, + ) -> JoinHandle + where + F: crate::future::Future + Send + 'static, + F::Output: Send + 'static, + { + let (handle, notified) = me.shared.owned.bind(future, me.clone(), id); + + if let Some(notified) = notified { + me.schedule(notified); + } + + handle + } + + /// Capture a snapshot of this runtime's state. + #[cfg(all( + tokio_unstable, + tokio_taskdump, + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + ))] + pub(crate) fn dump(&self) -> crate::runtime::Dump { + use crate::runtime::dump; + use task::trace::trace_current_thread; + + let mut traces = vec![]; + + // todo: how to make this work outside of a runtime context? + context::with_scheduler(|maybe_context| { + // drain the local queue + let context = if let Some(context) = maybe_context { + context.expect_current_thread() + } else { + return; + }; + let mut maybe_core = context.core.borrow_mut(); + let core = if let Some(core) = maybe_core.as_mut() { + core + } else { + return; + }; + let local = &mut core.tasks; + + if self.shared.inject.is_closed() { + return; + } + + traces = trace_current_thread(&self.shared.owned, local, &self.shared.inject) + .into_iter() + .map(dump::Task::new) + .collect(); + + // Avoid double borrow panic + drop(maybe_core); + + // Taking a taskdump could wakes every task, but we probably don't want + // the `yield_now` vector to be that large under normal circumstances. + // Therefore, we free its allocation. + wake_deferred_tasks_and_free(context); + }); + + dump::Dump::new(traces) + } + + fn next_remote_task(&self) -> Option { + self.shared.inject.pop() + } + + fn waker_ref(me: &Arc) -> WakerRef<'_> { + // Set woken to true when enter block_on, ensure outer future + // be polled for the first time when enter loop + me.shared.woken.store(true, Release); + waker_ref(me) + } + + // reset woken to false and return original value + pub(crate) fn reset_woken(&self) -> bool { + self.shared.woken.swap(false, AcqRel) + } +} + +cfg_metrics! { + impl Handle { + pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { + &self.shared.scheduler_metrics + } + + pub(crate) fn injection_queue_depth(&self) -> usize { + self.shared.inject.len() + } + + pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { + assert_eq!(0, worker); + &self.shared.worker_metrics + } + + pub(crate) fn num_blocking_threads(&self) -> usize { + self.blocking_spawner.num_threads() + } + + pub(crate) fn num_idle_blocking_threads(&self) -> usize { + self.blocking_spawner.num_idle_threads() + } + + pub(crate) fn blocking_queue_depth(&self) -> usize { + self.blocking_spawner.queue_depth() + } + + pub(crate) fn active_tasks_count(&self) -> usize { + self.shared.owned.active_tasks_count() + } + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("current_thread::Handle { ... }").finish() + } +} + +// ===== impl Shared ===== + +impl Schedule for Arc { + fn release(&self, task: &Task) -> Option> { + self.shared.owned.remove(task) + } + + fn schedule(&self, task: task::Notified) { + use scheduler::Context::CurrentThread; + + context::with_scheduler(|maybe_cx| match maybe_cx { + Some(CurrentThread(cx)) if Arc::ptr_eq(self, &cx.handle) => { + let mut core = cx.core.borrow_mut(); + + // If `None`, the runtime is shutting down, so there is no need + // to schedule the task. + if let Some(core) = core.as_mut() { + core.push_task(self, task); + } + } + _ => { + // Track that a task was scheduled from **outside** of the runtime. + self.shared.scheduler_metrics.inc_remote_schedule_count(); + + // Schedule the task + self.shared.inject.push(task); + self.driver.unpark(); + } + }); + } + + cfg_unstable! { + fn unhandled_panic(&self) { + use crate::runtime::UnhandledPanic; + + match self.shared.config.unhandled_panic { + UnhandledPanic::Ignore => { + // Do nothing + } + UnhandledPanic::ShutdownRuntime => { + use scheduler::Context::CurrentThread; + + // This hook is only called from within the runtime, so + // `context::with_scheduler` should match with `&self`, i.e. + // there is no opportunity for a nested scheduler to be + // called. + context::with_scheduler(|maybe_cx| match maybe_cx { + Some(CurrentThread(cx)) if Arc::ptr_eq(self, &cx.handle) => { + let mut core = cx.core.borrow_mut(); + + // If `None`, the runtime is shutting down, so there is no need to signal shutdown + if let Some(core) = core.as_mut() { + core.unhandled_panic = true; + self.shared.owned.close_and_shutdown_all(); + } + } + _ => unreachable!("runtime core not set in CURRENT thread-local"), + }) + } + } + } + } +} + +impl Wake for Handle { + fn wake(arc_self: Arc) { + Wake::wake_by_ref(&arc_self) + } + + /// Wake by reference + fn wake_by_ref(arc_self: &Arc) { + arc_self.shared.woken.store(true, Release); + arc_self.driver.unpark(); + } +} + +// ===== CoreGuard ===== + +/// Used to ensure we always place the `Core` value back into its slot in +/// `CurrentThread`, even if the future panics. +struct CoreGuard<'a> { + context: scheduler::Context, + scheduler: &'a CurrentThread, +} + +impl CoreGuard<'_> { + #[track_caller] + fn block_on(self, future: F) -> F::Output { + let ret = self.enter(|mut core, context| { + let waker = Handle::waker_ref(&context.handle); + let mut cx = std::task::Context::from_waker(&waker); + + pin!(future); + + core.metrics.start_processing_scheduled_tasks(); + + 'outer: loop { + let handle = &context.handle; + + if handle.reset_woken() { + let (c, res) = context.enter(core, || { + crate::runtime::coop::budget(|| future.as_mut().poll(&mut cx)) + }); + + core = c; + + if let Ready(v) = res { + return (core, Some(v)); + } + } + + for _ in 0..handle.shared.config.event_interval { + // Make sure we didn't hit an unhandled_panic + if core.unhandled_panic { + return (core, None); + } + + core.tick(); + + let entry = core.next_task(handle); + + let task = match entry { + Some(entry) => entry, + None => { + core.metrics.end_processing_scheduled_tasks(); + + core = if !context.defer.is_empty() { + context.park_yield(core, handle) + } else { + context.park(core, handle) + }; + + core.metrics.start_processing_scheduled_tasks(); + + // Try polling the `block_on` future next + continue 'outer; + } + }; + + let task = context.handle.shared.owned.assert_owner(task); + + let (c, _) = context.run_task(core, || { + task.run(); + }); + + core = c; + } + + core.metrics.end_processing_scheduled_tasks(); + + // Yield to the driver, this drives the timer and pulls any + // pending I/O events. + core = context.park_yield(core, handle); + + core.metrics.start_processing_scheduled_tasks(); + } + }); + + match ret { + Some(ret) => ret, + None => { + // `block_on` panicked. + panic!("a spawned task panicked and the runtime is configured to shut down on unhandled panic"); + } + } + } + + /// Enters the scheduler context. This sets the queue and other necessary + /// scheduler state in the thread-local. + fn enter(self, f: F) -> R + where + F: FnOnce(Box, &Context) -> (Box, R), + { + let context = self.context.expect_current_thread(); + + // Remove `core` from `context` to pass into the closure. + let core = context.core.borrow_mut().take().expect("core missing"); + + // Call the closure and place `core` back + let (core, ret) = context::set_scheduler(&self.context, || f(core, context)); + + *context.core.borrow_mut() = Some(core); + + ret + } +} + +impl Drop for CoreGuard<'_> { + fn drop(&mut self) { + let context = self.context.expect_current_thread(); + + if let Some(core) = context.core.borrow_mut().take() { + // Replace old scheduler back into the state to allow + // other threads to pick it up and drive it. + self.scheduler.core.set(core); + + // Wake up other possible threads that could steal the driver. + self.scheduler.notify.notify_one() + } + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/defer.rs b/third_party/rust/tokio/src/runtime/scheduler/defer.rs new file mode 100644 index 000000000000..a4be8ef2e596 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/defer.rs @@ -0,0 +1,43 @@ +use std::cell::RefCell; +use std::task::Waker; + +pub(crate) struct Defer { + deferred: RefCell>, +} + +impl Defer { + pub(crate) fn new() -> Defer { + Defer { + deferred: Default::default(), + } + } + + pub(crate) fn defer(&self, waker: &Waker) { + let mut deferred = self.deferred.borrow_mut(); + + // If the same task adds itself a bunch of times, then only add it once. + if let Some(last) = deferred.last() { + if last.will_wake(waker) { + return; + } + } + + deferred.push(waker.clone()); + } + + pub(crate) fn is_empty(&self) -> bool { + self.deferred.borrow().is_empty() + } + + pub(crate) fn wake(&self) { + while let Some(waker) = self.deferred.borrow_mut().pop() { + waker.wake(); + } + } + + #[cfg(tokio_taskdump)] + pub(crate) fn take_deferred(&self) -> Vec { + let mut deferred = self.deferred.borrow_mut(); + std::mem::take(&mut *deferred) + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/inject.rs b/third_party/rust/tokio/src/runtime/scheduler/inject.rs new file mode 100644 index 000000000000..39976fcd7a2d --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/inject.rs @@ -0,0 +1,72 @@ +//! Inject queue used to send wakeups to a work-stealing scheduler + +use crate::loom::sync::Mutex; +use crate::runtime::task; + +mod pop; +pub(crate) use pop::Pop; + +mod shared; +pub(crate) use shared::Shared; + +mod synced; +pub(crate) use synced::Synced; + +cfg_rt_multi_thread! { + mod rt_multi_thread; +} + +cfg_metrics! { + mod metrics; +} + +/// Growable, MPMC queue used to inject new tasks into the scheduler and as an +/// overflow queue when the local, fixed-size, array queue overflows. +pub(crate) struct Inject { + shared: Shared, + synced: Mutex, +} + +impl Inject { + pub(crate) fn new() -> Inject { + let (shared, synced) = Shared::new(); + + Inject { + shared, + synced: Mutex::new(synced), + } + } + + // Kind of annoying to have to include the cfg here + #[cfg(tokio_taskdump)] + pub(crate) fn is_closed(&self) -> bool { + let synced = self.synced.lock(); + self.shared.is_closed(&synced) + } + + /// Closes the injection queue, returns `true` if the queue is open when the + /// transition is made. + pub(crate) fn close(&self) -> bool { + let mut synced = self.synced.lock(); + self.shared.close(&mut synced) + } + + /// Pushes a value into the queue. + /// + /// This does nothing if the queue is closed. + pub(crate) fn push(&self, task: task::Notified) { + let mut synced = self.synced.lock(); + // safety: passing correct `Synced` + unsafe { self.shared.push(&mut synced, task) } + } + + pub(crate) fn pop(&self) -> Option> { + if self.shared.is_empty() { + return None; + } + + let mut synced = self.synced.lock(); + // safety: passing correct `Synced` + unsafe { self.shared.pop(&mut synced) } + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/inject/metrics.rs b/third_party/rust/tokio/src/runtime/scheduler/inject/metrics.rs new file mode 100644 index 000000000000..76f045fdbd60 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/inject/metrics.rs @@ -0,0 +1,7 @@ +use super::Inject; + +impl Inject { + pub(crate) fn len(&self) -> usize { + self.shared.len() + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/inject/pop.rs b/third_party/rust/tokio/src/runtime/scheduler/inject/pop.rs new file mode 100644 index 000000000000..4e6d5d3be3a4 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/inject/pop.rs @@ -0,0 +1,55 @@ +use super::Synced; + +use crate::runtime::task; + +use std::marker::PhantomData; + +pub(crate) struct Pop<'a, T: 'static> { + len: usize, + synced: &'a mut Synced, + _p: PhantomData, +} + +impl<'a, T: 'static> Pop<'a, T> { + pub(super) fn new(len: usize, synced: &'a mut Synced) -> Pop<'a, T> { + Pop { + len, + synced, + _p: PhantomData, + } + } +} + +impl<'a, T: 'static> Iterator for Pop<'a, T> { + type Item = task::Notified; + + fn next(&mut self) -> Option { + if self.len == 0 { + return None; + } + + let ret = self.synced.pop(); + + // Should be `Some` when `len > 0` + debug_assert!(ret.is_some()); + + self.len -= 1; + ret + } + + fn size_hint(&self) -> (usize, Option) { + (self.len, Some(self.len)) + } +} + +impl<'a, T: 'static> ExactSizeIterator for Pop<'a, T> { + fn len(&self) -> usize { + self.len + } +} + +impl<'a, T: 'static> Drop for Pop<'a, T> { + fn drop(&mut self) { + for _ in self.by_ref() {} + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs b/third_party/rust/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs new file mode 100644 index 000000000000..07d1063c5d82 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs @@ -0,0 +1,98 @@ +use super::{Shared, Synced}; + +use crate::runtime::scheduler::Lock; +use crate::runtime::task; + +use std::sync::atomic::Ordering::Release; + +impl<'a> Lock for &'a mut Synced { + type Handle = &'a mut Synced; + + fn lock(self) -> Self::Handle { + self + } +} + +impl AsMut for Synced { + fn as_mut(&mut self) -> &mut Synced { + self + } +} + +impl Shared { + /// Pushes several values into the queue. + /// + /// # Safety + /// + /// Must be called with the same `Synced` instance returned by `Inject::new` + #[inline] + pub(crate) unsafe fn push_batch(&self, shared: L, mut iter: I) + where + L: Lock, + I: Iterator>, + { + let first = match iter.next() { + Some(first) => first.into_raw(), + None => return, + }; + + // Link up all the tasks. + let mut prev = first; + let mut counter = 1; + + // We are going to be called with an `std::iter::Chain`, and that + // iterator overrides `for_each` to something that is easier for the + // compiler to optimize than a loop. + iter.for_each(|next| { + let next = next.into_raw(); + + // safety: Holding the Notified for a task guarantees exclusive + // access to the `queue_next` field. + unsafe { prev.set_queue_next(Some(next)) }; + prev = next; + counter += 1; + }); + + // Now that the tasks are linked together, insert them into the + // linked list. + self.push_batch_inner(shared, first, prev, counter); + } + + /// Inserts several tasks that have been linked together into the queue. + /// + /// The provided head and tail may be be the same task. In this case, a + /// single task is inserted. + #[inline] + unsafe fn push_batch_inner( + &self, + shared: L, + batch_head: task::RawTask, + batch_tail: task::RawTask, + num: usize, + ) where + L: Lock, + { + debug_assert!(unsafe { batch_tail.get_queue_next().is_none() }); + + let mut synced = shared.lock(); + let synced = synced.as_mut(); + + if let Some(tail) = synced.tail { + unsafe { + tail.set_queue_next(Some(batch_head)); + } + } else { + synced.head = Some(batch_head); + } + + synced.tail = Some(batch_tail); + + // Increment the count. + // + // safety: All updates to the len atomic are guarded by the mutex. As + // such, a non-atomic load followed by a store is safe. + let len = self.len.unsync_load(); + + self.len.store(len + num, Release); + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/inject/shared.rs b/third_party/rust/tokio/src/runtime/scheduler/inject/shared.rs new file mode 100644 index 000000000000..7fdd2839dd25 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/inject/shared.rs @@ -0,0 +1,119 @@ +use super::{Pop, Synced}; + +use crate::loom::sync::atomic::AtomicUsize; +use crate::runtime::task; + +use std::marker::PhantomData; +use std::sync::atomic::Ordering::{Acquire, Release}; + +pub(crate) struct Shared { + /// Number of pending tasks in the queue. This helps prevent unnecessary + /// locking in the hot path. + pub(super) len: AtomicUsize, + + _p: PhantomData, +} + +unsafe impl Send for Shared {} +unsafe impl Sync for Shared {} + +impl Shared { + pub(crate) fn new() -> (Shared, Synced) { + let inject = Shared { + len: AtomicUsize::new(0), + _p: PhantomData, + }; + + let synced = Synced { + is_closed: false, + head: None, + tail: None, + }; + + (inject, synced) + } + + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + + // Kind of annoying to have to include the cfg here + #[cfg(any(tokio_taskdump, all(feature = "rt-multi-thread", not(tokio_wasi))))] + pub(crate) fn is_closed(&self, synced: &Synced) -> bool { + synced.is_closed + } + + /// Closes the injection queue, returns `true` if the queue is open when the + /// transition is made. + pub(crate) fn close(&self, synced: &mut Synced) -> bool { + if synced.is_closed { + return false; + } + + synced.is_closed = true; + true + } + + pub(crate) fn len(&self) -> usize { + self.len.load(Acquire) + } + + /// Pushes a value into the queue. + /// + /// This does nothing if the queue is closed. + /// + /// # Safety + /// + /// Must be called with the same `Synced` instance returned by `Inject::new` + pub(crate) unsafe fn push(&self, synced: &mut Synced, task: task::Notified) { + if synced.is_closed { + return; + } + + // safety: only mutated with the lock held + let len = self.len.unsync_load(); + let task = task.into_raw(); + + // The next pointer should already be null + debug_assert!(unsafe { task.get_queue_next().is_none() }); + + if let Some(tail) = synced.tail { + // safety: Holding the Notified for a task guarantees exclusive + // access to the `queue_next` field. + unsafe { tail.set_queue_next(Some(task)) }; + } else { + synced.head = Some(task); + } + + synced.tail = Some(task); + self.len.store(len + 1, Release); + } + + /// Pop a value from the queue. + /// + /// # Safety + /// + /// Must be called with the same `Synced` instance returned by `Inject::new` + pub(crate) unsafe fn pop(&self, synced: &mut Synced) -> Option> { + self.pop_n(synced, 1).next() + } + + /// Pop `n` values from the queue + /// + /// # Safety + /// + /// Must be called with the same `Synced` instance returned by `Inject::new` + pub(crate) unsafe fn pop_n<'a>(&'a self, synced: &'a mut Synced, n: usize) -> Pop<'a, T> { + use std::cmp; + + // safety: All updates to the len atomic are guarded by the mutex. As + // such, a non-atomic load followed by a store is safe. + let len = self.len.unsync_load(); + let n = cmp::min(n, len); + + // Decrement the count. + self.len.store(len - n, Release); + + Pop::new(n, synced) + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/inject/synced.rs b/third_party/rust/tokio/src/runtime/scheduler/inject/synced.rs new file mode 100644 index 000000000000..6847f68e5dba --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/inject/synced.rs @@ -0,0 +1,32 @@ +use crate::runtime::task; + +pub(crate) struct Synced { + /// True if the queue is closed. + pub(super) is_closed: bool, + + /// Linked-list head. + pub(super) head: Option, + + /// Linked-list tail. + pub(super) tail: Option, +} + +unsafe impl Send for Synced {} +unsafe impl Sync for Synced {} + +impl Synced { + pub(super) fn pop(&mut self) -> Option> { + let task = self.head?; + + self.head = unsafe { task.get_queue_next() }; + + if self.head.is_none() { + self.tail = None; + } + + unsafe { task.set_queue_next(None) }; + + // safety: a `Notified` is pushed into the queue and now it is popped! + Some(unsafe { task::Notified::from_raw(task) }) + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/lock.rs b/third_party/rust/tokio/src/runtime/scheduler/lock.rs new file mode 100644 index 000000000000..0901c2b37ca6 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/lock.rs @@ -0,0 +1,6 @@ +/// A lock (mutex) yielding generic data. +pub(crate) trait Lock { + type Handle: AsMut; + + fn lock(self) -> Self::Handle; +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/mod.rs b/third_party/rust/tokio/src/runtime/scheduler/mod.rs new file mode 100644 index 000000000000..3e3151711f5d --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/mod.rs @@ -0,0 +1,249 @@ +cfg_rt! { + pub(crate) mod current_thread; + pub(crate) use current_thread::CurrentThread; + + mod defer; + use defer::Defer; + + pub(crate) mod inject; + pub(crate) use inject::Inject; +} + +cfg_rt_multi_thread! { + mod lock; + use lock::Lock; + + pub(crate) mod multi_thread; + pub(crate) use multi_thread::MultiThread; +} + +use crate::runtime::driver; + +#[derive(Debug, Clone)] +pub(crate) enum Handle { + #[cfg(feature = "rt")] + CurrentThread(Arc), + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + MultiThread(Arc), + + // TODO: This is to avoid triggering "dead code" warnings many other places + // in the codebase. Remove this during a later cleanup + #[cfg(not(feature = "rt"))] + #[allow(dead_code)] + Disabled, +} + +#[cfg(feature = "rt")] +pub(super) enum Context { + CurrentThread(current_thread::Context), + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + MultiThread(multi_thread::Context), +} + +impl Handle { + #[cfg_attr(not(feature = "full"), allow(dead_code))] + pub(crate) fn driver(&self) -> &driver::Handle { + match *self { + #[cfg(feature = "rt")] + Handle::CurrentThread(ref h) => &h.driver, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(ref h) => &h.driver, + + #[cfg(not(feature = "rt"))] + Handle::Disabled => unreachable!(), + } + } +} + +cfg_rt! { + use crate::future::Future; + use crate::loom::sync::Arc; + use crate::runtime::{blocking, task::Id}; + use crate::runtime::context; + use crate::task::JoinHandle; + use crate::util::RngSeedGenerator; + use std::task::Waker; + + impl Handle { + #[track_caller] + pub(crate) fn current() -> Handle { + match context::with_current(Clone::clone) { + Ok(handle) => handle, + Err(e) => panic!("{}", e), + } + } + + pub(crate) fn blocking_spawner(&self) -> &blocking::Spawner { + match self { + Handle::CurrentThread(h) => &h.blocking_spawner, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(h) => &h.blocking_spawner, + } + } + + pub(crate) fn spawn(&self, future: F, id: Id) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + match self { + Handle::CurrentThread(h) => current_thread::Handle::spawn(h, future, id), + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(h) => multi_thread::Handle::spawn(h, future, id), + } + } + + pub(crate) fn shutdown(&self) { + match *self { + Handle::CurrentThread(_) => {}, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(ref h) => h.shutdown(), + } + } + + pub(crate) fn seed_generator(&self) -> &RngSeedGenerator { + match self { + Handle::CurrentThread(h) => &h.seed_generator, + + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(h) => &h.seed_generator, + } + } + + pub(crate) fn as_current_thread(&self) -> &Arc { + match self { + Handle::CurrentThread(handle) => handle, + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + _ => panic!("not a CurrentThread handle"), + } + } + } + + cfg_metrics! { + use crate::runtime::{SchedulerMetrics, WorkerMetrics}; + + impl Handle { + pub(crate) fn num_workers(&self) -> usize { + match self { + Handle::CurrentThread(_) => 1, + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.num_workers(), + } + } + + pub(crate) fn num_blocking_threads(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.num_blocking_threads(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.num_blocking_threads(), + } + } + + pub(crate) fn num_idle_blocking_threads(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.num_idle_blocking_threads(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.num_idle_blocking_threads(), + } + } + + pub(crate) fn active_tasks_count(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.active_tasks_count(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.active_tasks_count(), + } + } + + pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { + match self { + Handle::CurrentThread(handle) => handle.scheduler_metrics(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.scheduler_metrics(), + } + } + + pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { + match self { + Handle::CurrentThread(handle) => handle.worker_metrics(worker), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.worker_metrics(worker), + } + } + + pub(crate) fn injection_queue_depth(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.injection_queue_depth(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.injection_queue_depth(), + } + } + + pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize { + match self { + Handle::CurrentThread(handle) => handle.worker_metrics(worker).queue_depth(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.worker_local_queue_depth(worker), + } + } + + pub(crate) fn blocking_queue_depth(&self) -> usize { + match self { + Handle::CurrentThread(handle) => handle.blocking_queue_depth(), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Handle::MultiThread(handle) => handle.blocking_queue_depth(), + } + } + } + } + + impl Context { + #[track_caller] + pub(crate) fn expect_current_thread(&self) -> ¤t_thread::Context { + match self { + Context::CurrentThread(context) => context, + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + _ => panic!("expected `CurrentThread::Context`") + } + } + + pub(crate) fn defer(&self, waker: &Waker) { + match self { + Context::CurrentThread(context) => context.defer(waker), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + Context::MultiThread(context) => context.defer(waker), + } + } + + cfg_rt_multi_thread! { + #[track_caller] + pub(crate) fn expect_multi_thread(&self) -> &multi_thread::Context { + match self { + Context::MultiThread(context) => context, + _ => panic!("expected `MultiThread::Context`") + } + } + } + } +} + +cfg_not_rt! { + #[cfg(any( + feature = "net", + all(unix, feature = "process"), + all(unix, feature = "signal"), + feature = "time", + ))] + impl Handle { + #[track_caller] + pub(crate) fn current() -> Handle { + panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) + } + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/counters.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/counters.rs new file mode 100644 index 000000000000..50bcc11985f6 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/counters.rs @@ -0,0 +1,62 @@ +#[cfg(tokio_internal_mt_counters)] +mod imp { + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering::Relaxed; + + static NUM_MAINTENANCE: AtomicUsize = AtomicUsize::new(0); + static NUM_NOTIFY_LOCAL: AtomicUsize = AtomicUsize::new(0); + static NUM_UNPARKS_LOCAL: AtomicUsize = AtomicUsize::new(0); + static NUM_LIFO_SCHEDULES: AtomicUsize = AtomicUsize::new(0); + static NUM_LIFO_CAPPED: AtomicUsize = AtomicUsize::new(0); + + impl Drop for super::Counters { + fn drop(&mut self) { + let notifies_local = NUM_NOTIFY_LOCAL.load(Relaxed); + let unparks_local = NUM_UNPARKS_LOCAL.load(Relaxed); + let maintenance = NUM_MAINTENANCE.load(Relaxed); + let lifo_scheds = NUM_LIFO_SCHEDULES.load(Relaxed); + let lifo_capped = NUM_LIFO_CAPPED.load(Relaxed); + + println!("---"); + println!("notifies (local): {}", notifies_local); + println!(" unparks (local): {}", unparks_local); + println!(" maintenance: {}", maintenance); + println!(" LIFO schedules: {}", lifo_scheds); + println!(" LIFO capped: {}", lifo_capped); + } + } + + pub(crate) fn inc_num_inc_notify_local() { + NUM_NOTIFY_LOCAL.fetch_add(1, Relaxed); + } + + pub(crate) fn inc_num_unparks_local() { + NUM_UNPARKS_LOCAL.fetch_add(1, Relaxed); + } + + pub(crate) fn inc_num_maintenance() { + NUM_MAINTENANCE.fetch_add(1, Relaxed); + } + + pub(crate) fn inc_lifo_schedules() { + NUM_LIFO_SCHEDULES.fetch_add(1, Relaxed); + } + + pub(crate) fn inc_lifo_capped() { + NUM_LIFO_CAPPED.fetch_add(1, Relaxed); + } +} + +#[cfg(not(tokio_internal_mt_counters))] +mod imp { + pub(crate) fn inc_num_inc_notify_local() {} + pub(crate) fn inc_num_unparks_local() {} + pub(crate) fn inc_num_maintenance() {} + pub(crate) fn inc_lifo_schedules() {} + pub(crate) fn inc_lifo_capped() {} +} + +#[derive(Debug)] +pub(crate) struct Counters; + +pub(super) use imp::*; diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle.rs new file mode 100644 index 000000000000..98e476585601 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle.rs @@ -0,0 +1,68 @@ +use crate::future::Future; +use crate::loom::sync::Arc; +use crate::runtime::scheduler::multi_thread::worker; +use crate::runtime::{ + blocking, driver, + task::{self, JoinHandle}, +}; +use crate::util::RngSeedGenerator; + +use std::fmt; + +cfg_metrics! { + mod metrics; +} + +cfg_taskdump! { + mod taskdump; +} + +/// Handle to the multi thread scheduler +pub(crate) struct Handle { + /// Task spawner + pub(super) shared: worker::Shared, + + /// Resource driver handles + pub(crate) driver: driver::Handle, + + /// Blocking pool spawner + pub(crate) blocking_spawner: blocking::Spawner, + + /// Current random number generator seed + pub(crate) seed_generator: RngSeedGenerator, +} + +impl Handle { + /// Spawns a future onto the thread pool + pub(crate) fn spawn(me: &Arc, future: F, id: task::Id) -> JoinHandle + where + F: crate::future::Future + Send + 'static, + F::Output: Send + 'static, + { + Self::bind_new_task(me, future, id) + } + + pub(crate) fn shutdown(&self) { + self.close(); + } + + pub(super) fn bind_new_task(me: &Arc, future: T, id: task::Id) -> JoinHandle + where + T: Future + Send + 'static, + T::Output: Send + 'static, + { + let (handle, notified) = me.shared.owned.bind(future, me.clone(), id); + + if let Some(notified) = notified { + me.schedule_task(notified, false); + } + + handle + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("multi_thread::Handle { ... }").finish() + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs new file mode 100644 index 000000000000..838694fc89e2 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs @@ -0,0 +1,41 @@ +use super::Handle; + +use crate::runtime::{SchedulerMetrics, WorkerMetrics}; + +impl Handle { + pub(crate) fn num_workers(&self) -> usize { + self.shared.worker_metrics.len() + } + + pub(crate) fn num_blocking_threads(&self) -> usize { + self.blocking_spawner.num_threads() + } + + pub(crate) fn num_idle_blocking_threads(&self) -> usize { + self.blocking_spawner.num_idle_threads() + } + + pub(crate) fn active_tasks_count(&self) -> usize { + self.shared.owned.active_tasks_count() + } + + pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { + &self.shared.scheduler_metrics + } + + pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { + &self.shared.worker_metrics[worker] + } + + pub(crate) fn injection_queue_depth(&self) -> usize { + self.shared.injection_queue_depth() + } + + pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize { + self.shared.worker_local_queue_depth(worker) + } + + pub(crate) fn blocking_queue_depth(&self) -> usize { + self.blocking_spawner.queue_depth() + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/taskdump.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/taskdump.rs new file mode 100644 index 000000000000..477d857d88f7 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/handle/taskdump.rs @@ -0,0 +1,26 @@ +use super::Handle; + +use crate::runtime::Dump; + +impl Handle { + pub(crate) async fn dump(&self) -> Dump { + let trace_status = &self.shared.trace_status; + + // If a dump is in progress, block. + trace_status.start_trace_request(&self).await; + + let result = loop { + if let Some(result) = trace_status.take_result() { + break result; + } else { + self.notify_all(); + trace_status.result_ready.notified().await; + } + }; + + // Allow other queued dumps to proceed. + trace_status.end_trace_request(&self).await; + + result + } +} diff --git a/third_party/rust/tokio/src/runtime/thread_pool/idle.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/idle.rs similarity index 84% rename from third_party/rust/tokio/src/runtime/thread_pool/idle.rs rename to third_party/rust/tokio/src/runtime/scheduler/multi_thread/idle.rs index a57bf6a0b13c..834bc2b66fc0 100644 --- a/third_party/rust/tokio/src/runtime/thread_pool/idle.rs +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/idle.rs @@ -1,7 +1,7 @@ //! Coordinates idling workers use crate::loom::sync::atomic::AtomicUsize; -use crate::loom::sync::Mutex; +use crate::runtime::scheduler::multi_thread::Shared; use std::fmt; use std::sync::atomic::Ordering::{self, SeqCst}; @@ -13,13 +13,16 @@ pub(super) struct Idle { /// Used as a fast-path to avoid acquiring the lock when needed. state: AtomicUsize, - /// Sleeping workers - sleepers: Mutex>, - /// Total number of workers. num_workers: usize, } +/// Data synchronized by the scheduler mutex +pub(super) struct Synced { + /// Sleeping workers + sleepers: Vec, +} + const UNPARK_SHIFT: usize = 16; const UNPARK_MASK: usize = !SEARCH_MASK; const SEARCH_MASK: usize = (1 << UNPARK_SHIFT) - 1; @@ -28,19 +31,24 @@ const SEARCH_MASK: usize = (1 << UNPARK_SHIFT) - 1; struct State(usize); impl Idle { - pub(super) fn new(num_workers: usize) -> Idle { + pub(super) fn new(num_workers: usize) -> (Idle, Synced) { let init = State::new(num_workers); - Idle { + let idle = Idle { state: AtomicUsize::new(init.into()), - sleepers: Mutex::new(Vec::with_capacity(num_workers)), num_workers, - } + }; + + let synced = Synced { + sleepers: Vec::with_capacity(num_workers), + }; + + (idle, synced) } /// If there are no workers actively searching, returns the index of a /// worker currently sleeping. - pub(super) fn worker_to_notify(&self) -> Option { + pub(super) fn worker_to_notify(&self, shared: &Shared) -> Option { // If at least one worker is spinning, work being notified will // eventually be found. A searching thread will find **some** work and // notify another worker, eventually leading to our work being found. @@ -55,7 +63,7 @@ impl Idle { } // Acquire the lock - let mut sleepers = self.sleepers.lock(); + let mut lock = shared.synced.lock(); // Check again, now that the lock is acquired if !self.notify_should_wakeup() { @@ -67,7 +75,7 @@ impl Idle { State::unpark_one(&self.state, 1); // Get the worker to unpark - let ret = sleepers.pop(); + let ret = lock.idle.sleepers.pop(); debug_assert!(ret.is_some()); ret @@ -75,15 +83,20 @@ impl Idle { /// Returns `true` if the worker needs to do a final check for submitted /// work. - pub(super) fn transition_worker_to_parked(&self, worker: usize, is_searching: bool) -> bool { + pub(super) fn transition_worker_to_parked( + &self, + shared: &Shared, + worker: usize, + is_searching: bool, + ) -> bool { // Acquire the lock - let mut sleepers = self.sleepers.lock(); + let mut lock = shared.synced.lock(); // Decrement the number of unparked threads let ret = State::dec_num_unparked(&self.state, is_searching); // Track the sleeping worker - sleepers.push(worker); + lock.idle.sleepers.push(worker); ret } @@ -113,8 +126,9 @@ impl Idle { /// within the worker's park routine. /// /// Returns `true` if the worker was parked before calling the method. - pub(super) fn unpark_worker_by_id(&self, worker_id: usize) -> bool { - let mut sleepers = self.sleepers.lock(); + pub(super) fn unpark_worker_by_id(&self, shared: &Shared, worker_id: usize) -> bool { + let mut lock = shared.synced.lock(); + let sleepers = &mut lock.idle.sleepers; for index in 0..sleepers.len() { if sleepers[index] == worker_id { @@ -131,9 +145,9 @@ impl Idle { } /// Returns `true` if `worker_id` is contained in the sleep set. - pub(super) fn is_parked(&self, worker_id: usize) -> bool { - let sleepers = self.sleepers.lock(); - sleepers.contains(&worker_id) + pub(super) fn is_parked(&self, shared: &Shared, worker_id: usize) -> bool { + let lock = shared.synced.lock(); + lock.idle.sleepers.contains(&worker_id) } fn notify_should_wakeup(&self) -> bool { diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/mod.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/mod.rs new file mode 100644 index 000000000000..d85a0ae0a2ab --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/mod.rs @@ -0,0 +1,103 @@ +//! Multi-threaded runtime + +mod counters; +use counters::Counters; + +mod handle; +pub(crate) use handle::Handle; + +mod overflow; +pub(crate) use overflow::Overflow; + +mod idle; +use self::idle::Idle; + +mod stats; +pub(crate) use stats::Stats; + +mod park; +pub(crate) use park::{Parker, Unparker}; + +pub(crate) mod queue; + +mod worker; +pub(crate) use worker::{Context, Launch, Shared}; + +cfg_taskdump! { + mod trace; + use trace::TraceStatus; + + pub(crate) use worker::Synced; +} + +cfg_not_taskdump! { + mod trace_mock; + use trace_mock::TraceStatus; +} + +pub(crate) use worker::block_in_place; + +use crate::loom::sync::Arc; +use crate::runtime::{ + blocking, + driver::{self, Driver}, + scheduler, Config, +}; +use crate::util::RngSeedGenerator; + +use std::fmt; +use std::future::Future; + +/// Work-stealing based thread pool for executing futures. +pub(crate) struct MultiThread; + +// ===== impl MultiThread ===== + +impl MultiThread { + pub(crate) fn new( + size: usize, + driver: Driver, + driver_handle: driver::Handle, + blocking_spawner: blocking::Spawner, + seed_generator: RngSeedGenerator, + config: Config, + ) -> (MultiThread, Arc, Launch) { + let parker = Parker::new(driver); + let (handle, launch) = worker::create( + size, + parker, + driver_handle, + blocking_spawner, + seed_generator, + config, + ); + + (MultiThread, handle, launch) + } + + /// Blocks the current thread waiting for the future to complete. + /// + /// The future will execute on the current thread, but all spawned tasks + /// will be executed on the thread pool. + pub(crate) fn block_on(&self, handle: &scheduler::Handle, future: F) -> F::Output + where + F: Future, + { + crate::runtime::context::enter_runtime(handle, true, |blocking| { + blocking.block_on(future).expect("failed to park thread") + }) + } + + pub(crate) fn shutdown(&mut self, handle: &scheduler::Handle) { + match handle { + scheduler::Handle::MultiThread(handle) => handle.shutdown(), + _ => panic!("expected MultiThread scheduler"), + } + } +} + +impl fmt::Debug for MultiThread { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("MultiThread").finish() + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/overflow.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/overflow.rs new file mode 100644 index 000000000000..ab664811cff3 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/overflow.rs @@ -0,0 +1,26 @@ +use crate::runtime::task; + +#[cfg(test)] +use std::cell::RefCell; + +pub(crate) trait Overflow { + fn push(&self, task: task::Notified); + + fn push_batch(&self, iter: I) + where + I: Iterator>; +} + +#[cfg(test)] +impl Overflow for RefCell>> { + fn push(&self, task: task::Notified) { + self.borrow_mut().push(task); + } + + fn push_batch(&self, iter: I) + where + I: Iterator>, + { + self.borrow_mut().extend(iter); + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/park.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/park.rs new file mode 100644 index 000000000000..0a00ea004ee9 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/park.rs @@ -0,0 +1,232 @@ +//! Parks the runtime. +//! +//! A combination of the various resource driver park handles. + +use crate::loom::sync::atomic::AtomicUsize; +use crate::loom::sync::{Arc, Condvar, Mutex}; +use crate::runtime::driver::{self, Driver}; +use crate::util::TryLock; + +use std::sync::atomic::Ordering::SeqCst; +use std::time::Duration; + +pub(crate) struct Parker { + inner: Arc, +} + +pub(crate) struct Unparker { + inner: Arc, +} + +struct Inner { + /// Avoids entering the park if possible + state: AtomicUsize, + + /// Used to coordinate access to the driver / condvar + mutex: Mutex<()>, + + /// Condvar to block on if the driver is unavailable. + condvar: Condvar, + + /// Resource (I/O, time, ...) driver + shared: Arc, +} + +const EMPTY: usize = 0; +const PARKED_CONDVAR: usize = 1; +const PARKED_DRIVER: usize = 2; +const NOTIFIED: usize = 3; + +/// Shared across multiple Parker handles +struct Shared { + /// Shared driver. Only one thread at a time can use this + driver: TryLock, +} + +impl Parker { + pub(crate) fn new(driver: Driver) -> Parker { + Parker { + inner: Arc::new(Inner { + state: AtomicUsize::new(EMPTY), + mutex: Mutex::new(()), + condvar: Condvar::new(), + shared: Arc::new(Shared { + driver: TryLock::new(driver), + }), + }), + } + } + + pub(crate) fn unpark(&self) -> Unparker { + Unparker { + inner: self.inner.clone(), + } + } + + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.inner.park(handle); + } + + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + // Only parking with zero is supported... + assert_eq!(duration, Duration::from_millis(0)); + + if let Some(mut driver) = self.inner.shared.driver.try_lock() { + driver.park_timeout(handle, duration) + } + } + + pub(crate) fn shutdown(&mut self, handle: &driver::Handle) { + self.inner.shutdown(handle); + } +} + +impl Clone for Parker { + fn clone(&self) -> Parker { + Parker { + inner: Arc::new(Inner { + state: AtomicUsize::new(EMPTY), + mutex: Mutex::new(()), + condvar: Condvar::new(), + shared: self.inner.shared.clone(), + }), + } + } +} + +impl Unparker { + pub(crate) fn unpark(&self, driver: &driver::Handle) { + self.inner.unpark(driver); + } +} + +impl Inner { + /// Parks the current thread for at most `dur`. + fn park(&self, handle: &driver::Handle) { + // If we were previously notified then we consume this notification and + // return quickly. + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + return; + } + + if let Some(mut driver) = self.shared.driver.try_lock() { + self.park_driver(&mut driver, handle); + } else { + self.park_condvar(); + } + } + + fn park_condvar(&self) { + // Otherwise we need to coordinate going to sleep + let mut m = self.mutex.lock(); + + match self + .state + .compare_exchange(EMPTY, PARKED_CONDVAR, SeqCst, SeqCst) + { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read here, even though we know it will be `NOTIFIED`. + // This is because `unpark` may have been called again since we read + // `NOTIFIED` in the `compare_exchange` above. We must perform an + // acquire operation that synchronizes with that `unpark` to observe + // any writes it made before the call to unpark. To do that we must + // read from the write it made to `state`. + let old = self.state.swap(EMPTY, SeqCst); + debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + + return; + } + Err(actual) => panic!("inconsistent park state; actual = {}", actual), + } + + loop { + m = self.condvar.wait(m).unwrap(); + + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + // got a notification + return; + } + + // spurious wakeup, go back to sleep + } + } + + fn park_driver(&self, driver: &mut Driver, handle: &driver::Handle) { + match self + .state + .compare_exchange(EMPTY, PARKED_DRIVER, SeqCst, SeqCst) + { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read here, even though we know it will be `NOTIFIED`. + // This is because `unpark` may have been called again since we read + // `NOTIFIED` in the `compare_exchange` above. We must perform an + // acquire operation that synchronizes with that `unpark` to observe + // any writes it made before the call to unpark. To do that we must + // read from the write it made to `state`. + let old = self.state.swap(EMPTY, SeqCst); + debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + + return; + } + Err(actual) => panic!("inconsistent park state; actual = {}", actual), + } + + driver.park(handle); + + match self.state.swap(EMPTY, SeqCst) { + NOTIFIED => {} // got a notification, hurray! + PARKED_DRIVER => {} // no notification, alas + n => panic!("inconsistent park_timeout state: {}", n), + } + } + + fn unpark(&self, driver: &driver::Handle) { + // To ensure the unparked thread will observe any writes we made before + // this call, we must perform a release operation that `park` can + // synchronize with. To do that we must write `NOTIFIED` even if `state` + // is already `NOTIFIED`. That is why this must be a swap rather than a + // compare-and-swap that returns if it reads `NOTIFIED` on failure. + match self.state.swap(NOTIFIED, SeqCst) { + EMPTY => {} // no one was waiting + NOTIFIED => {} // already unparked + PARKED_CONDVAR => self.unpark_condvar(), + PARKED_DRIVER => driver.unpark(), + actual => panic!("inconsistent state in unpark; actual = {}", actual), + } + } + + fn unpark_condvar(&self) { + // There is a period between when the parked thread sets `state` to + // `PARKED` (or last checked `state` in the case of a spurious wake + // up) and when it actually waits on `cvar`. If we were to notify + // during this period it would be ignored and then when the parked + // thread went to sleep it would never wake up. Fortunately, it has + // `lock` locked at this stage so we can acquire `lock` to wait until + // it is ready to receive the notification. + // + // Releasing `lock` before the call to `notify_one` means that when the + // parked thread wakes it doesn't get woken only to have to wait for us + // to release `lock`. + drop(self.mutex.lock()); + + self.condvar.notify_one() + } + + fn shutdown(&self, handle: &driver::Handle) { + if let Some(mut driver) = self.shared.driver.try_lock() { + driver.shutdown(handle); + } + + self.condvar.notify_all(); + } +} diff --git a/third_party/rust/tokio/src/runtime/queue.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/queue.rs similarity index 69% rename from third_party/rust/tokio/src/runtime/queue.rs rename to third_party/rust/tokio/src/runtime/scheduler/multi_thread/queue.rs index ad9085a65455..dd66fa2dde16 100644 --- a/third_party/rust/tokio/src/runtime/queue.rs +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/queue.rs @@ -1,39 +1,56 @@ //! Run-queue structures to support a work-stealing scheduler use crate::loom::cell::UnsafeCell; -use crate::loom::sync::atomic::{AtomicU16, AtomicU32}; use crate::loom::sync::Arc; -use crate::runtime::task::{self, Inject}; -use crate::runtime::MetricsBatch; +use crate::runtime::scheduler::multi_thread::{Overflow, Stats}; +use crate::runtime::task; -use std::mem::MaybeUninit; +use std::mem::{self, MaybeUninit}; use std::ptr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; +// Use wider integers when possible to increase ABA resilience. +// +// See issue #5041: . +cfg_has_atomic_u64! { + type UnsignedShort = u32; + type UnsignedLong = u64; + type AtomicUnsignedShort = crate::loom::sync::atomic::AtomicU32; + type AtomicUnsignedLong = crate::loom::sync::atomic::AtomicU64; +} +cfg_not_has_atomic_u64! { + type UnsignedShort = u16; + type UnsignedLong = u32; + type AtomicUnsignedShort = crate::loom::sync::atomic::AtomicU16; + type AtomicUnsignedLong = crate::loom::sync::atomic::AtomicU32; +} + /// Producer handle. May only be used from a single thread. -pub(super) struct Local { +pub(crate) struct Local { inner: Arc>, } /// Consumer handle. May be used from many threads. -pub(super) struct Steal(Arc>); +pub(crate) struct Steal(Arc>); -pub(super) struct Inner { +pub(crate) struct Inner { /// Concurrently updated by many threads. /// - /// Contains two `u16` values. The LSB byte is the "real" head of the queue. - /// The `u16` in the MSB is set by a stealer in process of stealing values. - /// It represents the first value being stolen in the batch. `u16` is used - /// in order to distinguish between `head == tail` and `head == tail - - /// capacity`. + /// Contains two `UnsignedShort` values. The LSB byte is the "real" head of + /// the queue. The `UnsignedShort` in the MSB is set by a stealer in process + /// of stealing values. It represents the first value being stolen in the + /// batch. The `UnsignedShort` indices are intentionally wider than strictly + /// required for buffer indexing in order to provide ABA mitigation and make + /// it possible to distinguish between full and empty buffers. /// - /// When both `u16` values are the same, there is no active stealer. + /// When both `UnsignedShort` values are the same, there is no active + /// stealer. /// /// Tracking an in-progress stealer prevents a wrapping scenario. - head: AtomicU32, + head: AtomicUnsignedLong, /// Only updated by producer thread but read by many threads. - tail: AtomicU16, + tail: AtomicUnsignedShort, /// Elements buffer: Box<[UnsafeCell>>; LOCAL_QUEUE_CAPACITY]>, @@ -65,7 +82,7 @@ fn make_fixed_size(buffer: Box<[T]>) -> Box<[T; LOCAL_QUEUE_CAPACITY]> { } /// Create a new local run-queue -pub(super) fn local() -> (Steal, Local) { +pub(crate) fn local() -> (Steal, Local) { let mut buffer = Vec::with_capacity(LOCAL_QUEUE_CAPACITY); for _ in 0..LOCAL_QUEUE_CAPACITY { @@ -73,8 +90,8 @@ pub(super) fn local() -> (Steal, Local) { } let inner = Arc::new(Inner { - head: AtomicU32::new(0), - tail: AtomicU16::new(0), + head: AtomicUnsignedLong::new(0), + tail: AtomicUnsignedShort::new(0), buffer: make_fixed_size(buffer.into_boxed_slice()), }); @@ -88,25 +105,88 @@ pub(super) fn local() -> (Steal, Local) { } impl Local { - /// Returns true if the queue has entries that can be stealed. - pub(super) fn is_stealable(&self) -> bool { - !self.inner.is_empty() + /// Returns the number of entries in the queue + pub(crate) fn len(&self) -> usize { + self.inner.len() as usize + } + + /// How many tasks can be pushed into the queue + pub(crate) fn remaining_slots(&self) -> usize { + self.inner.remaining_slots() + } + + pub(crate) fn max_capacity(&self) -> usize { + LOCAL_QUEUE_CAPACITY } /// Returns false if there are any entries in the queue /// /// Separate to is_stealable so that refactors of is_stealable to "protect" /// some tasks from stealing won't affect this - pub(super) fn has_tasks(&self) -> bool { + pub(crate) fn has_tasks(&self) -> bool { !self.inner.is_empty() } - /// Pushes a task to the back of the local queue, skipping the LIFO slot. - pub(super) fn push_back( + /// Pushes a batch of tasks to the back of the queue. All tasks must fit in + /// the local queue. + /// + /// # Panics + /// + /// The method panics if there is not enough capacity to fit in the queue. + pub(crate) fn push_back(&mut self, tasks: impl ExactSizeIterator>) { + let len = tasks.len(); + assert!(len <= LOCAL_QUEUE_CAPACITY); + + if len == 0 { + // Nothing to do + return; + } + + let head = self.inner.head.load(Acquire); + let (steal, _) = unpack(head); + + // safety: this is the **only** thread that updates this cell. + let mut tail = unsafe { self.inner.tail.unsync_load() }; + + if tail.wrapping_sub(steal) <= (LOCAL_QUEUE_CAPACITY - len) as UnsignedShort { + // Yes, this if condition is structured a bit weird (first block + // does nothing, second returns an error). It is this way to match + // `push_back_or_overflow`. + } else { + panic!() + } + + for task in tasks { + let idx = tail as usize & MASK; + + self.inner.buffer[idx].with_mut(|ptr| { + // Write the task to the slot + // + // Safety: There is only one producer and the above `if` + // condition ensures we don't touch a cell if there is a + // value, thus no consumer. + unsafe { + ptr::write((*ptr).as_mut_ptr(), task); + } + }); + + tail = tail.wrapping_add(1); + } + + self.inner.tail.store(tail, Release); + } + + /// Pushes a task to the back of the local queue, if there is not enough + /// capacity in the queue, this triggers the overflow operation. + /// + /// When the queue overflows, half of the curent contents of the queue is + /// moved to the given Injection queue. This frees up capacity for more + /// tasks to be pushed into the local queue. + pub(crate) fn push_back_or_overflow>( &mut self, mut task: task::Notified, - inject: &Inject, - metrics: &mut MetricsBatch, + overflow: &O, + stats: &mut Stats, ) { let tail = loop { let head = self.inner.head.load(Acquire); @@ -115,18 +195,18 @@ impl Local { // safety: this is the **only** thread that updates this cell. let tail = unsafe { self.inner.tail.unsync_load() }; - if tail.wrapping_sub(steal) < LOCAL_QUEUE_CAPACITY as u16 { + if tail.wrapping_sub(steal) < LOCAL_QUEUE_CAPACITY as UnsignedShort { // There is capacity for the task break tail; } else if steal != real { // Concurrently stealing, this will free up capacity, so only // push the task onto the inject queue - inject.push(task); + overflow.push(task); return; } else { // Push the current task and half of the queue into the // inject queue. - match self.push_overflow(task, real, tail, inject, metrics) { + match self.push_overflow(task, real, tail, overflow, stats) { Ok(_) => return, // Lost the race, try again Err(v) => { @@ -136,6 +216,11 @@ impl Local { } }; + self.push_back_finish(task, tail); + } + + // Second half of `push_back` + fn push_back_finish(&self, task: task::Notified, tail: UnsignedShort) { // Map the position to a slot index. let idx = tail as usize & MASK; @@ -162,19 +247,19 @@ impl Local { /// workers "missed" some of the tasks during a steal, they will get /// another opportunity. #[inline(never)] - fn push_overflow( + fn push_overflow>( &mut self, task: task::Notified, - head: u16, - tail: u16, - inject: &Inject, - metrics: &mut MetricsBatch, + head: UnsignedShort, + tail: UnsignedShort, + overflow: &O, + stats: &mut Stats, ) -> Result<(), task::Notified> { /// How many elements are we taking from the local queue. /// /// This is one less than the number of tasks pushed to the inject /// queue as we are also inserting the `task` argument. - const NUM_TASKS_TAKEN: u16 = (LOCAL_QUEUE_CAPACITY / 2) as u16; + const NUM_TASKS_TAKEN: UnsignedShort = (LOCAL_QUEUE_CAPACITY / 2) as UnsignedShort; assert_eq!( tail.wrapping_sub(head) as usize, @@ -219,15 +304,15 @@ impl Local { /// An iterator that takes elements out of the run queue. struct BatchTaskIter<'a, T: 'static> { buffer: &'a [UnsafeCell>>; LOCAL_QUEUE_CAPACITY], - head: u32, - i: u32, + head: UnsignedLong, + i: UnsignedLong, } impl<'a, T: 'static> Iterator for BatchTaskIter<'a, T> { type Item = task::Notified; #[inline] fn next(&mut self) -> Option> { - if self.i == u32::from(NUM_TASKS_TAKEN) { + if self.i == UnsignedLong::from(NUM_TASKS_TAKEN) { None } else { let i_idx = self.i.wrapping_add(self.head) as usize & MASK; @@ -246,20 +331,20 @@ impl Local { // safety: The CAS above ensures that no consumer will look at these // values again, and we are the only producer. let batch_iter = BatchTaskIter { - buffer: &*self.inner.buffer, - head: head as u32, + buffer: &self.inner.buffer, + head: head as UnsignedLong, i: 0, }; - inject.push_batch(batch_iter.chain(std::iter::once(task))); + overflow.push_batch(batch_iter.chain(std::iter::once(task))); // Add 1 to factor in the task currently being scheduled. - metrics.incr_overflow_count(); + stats.incr_overflow_count(); Ok(()) } /// Pops a task from the local queue. - pub(super) fn pop(&mut self) -> Option> { + pub(crate) fn pop(&mut self) -> Option> { let mut head = self.inner.head.load(Acquire); let idx = loop { @@ -301,15 +386,15 @@ impl Local { } impl Steal { - pub(super) fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.0.is_empty() } /// Steals half the tasks from self and place them into `dst`. - pub(super) fn steal_into( + pub(crate) fn steal_into( &self, dst: &mut Local, - dst_metrics: &mut MetricsBatch, + dst_stats: &mut Stats, ) -> Option> { // Safety: the caller is the only thread that mutates `dst.tail` and // holds a mutable reference. @@ -320,7 +405,7 @@ impl Steal { // from `dst` there may not be enough capacity to steal. let (steal, _) = unpack(dst.inner.head.load(Acquire)); - if dst_tail.wrapping_sub(steal) > LOCAL_QUEUE_CAPACITY as u16 / 2 { + if dst_tail.wrapping_sub(steal) > LOCAL_QUEUE_CAPACITY as UnsignedShort / 2 { // we *could* try to steal less here, but for simplicity, we're just // going to abort. return None; @@ -335,7 +420,8 @@ impl Steal { return None; } - dst_metrics.incr_steal_count(n); + dst_stats.incr_steal_count(n as u16); + dst_stats.incr_steal_operations(); // We are returning a task here n -= 1; @@ -360,7 +446,7 @@ impl Steal { // Steal tasks from `self`, placing them into `dst`. Returns the number of // tasks that were stolen. - fn steal_into2(&self, dst: &mut Local, dst_tail: u16) -> u16 { + fn steal_into2(&self, dst: &mut Local, dst_tail: UnsignedShort) -> UnsignedShort { let mut prev_packed = self.0.head.load(Acquire); let mut next_packed; @@ -402,7 +488,11 @@ impl Steal { } }; - assert!(n <= LOCAL_QUEUE_CAPACITY as u16 / 2, "actual = {}", n); + assert!( + n <= LOCAL_QUEUE_CAPACITY as UnsignedShort / 2, + "actual = {}", + n + ); let (first, _) = unpack(next_packed); @@ -479,7 +569,14 @@ impl Drop for Local { } impl Inner { - fn len(&self) -> u16 { + fn remaining_slots(&self) -> usize { + let (steal, _) = unpack(self.head.load(Acquire)); + let tail = self.tail.load(Acquire); + + LOCAL_QUEUE_CAPACITY - (tail.wrapping_sub(steal) as usize) + } + + fn len(&self) -> UnsignedShort { let (_, head) = unpack(self.head.load(Acquire)); let tail = self.tail.load(Acquire); @@ -493,16 +590,16 @@ impl Inner { /// Split the head value into the real head and the index a stealer is working /// on. -fn unpack(n: u32) -> (u16, u16) { - let real = n & u16::MAX as u32; - let steal = n >> 16; +fn unpack(n: UnsignedLong) -> (UnsignedShort, UnsignedShort) { + let real = n & UnsignedShort::MAX as UnsignedLong; + let steal = n >> (mem::size_of::() * 8); - (steal as u16, real as u16) + (steal as UnsignedShort, real as UnsignedShort) } /// Join the two head values -fn pack(steal: u16, real: u16) -> u32 { - (real as u32) | ((steal as u32) << 16) +fn pack(steal: UnsignedShort, real: UnsignedShort) -> UnsignedLong { + (real as UnsignedLong) | ((steal as UnsignedLong) << (mem::size_of::() * 8)) } #[test] diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/stats.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/stats.rs new file mode 100644 index 000000000000..f01daaa1bffb --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/stats.rs @@ -0,0 +1,140 @@ +use crate::runtime::{Config, MetricsBatch, WorkerMetrics}; + +use std::cmp; +use std::time::{Duration, Instant}; + +/// Per-worker statistics. This is used for both tuning the scheduler and +/// reporting runtime-level metrics/stats. +pub(crate) struct Stats { + /// The metrics batch used to report runtime-level metrics/stats to the + /// user. + batch: MetricsBatch, + + /// Instant at which work last resumed (continued after park). + /// + /// This duplicates the value stored in `MetricsBatch`. We will unify + /// `Stats` and `MetricsBatch` when we stabilize metrics. + processing_scheduled_tasks_started_at: Instant, + + /// Number of tasks polled in the batch of scheduled tasks + tasks_polled_in_batch: usize, + + /// Exponentially-weighted moving average of time spent polling scheduled a + /// task. + /// + /// Tracked in nanoseconds, stored as a f64 since that is what we use with + /// the EWMA calculations + task_poll_time_ewma: f64, +} + +/// How to weigh each individual poll time, value is plucked from thin air. +const TASK_POLL_TIME_EWMA_ALPHA: f64 = 0.1; + +/// Ideally, we wouldn't go above this, value is plucked from thin air. +const TARGET_GLOBAL_QUEUE_INTERVAL: f64 = Duration::from_micros(200).as_nanos() as f64; + +/// Max value for the global queue interval. This is 2x the previous default +const MAX_TASKS_POLLED_PER_GLOBAL_QUEUE_INTERVAL: u32 = 127; + +/// This is the previous default +const TARGET_TASKS_POLLED_PER_GLOBAL_QUEUE_INTERVAL: u32 = 61; + +impl Stats { + pub(crate) fn new(worker_metrics: &WorkerMetrics) -> Stats { + // Seed the value with what we hope to see. + let task_poll_time_ewma = + TARGET_GLOBAL_QUEUE_INTERVAL / TARGET_TASKS_POLLED_PER_GLOBAL_QUEUE_INTERVAL as f64; + + Stats { + batch: MetricsBatch::new(worker_metrics), + processing_scheduled_tasks_started_at: Instant::now(), + tasks_polled_in_batch: 0, + task_poll_time_ewma, + } + } + + pub(crate) fn tuned_global_queue_interval(&self, config: &Config) -> u32 { + // If an interval is explicitly set, don't tune. + if let Some(configured) = config.global_queue_interval { + return configured; + } + + // As of Rust 1.45, casts from f64 -> u32 are saturating, which is fine here. + let tasks_per_interval = (TARGET_GLOBAL_QUEUE_INTERVAL / self.task_poll_time_ewma) as u32; + + cmp::max( + // We don't want to return less than 2 as that would result in the + // global queue always getting checked first. + 2, + cmp::min( + MAX_TASKS_POLLED_PER_GLOBAL_QUEUE_INTERVAL, + tasks_per_interval, + ), + ) + } + + pub(crate) fn submit(&mut self, to: &WorkerMetrics) { + self.batch.submit(to); + } + + pub(crate) fn about_to_park(&mut self) { + self.batch.about_to_park(); + } + + pub(crate) fn inc_local_schedule_count(&mut self) { + self.batch.inc_local_schedule_count(); + } + + pub(crate) fn start_processing_scheduled_tasks(&mut self) { + self.batch.start_processing_scheduled_tasks(); + + self.processing_scheduled_tasks_started_at = Instant::now(); + self.tasks_polled_in_batch = 0; + } + + pub(crate) fn end_processing_scheduled_tasks(&mut self) { + self.batch.end_processing_scheduled_tasks(); + + // Update the EWMA task poll time + if self.tasks_polled_in_batch > 0 { + let now = Instant::now(); + + // If we "overflow" this conversion, we have bigger problems than + // slightly off stats. + let elapsed = (now - self.processing_scheduled_tasks_started_at).as_nanos() as f64; + let num_polls = self.tasks_polled_in_batch as f64; + + // Calculate the mean poll duration for a single task in the batch + let mean_poll_duration = elapsed / num_polls; + + // Compute the alpha weighted by the number of tasks polled this batch. + let weighted_alpha = 1.0 - (1.0 - TASK_POLL_TIME_EWMA_ALPHA).powf(num_polls); + + // Now compute the new weighted average task poll time. + self.task_poll_time_ewma = weighted_alpha * mean_poll_duration + + (1.0 - weighted_alpha) * self.task_poll_time_ewma; + } + } + + pub(crate) fn start_poll(&mut self) { + self.batch.start_poll(); + + self.tasks_polled_in_batch += 1; + } + + pub(crate) fn end_poll(&mut self) { + self.batch.end_poll(); + } + + pub(crate) fn incr_steal_count(&mut self, by: u16) { + self.batch.incr_steal_count(by); + } + + pub(crate) fn incr_steal_operations(&mut self) { + self.batch.incr_steal_operations(); + } + + pub(crate) fn incr_overflow_count(&mut self) { + self.batch.incr_overflow_count(); + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace.rs new file mode 100644 index 000000000000..7b4aeb5c1d7a --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace.rs @@ -0,0 +1,61 @@ +use crate::loom::sync::atomic::{AtomicBool, Ordering}; +use crate::loom::sync::{Barrier, Mutex}; +use crate::runtime::dump::Dump; +use crate::runtime::scheduler::multi_thread::Handle; +use crate::sync::notify::Notify; + +/// Tracing status of the worker. +pub(super) struct TraceStatus { + pub(super) trace_requested: AtomicBool, + pub(super) trace_start: Barrier, + pub(super) trace_end: Barrier, + pub(super) result_ready: Notify, + pub(super) trace_result: Mutex>, +} + +impl TraceStatus { + pub(super) fn new(remotes_len: usize) -> Self { + Self { + trace_requested: AtomicBool::new(false), + trace_start: Barrier::new(remotes_len), + trace_end: Barrier::new(remotes_len), + result_ready: Notify::new(), + trace_result: Mutex::new(None), + } + } + + pub(super) fn trace_requested(&self) -> bool { + self.trace_requested.load(Ordering::Relaxed) + } + + pub(super) async fn start_trace_request(&self, handle: &Handle) { + while self + .trace_requested + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + handle.notify_all(); + crate::task::yield_now().await; + } + } + + pub(super) fn stash_result(&self, dump: Dump) { + let _ = self.trace_result.lock().insert(dump); + self.result_ready.notify_one(); + } + + pub(super) fn take_result(&self) -> Option { + self.trace_result.lock().take() + } + + pub(super) async fn end_trace_request(&self, handle: &Handle) { + while self + .trace_requested + .compare_exchange(true, false, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + handle.notify_all(); + crate::task::yield_now().await; + } + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace_mock.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace_mock.rs new file mode 100644 index 000000000000..2c17a4e38b58 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/trace_mock.rs @@ -0,0 +1,11 @@ +pub(super) struct TraceStatus {} + +impl TraceStatus { + pub(super) fn new(_: usize) -> Self { + Self {} + } + + pub(super) fn trace_requested(&self) -> bool { + false + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker.rs new file mode 100644 index 000000000000..6ae11463373c --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -0,0 +1,1216 @@ +//! A scheduler is initialized with a fixed number of workers. Each worker is +//! driven by a thread. Each worker has a "core" which contains data such as the +//! run queue and other state. When `block_in_place` is called, the worker's +//! "core" is handed off to a new thread allowing the scheduler to continue to +//! make progress while the originating thread blocks. +//! +//! # Shutdown +//! +//! Shutting down the runtime involves the following steps: +//! +//! 1. The Shared::close method is called. This closes the inject queue and +//! OwnedTasks instance and wakes up all worker threads. +//! +//! 2. Each worker thread observes the close signal next time it runs +//! Core::maintenance by checking whether the inject queue is closed. +//! The Core::is_shutdown flag is set to true. +//! +//! 3. The worker thread calls `pre_shutdown` in parallel. Here, the worker +//! will keep removing tasks from OwnedTasks until it is empty. No new +//! tasks can be pushed to the OwnedTasks during or after this step as it +//! was closed in step 1. +//! +//! 5. The workers call Shared::shutdown to enter the single-threaded phase of +//! shutdown. These calls will push their core to Shared::shutdown_cores, +//! and the last thread to push its core will finish the shutdown procedure. +//! +//! 6. The local run queue of each core is emptied, then the inject queue is +//! emptied. +//! +//! At this point, shutdown has completed. It is not possible for any of the +//! collections to contain any tasks at this point, as each collection was +//! closed first, then emptied afterwards. +//! +//! ## Spawns during shutdown +//! +//! When spawning tasks during shutdown, there are two cases: +//! +//! * The spawner observes the OwnedTasks being open, and the inject queue is +//! closed. +//! * The spawner observes the OwnedTasks being closed and doesn't check the +//! inject queue. +//! +//! The first case can only happen if the OwnedTasks::bind call happens before +//! or during step 1 of shutdown. In this case, the runtime will clean up the +//! task in step 3 of shutdown. +//! +//! In the latter case, the task was not spawned and the task is immediately +//! cancelled by the spawner. +//! +//! The correctness of shutdown requires both the inject queue and OwnedTasks +//! collection to have a closed bit. With a close bit on only the inject queue, +//! spawning could run in to a situation where a task is successfully bound long +//! after the runtime has shut down. With a close bit on only the OwnedTasks, +//! the first spawning situation could result in the notification being pushed +//! to the inject queue after step 6 of shutdown, which would leave a task in +//! the inject queue indefinitely. This would be a ref-count cycle and a memory +//! leak. + +use crate::loom::sync::{Arc, Mutex}; +use crate::runtime; +use crate::runtime::context; +use crate::runtime::scheduler::multi_thread::{ + idle, queue, Counters, Handle, Idle, Overflow, Parker, Stats, TraceStatus, Unparker, +}; +use crate::runtime::scheduler::{inject, Defer, Lock}; +use crate::runtime::task::OwnedTasks; +use crate::runtime::{ + blocking, coop, driver, scheduler, task, Config, SchedulerMetrics, WorkerMetrics, +}; +use crate::util::atomic_cell::AtomicCell; +use crate::util::rand::{FastRand, RngSeedGenerator}; + +use std::cell::RefCell; +use std::task::Waker; +use std::time::Duration; + +cfg_metrics! { + mod metrics; +} + +cfg_taskdump! { + mod taskdump; +} + +cfg_not_taskdump! { + mod taskdump_mock; +} + +/// A scheduler worker +pub(super) struct Worker { + /// Reference to scheduler's handle + handle: Arc, + + /// Index holding this worker's remote state + index: usize, + + /// Used to hand-off a worker's core to another thread. + core: AtomicCell, +} + +/// Core data +struct Core { + /// Used to schedule bookkeeping tasks every so often. + tick: u32, + + /// When a task is scheduled from a worker, it is stored in this slot. The + /// worker will check this slot for a task **before** checking the run + /// queue. This effectively results in the **last** scheduled task to be run + /// next (LIFO). This is an optimization for improving locality which + /// benefits message passing patterns and helps to reduce latency. + lifo_slot: Option, + + /// When `true`, locally scheduled tasks go to the LIFO slot. When `false`, + /// they go to the back of the `run_queue`. + lifo_enabled: bool, + + /// The worker-local run queue. + run_queue: queue::Local>, + + /// True if the worker is currently searching for more work. Searching + /// involves attempting to steal from other workers. + is_searching: bool, + + /// True if the scheduler is being shutdown + is_shutdown: bool, + + /// True if the scheduler is being traced + is_traced: bool, + + /// Parker + /// + /// Stored in an `Option` as the parker is added / removed to make the + /// borrow checker happy. + park: Option, + + /// Per-worker runtime stats + stats: Stats, + + /// How often to check the global queue + global_queue_interval: u32, + + /// Fast random number generator. + rand: FastRand, +} + +/// State shared across all workers +pub(crate) struct Shared { + /// Per-worker remote state. All other workers have access to this and is + /// how they communicate between each other. + remotes: Box<[Remote]>, + + /// Global task queue used for: + /// 1. Submit work to the scheduler while **not** currently on a worker thread. + /// 2. Submit work to the scheduler when a worker run queue is saturated + pub(super) inject: inject::Shared>, + + /// Coordinates idle workers + idle: Idle, + + /// Collection of all active tasks spawned onto this executor. + pub(super) owned: OwnedTasks>, + + /// Data synchronized by the scheduler mutex + pub(super) synced: Mutex, + + /// Cores that have observed the shutdown signal + /// + /// The core is **not** placed back in the worker to avoid it from being + /// stolen by a thread that was spawned as part of `block_in_place`. + #[allow(clippy::vec_box)] // we're moving an already-boxed value + shutdown_cores: Mutex>>, + + /// The number of cores that have observed the trace signal. + pub(super) trace_status: TraceStatus, + + /// Scheduler configuration options + config: Config, + + /// Collects metrics from the runtime. + pub(super) scheduler_metrics: SchedulerMetrics, + + pub(super) worker_metrics: Box<[WorkerMetrics]>, + + /// Only held to trigger some code on drop. This is used to get internal + /// runtime metrics that can be useful when doing performance + /// investigations. This does nothing (empty struct, no drop impl) unless + /// the `tokio_internal_mt_counters` cfg flag is set. + _counters: Counters, +} + +/// Data synchronized by the scheduler mutex +pub(crate) struct Synced { + /// Synchronized state for `Idle`. + pub(super) idle: idle::Synced, + + /// Synchronized state for `Inject`. + pub(crate) inject: inject::Synced, +} + +/// Used to communicate with a worker from other threads. +struct Remote { + /// Steals tasks from this worker. + pub(super) steal: queue::Steal>, + + /// Unparks the associated worker thread + unpark: Unparker, +} + +/// Thread-local context +pub(crate) struct Context { + /// Worker + worker: Arc, + + /// Core data + core: RefCell>>, + + /// Tasks to wake after resource drivers are polled. This is mostly to + /// handle yielded tasks. + pub(crate) defer: Defer, +} + +/// Starts the workers +pub(crate) struct Launch(Vec>); + +/// Running a task may consume the core. If the core is still available when +/// running the task completes, it is returned. Otherwise, the worker will need +/// to stop processing. +type RunResult = Result, ()>; + +/// A task handle +type Task = task::Task>; + +/// A notified task handle +type Notified = task::Notified>; + +/// Value picked out of thin-air. Running the LIFO slot a handful of times +/// seemms sufficient to benefit from locality. More than 3 times probably is +/// overweighing. The value can be tuned in the future with data that shows +/// improvements. +const MAX_LIFO_POLLS_PER_TICK: usize = 3; + +pub(super) fn create( + size: usize, + park: Parker, + driver_handle: driver::Handle, + blocking_spawner: blocking::Spawner, + seed_generator: RngSeedGenerator, + config: Config, +) -> (Arc, Launch) { + let mut cores = Vec::with_capacity(size); + let mut remotes = Vec::with_capacity(size); + let mut worker_metrics = Vec::with_capacity(size); + + // Create the local queues + for _ in 0..size { + let (steal, run_queue) = queue::local(); + + let park = park.clone(); + let unpark = park.unpark(); + let metrics = WorkerMetrics::from_config(&config); + let stats = Stats::new(&metrics); + + cores.push(Box::new(Core { + tick: 0, + lifo_slot: None, + lifo_enabled: !config.disable_lifo_slot, + run_queue, + is_searching: false, + is_shutdown: false, + is_traced: false, + park: Some(park), + global_queue_interval: stats.tuned_global_queue_interval(&config), + stats, + rand: FastRand::from_seed(config.seed_generator.next_seed()), + })); + + remotes.push(Remote { steal, unpark }); + worker_metrics.push(metrics); + } + + let (idle, idle_synced) = Idle::new(size); + let (inject, inject_synced) = inject::Shared::new(); + + let remotes_len = remotes.len(); + let handle = Arc::new(Handle { + shared: Shared { + remotes: remotes.into_boxed_slice(), + inject, + idle, + owned: OwnedTasks::new(), + synced: Mutex::new(Synced { + idle: idle_synced, + inject: inject_synced, + }), + shutdown_cores: Mutex::new(vec![]), + trace_status: TraceStatus::new(remotes_len), + config, + scheduler_metrics: SchedulerMetrics::new(), + worker_metrics: worker_metrics.into_boxed_slice(), + _counters: Counters, + }, + driver: driver_handle, + blocking_spawner, + seed_generator, + }); + + let mut launch = Launch(vec![]); + + for (index, core) in cores.drain(..).enumerate() { + launch.0.push(Arc::new(Worker { + handle: handle.clone(), + index, + core: AtomicCell::new(Some(core)), + })); + } + + (handle, launch) +} + +#[track_caller] +pub(crate) fn block_in_place(f: F) -> R +where + F: FnOnce() -> R, +{ + // Try to steal the worker core back + struct Reset { + take_core: bool, + budget: coop::Budget, + } + + impl Drop for Reset { + fn drop(&mut self) { + with_current(|maybe_cx| { + if let Some(cx) = maybe_cx { + if self.take_core { + let core = cx.worker.core.take(); + let mut cx_core = cx.core.borrow_mut(); + assert!(cx_core.is_none()); + *cx_core = core; + } + + // Reset the task budget as we are re-entering the + // runtime. + coop::set(self.budget); + } + }); + } + } + + let mut had_entered = false; + let mut take_core = false; + + let setup_result = with_current(|maybe_cx| { + match ( + crate::runtime::context::current_enter_context(), + maybe_cx.is_some(), + ) { + (context::EnterRuntime::Entered { .. }, true) => { + // We are on a thread pool runtime thread, so we just need to + // set up blocking. + had_entered = true; + } + ( + context::EnterRuntime::Entered { + allow_block_in_place, + }, + false, + ) => { + // We are on an executor, but _not_ on the thread pool. That is + // _only_ okay if we are in a thread pool runtime's block_on + // method: + if allow_block_in_place { + had_entered = true; + return Ok(()); + } else { + // This probably means we are on the current_thread runtime or in a + // LocalSet, where it is _not_ okay to block. + return Err( + "can call blocking only when running on the multi-threaded runtime", + ); + } + } + (context::EnterRuntime::NotEntered, true) => { + // This is a nested call to block_in_place (we already exited). + // All the necessary setup has already been done. + return Ok(()); + } + (context::EnterRuntime::NotEntered, false) => { + // We are outside of the tokio runtime, so blocking is fine. + // We can also skip all of the thread pool blocking setup steps. + return Ok(()); + } + } + + let cx = maybe_cx.expect("no .is_some() == false cases above should lead here"); + + // Get the worker core. If none is set, then blocking is fine! + let core = match cx.core.borrow_mut().take() { + Some(core) => core, + None => return Ok(()), + }; + + // We are taking the core from the context and sending it to another + // thread. + take_core = true; + + // The parker should be set here + assert!(core.park.is_some()); + + // In order to block, the core must be sent to another thread for + // execution. + // + // First, move the core back into the worker's shared core slot. + cx.worker.core.set(core); + + // Next, clone the worker handle and send it to a new thread for + // processing. + // + // Once the blocking task is done executing, we will attempt to + // steal the core back. + let worker = cx.worker.clone(); + runtime::spawn_blocking(move || run(worker)); + Ok(()) + }); + + if let Err(panic_message) = setup_result { + panic!("{}", panic_message); + } + + if had_entered { + // Unset the current task's budget. Blocking sections are not + // constrained by task budgets. + let _reset = Reset { + take_core, + budget: coop::stop(), + }; + + crate::runtime::context::exit_runtime(f) + } else { + f() + } +} + +impl Launch { + pub(crate) fn launch(mut self) { + for worker in self.0.drain(..) { + runtime::spawn_blocking(move || run(worker)); + } + } +} + +fn run(worker: Arc) { + struct AbortOnPanic; + + impl Drop for AbortOnPanic { + fn drop(&mut self) { + if std::thread::panicking() { + eprintln!("worker thread panicking; aborting process"); + std::process::abort(); + } + } + } + + // Catching panics on worker threads in tests is quite tricky. Instead, when + // debug assertions are enabled, we just abort the process. + #[cfg(debug_assertions)] + let _abort_on_panic = AbortOnPanic; + + // Acquire a core. If this fails, then another thread is running this + // worker and there is nothing further to do. + let core = match worker.core.take() { + Some(core) => core, + None => return, + }; + + let handle = scheduler::Handle::MultiThread(worker.handle.clone()); + + crate::runtime::context::enter_runtime(&handle, true, |_| { + // Set the worker context. + let cx = scheduler::Context::MultiThread(Context { + worker, + core: RefCell::new(None), + defer: Defer::new(), + }); + + context::set_scheduler(&cx, || { + let cx = cx.expect_multi_thread(); + + // This should always be an error. It only returns a `Result` to support + // using `?` to short circuit. + assert!(cx.run(core).is_err()); + + // Check if there are any deferred tasks to notify. This can happen when + // the worker core is lost due to `block_in_place()` being called from + // within the task. + cx.defer.wake(); + }); + }); +} + +impl Context { + fn run(&self, mut core: Box) -> RunResult { + // Reset `lifo_enabled` here in case the core was previously stolen from + // a task that had the LIFO slot disabled. + self.reset_lifo_enabled(&mut core); + + // Start as "processing" tasks as polling tasks from the local queue + // will be one of the first things we do. + core.stats.start_processing_scheduled_tasks(); + + while !core.is_shutdown { + self.assert_lifo_enabled_is_correct(&core); + + if core.is_traced { + core = self.worker.handle.trace_core(core); + } + + // Increment the tick + core.tick(); + + // Run maintenance, if needed + core = self.maintenance(core); + + // First, check work available to the current worker. + if let Some(task) = core.next_task(&self.worker) { + core = self.run_task(task, core)?; + continue; + } + + // We consumed all work in the queues and will start searching for work. + core.stats.end_processing_scheduled_tasks(); + + // There is no more **local** work to process, try to steal work + // from other workers. + if let Some(task) = core.steal_work(&self.worker) { + // Found work, switch back to processing + core.stats.start_processing_scheduled_tasks(); + core = self.run_task(task, core)?; + } else { + // Wait for work + core = if !self.defer.is_empty() { + self.park_timeout(core, Some(Duration::from_millis(0))) + } else { + self.park(core) + }; + } + } + + core.pre_shutdown(&self.worker); + + // Signal shutdown + self.worker.handle.shutdown_core(core); + Err(()) + } + + fn run_task(&self, task: Notified, mut core: Box) -> RunResult { + let task = self.worker.handle.shared.owned.assert_owner(task); + + // Make sure the worker is not in the **searching** state. This enables + // another idle worker to try to steal work. + core.transition_from_searching(&self.worker); + + self.assert_lifo_enabled_is_correct(&core); + + // Measure the poll start time. Note that we may end up polling other + // tasks under this measurement. In this case, the tasks came from the + // LIFO slot and are considered part of the current task for scheduling + // purposes. These tasks inherent the "parent"'s limits. + core.stats.start_poll(); + + // Make the core available to the runtime context + *self.core.borrow_mut() = Some(core); + + // Run the task + coop::budget(|| { + task.run(); + let mut lifo_polls = 0; + + // As long as there is budget remaining and a task exists in the + // `lifo_slot`, then keep running. + loop { + // Check if we still have the core. If not, the core was stolen + // by another worker. + let mut core = match self.core.borrow_mut().take() { + Some(core) => core, + None => { + // In this case, we cannot call `reset_lifo_enabled()` + // because the core was stolen. The stealer will handle + // that at the top of `Context::run` + return Err(()); + } + }; + + // Check for a task in the LIFO slot + let task = match core.lifo_slot.take() { + Some(task) => task, + None => { + self.reset_lifo_enabled(&mut core); + core.stats.end_poll(); + return Ok(core); + } + }; + + if !coop::has_budget_remaining() { + core.stats.end_poll(); + + // Not enough budget left to run the LIFO task, push it to + // the back of the queue and return. + core.run_queue.push_back_or_overflow( + task, + &*self.worker.handle, + &mut core.stats, + ); + // If we hit this point, the LIFO slot should be enabled. + // There is no need to reset it. + debug_assert!(core.lifo_enabled); + return Ok(core); + } + + // Track that we are about to run a task from the LIFO slot. + lifo_polls += 1; + super::counters::inc_lifo_schedules(); + + // Disable the LIFO slot if we reach our limit + // + // In ping-ping style workloads where task A notifies task B, + // which notifies task A again, continuously prioritizing the + // LIFO slot can cause starvation as these two tasks will + // repeatedly schedule the other. To mitigate this, we limit the + // number of times the LIFO slot is prioritized. + if lifo_polls >= MAX_LIFO_POLLS_PER_TICK { + core.lifo_enabled = false; + super::counters::inc_lifo_capped(); + } + + // Run the LIFO task, then loop + *self.core.borrow_mut() = Some(core); + let task = self.worker.handle.shared.owned.assert_owner(task); + task.run(); + } + }) + } + + fn reset_lifo_enabled(&self, core: &mut Core) { + core.lifo_enabled = !self.worker.handle.shared.config.disable_lifo_slot; + } + + fn assert_lifo_enabled_is_correct(&self, core: &Core) { + debug_assert_eq!( + core.lifo_enabled, + !self.worker.handle.shared.config.disable_lifo_slot + ); + } + + fn maintenance(&self, mut core: Box) -> Box { + if core.tick % self.worker.handle.shared.config.event_interval == 0 { + super::counters::inc_num_maintenance(); + + core.stats.end_processing_scheduled_tasks(); + + // Call `park` with a 0 timeout. This enables the I/O driver, timer, ... + // to run without actually putting the thread to sleep. + core = self.park_timeout(core, Some(Duration::from_millis(0))); + + // Run regularly scheduled maintenance + core.maintenance(&self.worker); + + core.stats.start_processing_scheduled_tasks(); + } + + core + } + + /// Parks the worker thread while waiting for tasks to execute. + /// + /// This function checks if indeed there's no more work left to be done before parking. + /// Also important to notice that, before parking, the worker thread will try to take + /// ownership of the Driver (IO/Time) and dispatch any events that might have fired. + /// Whenever a worker thread executes the Driver loop, all waken tasks are scheduled + /// in its own local queue until the queue saturates (ntasks > LOCAL_QUEUE_CAPACITY). + /// When the local queue is saturated, the overflow tasks are added to the injection queue + /// from where other workers can pick them up. + /// Also, we rely on the workstealing algorithm to spread the tasks amongst workers + /// after all the IOs get dispatched + fn park(&self, mut core: Box) -> Box { + if let Some(f) = &self.worker.handle.shared.config.before_park { + f(); + } + + if core.transition_to_parked(&self.worker) { + while !core.is_shutdown && !core.is_traced { + core.stats.about_to_park(); + core = self.park_timeout(core, None); + + // Run regularly scheduled maintenance + core.maintenance(&self.worker); + + if core.transition_from_parked(&self.worker) { + break; + } + } + } + + if let Some(f) = &self.worker.handle.shared.config.after_unpark { + f(); + } + core + } + + fn park_timeout(&self, mut core: Box, duration: Option) -> Box { + self.assert_lifo_enabled_is_correct(&core); + + // Take the parker out of core + let mut park = core.park.take().expect("park missing"); + + // Store `core` in context + *self.core.borrow_mut() = Some(core); + + // Park thread + if let Some(timeout) = duration { + park.park_timeout(&self.worker.handle.driver, timeout); + } else { + park.park(&self.worker.handle.driver); + } + + self.defer.wake(); + + // Remove `core` from context + core = self.core.borrow_mut().take().expect("core missing"); + + // Place `park` back in `core` + core.park = Some(park); + + if core.should_notify_others() { + self.worker.handle.notify_parked_local(); + } + + core + } + + pub(crate) fn defer(&self, waker: &Waker) { + self.defer.defer(waker); + } +} + +impl Core { + /// Increment the tick + fn tick(&mut self) { + self.tick = self.tick.wrapping_add(1); + } + + /// Return the next notified task available to this worker. + fn next_task(&mut self, worker: &Worker) -> Option { + if self.tick % self.global_queue_interval == 0 { + // Update the global queue interval, if needed + self.tune_global_queue_interval(worker); + + worker + .handle + .next_remote_task() + .or_else(|| self.next_local_task()) + } else { + let maybe_task = self.next_local_task(); + + if maybe_task.is_some() { + return maybe_task; + } + + if worker.inject().is_empty() { + return None; + } + + // Other threads can only **remove** tasks from the current worker's + // `run_queue`. So, we can be confident that by the time we call + // `run_queue.push_back` below, there will be *at least* `cap` + // available slots in the queue. + let cap = usize::min( + self.run_queue.remaining_slots(), + self.run_queue.max_capacity() / 2, + ); + + // The worker is currently idle, pull a batch of work from the + // injection queue. We don't want to pull *all* the work so other + // workers can also get some. + let n = usize::min( + worker.inject().len() / worker.handle.shared.remotes.len() + 1, + cap, + ); + + let mut synced = worker.handle.shared.synced.lock(); + // safety: passing in the correct `inject::Synced`. + let mut tasks = unsafe { worker.inject().pop_n(&mut synced.inject, n) }; + + // Pop the first task to return immedietly + let ret = tasks.next(); + + // Push the rest of the on the run queue + self.run_queue.push_back(tasks); + + ret + } + } + + fn next_local_task(&mut self) -> Option { + self.lifo_slot.take().or_else(|| self.run_queue.pop()) + } + + /// Function responsible for stealing tasks from another worker + /// + /// Note: Only if less than half the workers are searching for tasks to steal + /// a new worker will actually try to steal. The idea is to make sure not all + /// workers will be trying to steal at the same time. + fn steal_work(&mut self, worker: &Worker) -> Option { + if !self.transition_to_searching(worker) { + return None; + } + + let num = worker.handle.shared.remotes.len(); + // Start from a random worker + let start = self.rand.fastrand_n(num as u32) as usize; + + for i in 0..num { + let i = (start + i) % num; + + // Don't steal from ourself! We know we don't have work. + if i == worker.index { + continue; + } + + let target = &worker.handle.shared.remotes[i]; + if let Some(task) = target + .steal + .steal_into(&mut self.run_queue, &mut self.stats) + { + return Some(task); + } + } + + // Fallback on checking the global queue + worker.handle.next_remote_task() + } + + fn transition_to_searching(&mut self, worker: &Worker) -> bool { + if !self.is_searching { + self.is_searching = worker.handle.shared.idle.transition_worker_to_searching(); + } + + self.is_searching + } + + fn transition_from_searching(&mut self, worker: &Worker) { + if !self.is_searching { + return; + } + + self.is_searching = false; + worker.handle.transition_worker_from_searching(); + } + + fn has_tasks(&self) -> bool { + self.lifo_slot.is_some() || self.run_queue.has_tasks() + } + + fn should_notify_others(&self) -> bool { + // If there are tasks available to steal, but this worker is not + // looking for tasks to steal, notify another worker. + if self.is_searching { + return false; + } + self.lifo_slot.is_some() as usize + self.run_queue.len() > 1 + } + + /// Prepares the worker state for parking. + /// + /// Returns true if the transition happened, false if there is work to do first. + fn transition_to_parked(&mut self, worker: &Worker) -> bool { + // Workers should not park if they have work to do + if self.has_tasks() || self.is_traced { + return false; + } + + // When the final worker transitions **out** of searching to parked, it + // must check all the queues one last time in case work materialized + // between the last work scan and transitioning out of searching. + let is_last_searcher = worker.handle.shared.idle.transition_worker_to_parked( + &worker.handle.shared, + worker.index, + self.is_searching, + ); + + // The worker is no longer searching. Setting this is the local cache + // only. + self.is_searching = false; + + if is_last_searcher { + worker.handle.notify_if_work_pending(); + } + + true + } + + /// Returns `true` if the transition happened. + fn transition_from_parked(&mut self, worker: &Worker) -> bool { + // If a task is in the lifo slot/run queue, then we must unpark regardless of + // being notified + if self.has_tasks() { + // When a worker wakes, it should only transition to the "searching" + // state when the wake originates from another worker *or* a new task + // is pushed. We do *not* want the worker to transition to "searching" + // when it wakes when the I/O driver receives new events. + self.is_searching = !worker + .handle + .shared + .idle + .unpark_worker_by_id(&worker.handle.shared, worker.index); + return true; + } + + if worker + .handle + .shared + .idle + .is_parked(&worker.handle.shared, worker.index) + { + return false; + } + + // When unparked, the worker is in the searching state. + self.is_searching = true; + true + } + + /// Runs maintenance work such as checking the pool's state. + fn maintenance(&mut self, worker: &Worker) { + self.stats + .submit(&worker.handle.shared.worker_metrics[worker.index]); + + if !self.is_shutdown { + // Check if the scheduler has been shutdown + let synced = worker.handle.shared.synced.lock(); + self.is_shutdown = worker.inject().is_closed(&synced.inject); + } + + if !self.is_traced { + // Check if the worker should be tracing. + self.is_traced = worker.handle.shared.trace_status.trace_requested(); + } + } + + /// Signals all tasks to shut down, and waits for them to complete. Must run + /// before we enter the single-threaded phase of shutdown processing. + fn pre_shutdown(&mut self, worker: &Worker) { + // Signal to all tasks to shut down. + worker.handle.shared.owned.close_and_shutdown_all(); + + self.stats + .submit(&worker.handle.shared.worker_metrics[worker.index]); + } + + /// Shuts down the core. + fn shutdown(&mut self, handle: &Handle) { + // Take the core + let mut park = self.park.take().expect("park missing"); + + // Drain the queue + while self.next_local_task().is_some() {} + + park.shutdown(&handle.driver); + } + + fn tune_global_queue_interval(&mut self, worker: &Worker) { + let next = self + .stats + .tuned_global_queue_interval(&worker.handle.shared.config); + + debug_assert!(next > 1); + + // Smooth out jitter + if abs_diff(self.global_queue_interval, next) > 2 { + self.global_queue_interval = next; + } + } +} + +impl Worker { + /// Returns a reference to the scheduler's injection queue. + fn inject(&self) -> &inject::Shared> { + &self.handle.shared.inject + } +} + +// TODO: Move `Handle` impls into handle.rs +impl task::Schedule for Arc { + fn release(&self, task: &Task) -> Option { + self.shared.owned.remove(task) + } + + fn schedule(&self, task: Notified) { + self.schedule_task(task, false); + } + + fn yield_now(&self, task: Notified) { + self.schedule_task(task, true); + } +} + +impl Handle { + pub(super) fn schedule_task(&self, task: Notified, is_yield: bool) { + with_current(|maybe_cx| { + if let Some(cx) = maybe_cx { + // Make sure the task is part of the **current** scheduler. + if self.ptr_eq(&cx.worker.handle) { + // And the current thread still holds a core + if let Some(core) = cx.core.borrow_mut().as_mut() { + self.schedule_local(core, task, is_yield); + return; + } + } + } + + // Otherwise, use the inject queue. + self.push_remote_task(task); + self.notify_parked_remote(); + }) + } + + fn schedule_local(&self, core: &mut Core, task: Notified, is_yield: bool) { + core.stats.inc_local_schedule_count(); + + // Spawning from the worker thread. If scheduling a "yield" then the + // task must always be pushed to the back of the queue, enabling other + // tasks to be executed. If **not** a yield, then there is more + // flexibility and the task may go to the front of the queue. + let should_notify = if is_yield || !core.lifo_enabled { + core.run_queue + .push_back_or_overflow(task, self, &mut core.stats); + true + } else { + // Push to the LIFO slot + let prev = core.lifo_slot.take(); + let ret = prev.is_some(); + + if let Some(prev) = prev { + core.run_queue + .push_back_or_overflow(prev, self, &mut core.stats); + } + + core.lifo_slot = Some(task); + + ret + }; + + // Only notify if not currently parked. If `park` is `None`, then the + // scheduling is from a resource driver. As notifications often come in + // batches, the notification is delayed until the park is complete. + if should_notify && core.park.is_some() { + self.notify_parked_local(); + } + } + + fn next_remote_task(&self) -> Option { + if self.shared.inject.is_empty() { + return None; + } + + let mut synced = self.shared.synced.lock(); + // safety: passing in correct `idle::Synced` + unsafe { self.shared.inject.pop(&mut synced.inject) } + } + + fn push_remote_task(&self, task: Notified) { + self.shared.scheduler_metrics.inc_remote_schedule_count(); + + let mut synced = self.shared.synced.lock(); + // safety: passing in correct `idle::Synced` + unsafe { + self.shared.inject.push(&mut synced.inject, task); + } + } + + pub(super) fn close(&self) { + if self + .shared + .inject + .close(&mut self.shared.synced.lock().inject) + { + self.notify_all(); + } + } + + fn notify_parked_local(&self) { + super::counters::inc_num_inc_notify_local(); + + if let Some(index) = self.shared.idle.worker_to_notify(&self.shared) { + super::counters::inc_num_unparks_local(); + self.shared.remotes[index].unpark.unpark(&self.driver); + } + } + + fn notify_parked_remote(&self) { + if let Some(index) = self.shared.idle.worker_to_notify(&self.shared) { + self.shared.remotes[index].unpark.unpark(&self.driver); + } + } + + pub(super) fn notify_all(&self) { + for remote in &self.shared.remotes[..] { + remote.unpark.unpark(&self.driver); + } + } + + fn notify_if_work_pending(&self) { + for remote in &self.shared.remotes[..] { + if !remote.steal.is_empty() { + self.notify_parked_local(); + return; + } + } + + if !self.shared.inject.is_empty() { + self.notify_parked_local(); + } + } + + fn transition_worker_from_searching(&self) { + if self.shared.idle.transition_worker_from_searching() { + // We are the final searching worker. Because work was found, we + // need to notify another worker. + self.notify_parked_local(); + } + } + + /// Signals that a worker has observed the shutdown signal and has replaced + /// its core back into its handle. + /// + /// If all workers have reached this point, the final cleanup is performed. + fn shutdown_core(&self, core: Box) { + let mut cores = self.shared.shutdown_cores.lock(); + cores.push(core); + + if cores.len() != self.shared.remotes.len() { + return; + } + + debug_assert!(self.shared.owned.is_empty()); + + for mut core in cores.drain(..) { + core.shutdown(self); + } + + // Drain the injection queue + // + // We already shut down every task, so we can simply drop the tasks. + while let Some(task) = self.next_remote_task() { + drop(task); + } + } + + fn ptr_eq(&self, other: &Handle) -> bool { + std::ptr::eq(self, other) + } +} + +impl Overflow> for Handle { + fn push(&self, task: task::Notified>) { + self.push_remote_task(task); + } + + fn push_batch(&self, iter: I) + where + I: Iterator>>, + { + unsafe { + self.shared.inject.push_batch(self, iter); + } + } +} + +pub(crate) struct InjectGuard<'a> { + lock: crate::loom::sync::MutexGuard<'a, Synced>, +} + +impl<'a> AsMut for InjectGuard<'a> { + fn as_mut(&mut self) -> &mut inject::Synced { + &mut self.lock.inject + } +} + +impl<'a> Lock for &'a Handle { + type Handle = InjectGuard<'a>; + + fn lock(self) -> Self::Handle { + InjectGuard { + lock: self.shared.synced.lock(), + } + } +} + +#[track_caller] +fn with_current(f: impl FnOnce(Option<&Context>) -> R) -> R { + use scheduler::Context::MultiThread; + + context::with_scheduler(|ctx| match ctx { + Some(MultiThread(ctx)) => f(Some(ctx)), + _ => f(None), + }) +} + +// `u32::abs_diff` is not available on Tokio's MSRV. +fn abs_diff(a: u32, b: u32) -> u32 { + if a > b { + a - b + } else { + b - a + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/metrics.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/metrics.rs new file mode 100644 index 000000000000..a9a5ab3ed600 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/metrics.rs @@ -0,0 +1,11 @@ +use super::Shared; + +impl Shared { + pub(crate) fn injection_queue_depth(&self) -> usize { + self.inject.len() + } + + pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize { + self.remotes[worker].steal.len() + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs new file mode 100644 index 000000000000..d310d9f6d350 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs @@ -0,0 +1,79 @@ +use super::{Core, Handle, Shared}; + +use crate::loom::sync::Arc; +use crate::runtime::scheduler::multi_thread::Stats; +use crate::runtime::task::trace::trace_multi_thread; +use crate::runtime::{dump, WorkerMetrics}; + +use std::time::Duration; + +impl Handle { + pub(super) fn trace_core(&self, mut core: Box) -> Box { + core.is_traced = false; + + if core.is_shutdown { + return core; + } + + // wait for other workers, or timeout without tracing + let timeout = Duration::from_millis(250); // a _very_ generous timeout + let barrier = + if let Some(barrier) = self.shared.trace_status.trace_start.wait_timeout(timeout) { + barrier + } else { + // don't attempt to trace + return core; + }; + + if !barrier.is_leader() { + // wait for leader to finish tracing + self.shared.trace_status.trace_end.wait(); + return core; + } + + // trace + + let owned = &self.shared.owned; + let mut local = self.shared.steal_all(); + let synced = &self.shared.synced; + let injection = &self.shared.inject; + + // safety: `trace_multi_thread` is invoked with the same `synced` that `injection` + // was created with. + let traces = unsafe { trace_multi_thread(owned, &mut local, synced, injection) } + .into_iter() + .map(dump::Task::new) + .collect(); + + let result = dump::Dump::new(traces); + + // stash the result + self.shared.trace_status.stash_result(result); + + // allow other workers to proceed + self.shared.trace_status.trace_end.wait(); + + core + } +} + +impl Shared { + /// Steal all tasks from remotes into a single local queue. + pub(super) fn steal_all(&self) -> super::queue::Local> { + let (_steal, mut local) = super::queue::local(); + + let worker_metrics = WorkerMetrics::new(); + let mut stats = Stats::new(&worker_metrics); + + for remote in self.remotes.iter() { + let steal = &remote.steal; + while !steal.is_empty() { + if let Some(task) = steal.steal_into(&mut local, &mut stats) { + local.push_back([task].into_iter()); + } + } + } + + local + } +} diff --git a/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump_mock.rs b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump_mock.rs new file mode 100644 index 000000000000..24c5600ce2d8 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/scheduler/multi_thread/worker/taskdump_mock.rs @@ -0,0 +1,7 @@ +use super::{Core, Handle}; + +impl Handle { + pub(super) fn trace_core(&self, core: Box) -> Box { + core + } +} diff --git a/third_party/rust/tokio/src/runtime/signal/mod.rs b/third_party/rust/tokio/src/runtime/signal/mod.rs new file mode 100644 index 000000000000..24f2f4c6cb56 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/signal/mod.rs @@ -0,0 +1,142 @@ +#![cfg_attr(not(feature = "rt"), allow(dead_code))] + +//! Signal driver + +use crate::runtime::{driver, io}; +use crate::signal::registry::globals; + +use mio::net::UnixStream; +use std::io::{self as std_io, Read}; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +/// Responsible for registering wakeups when an OS signal is received, and +/// subsequently dispatching notifications to any signal listeners as appropriate. +/// +/// Note: this driver relies on having an enabled IO driver in order to listen to +/// pipe write wakeups. +#[derive(Debug)] +pub(crate) struct Driver { + /// Thread parker. The `Driver` park implementation delegates to this. + io: io::Driver, + + /// A pipe for receiving wake events from the signal handler + receiver: UnixStream, + + /// Shared state. The driver keeps a strong ref and the handle keeps a weak + /// ref. The weak ref is used to check if the driver is still active before + /// trying to register a signal handler. + inner: Arc<()>, +} + +#[derive(Debug, Default)] +pub(crate) struct Handle { + /// Paired w/ the `Arc` above and is used to check if the driver is still + /// around before attempting to register a signal handler. + inner: Weak<()>, +} + +// ===== impl Driver ===== + +impl Driver { + /// Creates a new signal `Driver` instance that delegates wakeups to `park`. + pub(crate) fn new(io: io::Driver, io_handle: &io::Handle) -> std_io::Result { + use std::mem::ManuallyDrop; + use std::os::unix::io::{AsRawFd, FromRawFd}; + + // NB: We give each driver a "fresh" receiver file descriptor to avoid + // the issues described in alexcrichton/tokio-process#42. + // + // In the past we would reuse the actual receiver file descriptor and + // swallow any errors around double registration of the same descriptor. + // I'm not sure if the second (failed) registration simply doesn't end + // up receiving wake up notifications, or there could be some race + // condition when consuming readiness events, but having distinct + // descriptors appears to mitigate this. + // + // Unfortunately we cannot just use a single global UnixStream instance + // either, since we can't assume they will always be registered with the + // exact same reactor. + // + // Mio 0.7 removed `try_clone()` as an API due to unexpected behavior + // with registering dups with the same reactor. In this case, duping is + // safe as each dup is registered with separate reactors **and** we + // only expect at least one dup to receive the notification. + + // Manually drop as we don't actually own this instance of UnixStream. + let receiver_fd = globals().receiver.as_raw_fd(); + + // safety: there is nothing unsafe about this, but the `from_raw_fd` fn is marked as unsafe. + let original = + ManuallyDrop::new(unsafe { std::os::unix::net::UnixStream::from_raw_fd(receiver_fd) }); + let mut receiver = UnixStream::from_std(original.try_clone()?); + + io_handle.register_signal_receiver(&mut receiver)?; + + Ok(Self { + io, + receiver, + inner: Arc::new(()), + }) + } + + /// Returns a handle to this event loop which can be sent across threads + /// and can be used as a proxy to the event loop itself. + pub(crate) fn handle(&self) -> Handle { + Handle { + inner: Arc::downgrade(&self.inner), + } + } + + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.io.park(handle); + self.process(); + } + + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + self.io.park_timeout(handle, duration); + self.process(); + } + + pub(crate) fn shutdown(&mut self, handle: &driver::Handle) { + self.io.shutdown(handle) + } + + fn process(&mut self) { + // If the signal pipe has not received a readiness event, then there is + // nothing else to do. + if !self.io.consume_signal_ready() { + return; + } + + // Drain the pipe completely so we can receive a new readiness event + // if another signal has come in. + let mut buf = [0; 128]; + loop { + match self.receiver.read(&mut buf) { + Ok(0) => panic!("EOF on self-pipe"), + Ok(_) => continue, // Keep reading + Err(e) if e.kind() == std_io::ErrorKind::WouldBlock => break, + Err(e) => panic!("Bad read on self-pipe: {}", e), + } + } + + // Broadcast any signals which were received + globals().broadcast(); + } +} + +// ===== impl Handle ===== + +impl Handle { + pub(crate) fn check_inner(&self) -> std_io::Result<()> { + if self.inner.strong_count() > 0 { + Ok(()) + } else { + Err(std_io::Error::new( + std_io::ErrorKind::Other, + "signal driver gone", + )) + } + } +} diff --git a/third_party/rust/tokio/src/runtime/spawner.rs b/third_party/rust/tokio/src/runtime/spawner.rs deleted file mode 100644 index d81a806cb59a..000000000000 --- a/third_party/rust/tokio/src/runtime/spawner.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::future::Future; -use crate::runtime::basic_scheduler; -use crate::task::JoinHandle; - -cfg_rt_multi_thread! { - use crate::runtime::thread_pool; -} - -#[derive(Debug, Clone)] -pub(crate) enum Spawner { - Basic(basic_scheduler::Spawner), - #[cfg(feature = "rt-multi-thread")] - ThreadPool(thread_pool::Spawner), -} - -impl Spawner { - pub(crate) fn shutdown(&mut self) { - #[cfg(feature = "rt-multi-thread")] - { - if let Spawner::ThreadPool(spawner) = self { - spawner.shutdown(); - } - } - } - - pub(crate) fn spawn(&self, future: F) -> JoinHandle - where - F: Future + Send + 'static, - F::Output: Send + 'static, - { - match self { - Spawner::Basic(spawner) => spawner.spawn(future), - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.spawn(future), - } - } -} - -cfg_metrics! { - use crate::runtime::{SchedulerMetrics, WorkerMetrics}; - - impl Spawner { - pub(crate) fn num_workers(&self) -> usize { - match self { - Spawner::Basic(_) => 1, - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.num_workers(), - } - } - - pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { - match self { - Spawner::Basic(spawner) => spawner.scheduler_metrics(), - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.scheduler_metrics(), - } - } - - pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { - match self { - Spawner::Basic(spawner) => spawner.worker_metrics(worker), - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.worker_metrics(worker), - } - } - - pub(crate) fn injection_queue_depth(&self) -> usize { - match self { - Spawner::Basic(spawner) => spawner.injection_queue_depth(), - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.injection_queue_depth(), - } - } - - pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize { - match self { - Spawner::Basic(spawner) => spawner.worker_metrics(worker).queue_depth(), - #[cfg(feature = "rt-multi-thread")] - Spawner::ThreadPool(spawner) => spawner.worker_local_queue_depth(worker), - } - } - } -} diff --git a/third_party/rust/tokio/src/runtime/task/abort.rs b/third_party/rust/tokio/src/runtime/task/abort.rs new file mode 100644 index 000000000000..6edca1004043 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/task/abort.rs @@ -0,0 +1,87 @@ +use crate::runtime::task::{Header, RawTask}; +use std::fmt; +use std::panic::{RefUnwindSafe, UnwindSafe}; + +/// An owned permission to abort a spawned task, without awaiting its completion. +/// +/// Unlike a [`JoinHandle`], an `AbortHandle` does *not* represent the +/// permission to await the task's completion, only to terminate it. +/// +/// The task may be aborted by calling the [`AbortHandle::abort`] method. +/// Dropping an `AbortHandle` releases the permission to terminate the task +/// --- it does *not* abort the task. +/// +/// [`JoinHandle`]: crate::task::JoinHandle +#[cfg_attr(docsrs, doc(cfg(feature = "rt")))] +pub struct AbortHandle { + raw: RawTask, +} + +impl AbortHandle { + pub(super) fn new(raw: RawTask) -> Self { + Self { raw } + } + + /// Abort the task associated with the handle. + /// + /// Awaiting a cancelled task might complete as usual if the task was + /// already completed at the time it was cancelled, but most likely it + /// will fail with a [cancelled] `JoinError`. + /// + /// If the task was already cancelled, such as by [`JoinHandle::abort`], + /// this method will do nothing. + /// + /// [cancelled]: method@super::error::JoinError::is_cancelled + /// [`JoinHandle::abort`]: method@super::JoinHandle::abort + pub fn abort(&self) { + self.raw.remote_abort(); + } + + /// Checks if the task associated with this `AbortHandle` has finished. + /// + /// Please note that this method can return `false` even if `abort` has been + /// called on the task. This is because the cancellation process may take + /// some time, and this method does not return `true` until it has + /// completed. + pub fn is_finished(&self) -> bool { + let state = self.raw.state().load(); + state.is_complete() + } + + /// Returns a [task ID] that uniquely identifies this task relative to other + /// currently spawned tasks. + /// + /// **Note**: This is an [unstable API][unstable]. The public API of this type + /// may break in 1.x releases. See [the documentation on unstable + /// features][unstable] for details. + /// + /// [task ID]: crate::task::Id + /// [unstable]: crate#unstable-features + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn id(&self) -> super::Id { + // Safety: The header pointer is valid. + unsafe { Header::get_id(self.raw.header_ptr()) } + } +} + +unsafe impl Send for AbortHandle {} +unsafe impl Sync for AbortHandle {} + +impl UnwindSafe for AbortHandle {} +impl RefUnwindSafe for AbortHandle {} + +impl fmt::Debug for AbortHandle { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + // Safety: The header pointer is valid. + let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) }; + let id = unsafe { id_ptr.as_ref() }; + fmt.debug_struct("AbortHandle").field("id", id).finish() + } +} + +impl Drop for AbortHandle { + fn drop(&mut self) { + self.raw.drop_abort_handle(); + } +} diff --git a/third_party/rust/tokio/src/runtime/task/core.rs b/third_party/rust/tokio/src/runtime/task/core.rs index 776e8341f37c..110933e58f0e 100644 --- a/third_party/rust/tokio/src/runtime/task/core.rs +++ b/third_party/rust/tokio/src/runtime/task/core.rs @@ -11,9 +11,10 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; +use crate::runtime::context; use crate::runtime::task::raw::{self, Vtable}; use crate::runtime::task::state::State; -use crate::runtime::task::Schedule; +use crate::runtime::task::{Id, Schedule}; use crate::util::linked_list; use std::pin::Pin; @@ -24,6 +25,97 @@ use std::task::{Context, Poll, Waker}; /// /// It is critical for `Header` to be the first field as the task structure will /// be referenced by both *mut Cell and *mut Header. +/// +/// Any changes to the layout of this struct _must_ also be reflected in the +/// const fns in raw.rs. +/// +// # This struct should be cache padded to avoid false sharing. The cache padding rules are copied +// from crossbeam-utils/src/cache_padded.rs +// +// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache +// lines at a time, so we have to align to 128 bytes rather than 64. +// +// Sources: +// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf +// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107 +// +// ARM's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size. +// +// Sources: +// - https://www.mono-project.com/news/2016/09/12/arm64-icache/ +// +// powerpc64 has 128-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9 +#[cfg_attr( + any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "powerpc64", + ), + repr(align(128)) +)] +// arm, mips, mips64, riscv64, sparc, and hexagon have 32-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_riscv64.go#L7 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L17 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/hexagon/include/asm/cache.h#L12 +// +// riscv32 is assumed not to exceed the cache line size of riscv64. +#[cfg_attr( + any( + target_arch = "arm", + target_arch = "mips", + target_arch = "mips64", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "sparc", + target_arch = "hexagon", + ), + repr(align(32)) +)] +// m68k has 16-byte cache line size. +// +// Sources: +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/m68k/include/asm/cache.h#L9 +#[cfg_attr(target_arch = "m68k", repr(align(16)))] +// s390x has 256-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/s390/include/asm/cache.h#L13 +#[cfg_attr(target_arch = "s390x", repr(align(256)))] +// x86, wasm, and sparc64 have 64-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L19 +// +// All others are assumed to have 64-byte cache line size. +#[cfg_attr( + not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "arm", + target_arch = "mips", + target_arch = "mips64", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "sparc", + target_arch = "hexagon", + target_arch = "m68k", + target_arch = "s390x", + )), + repr(align(64)) +)] #[repr(C)] pub(super) struct Cell { /// Hot task state data @@ -43,10 +135,17 @@ pub(super) struct CoreStage { /// The core of the task. /// /// Holds the future or output, depending on the stage of execution. +/// +/// Any changes to the layout of this struct _must_ also be reflected in the +/// const fns in raw.rs. +#[repr(C)] pub(super) struct Core { /// Scheduler used to drive this future. pub(super) scheduler: S, + /// The task's ID, used for populating `JoinError`s. + pub(super) task_id: Id, + /// Either the future or the output. pub(super) stage: CoreStage, } @@ -57,8 +156,6 @@ pub(crate) struct Header { /// Task state. pub(super) state: State, - pub(super) owned: UnsafeCell>, - /// Pointer to next task, used with the injection queue. pub(super) queue_next: UnsafeCell>>, @@ -80,18 +177,29 @@ pub(crate) struct Header { /// The tracing ID for this instrumented task. #[cfg(all(tokio_unstable, feature = "tracing"))] - pub(super) id: Option, + pub(super) tracing_id: Option, } unsafe impl Send for Header {} unsafe impl Sync for Header {} -/// Cold data is stored after the future. +/// Cold data is stored after the future. Data is considered cold if it is only +/// used during creation or shutdown of the task. pub(super) struct Trailer { + /// Pointers for the linked list in the `OwnedTasks` that owns this task. + pub(super) owned: linked_list::Pointers

, /// Consumer task waiting on completion of this task. pub(super) waker: UnsafeCell>, } +generate_addr_of_methods! { + impl<> Trailer { + pub(super) unsafe fn addr_of_owned(self: NonNull) -> NonNull> { + &self.owned + } + } +} + /// Either the future or the output. pub(super) enum Stage { Running(T), @@ -102,29 +210,48 @@ pub(super) enum Stage { impl Cell { /// Allocates a new task cell, containing the header, trailer, and core /// structures. - pub(super) fn new(future: T, scheduler: S, state: State) -> Box> { + pub(super) fn new(future: T, scheduler: S, state: State, task_id: Id) -> Box> { #[cfg(all(tokio_unstable, feature = "tracing"))] - let id = future.id(); - Box::new(Cell { + let tracing_id = future.id(); + let result = Box::new(Cell { header: Header { state, - owned: UnsafeCell::new(linked_list::Pointers::new()), queue_next: UnsafeCell::new(None), vtable: raw::vtable::(), owner_id: UnsafeCell::new(0), #[cfg(all(tokio_unstable, feature = "tracing"))] - id, + tracing_id, }, core: Core { scheduler, stage: CoreStage { stage: UnsafeCell::new(Stage::Running(future)), }, + task_id, }, trailer: Trailer { waker: UnsafeCell::new(None), + owned: linked_list::Pointers::new(), }, - }) + }); + + #[cfg(debug_assertions)] + { + let trailer_addr = (&result.trailer) as *const Trailer as usize; + let trailer_ptr = unsafe { Header::get_trailer(NonNull::from(&result.header)) }; + assert_eq!(trailer_addr, trailer_ptr.as_ptr() as usize); + + let scheduler_addr = (&result.core.scheduler) as *const S as usize; + let scheduler_ptr = + unsafe { Header::get_scheduler::(NonNull::from(&result.header)) }; + assert_eq!(scheduler_addr, scheduler_ptr.as_ptr() as usize); + + let id_addr = (&result.core.task_id) as *const Id as usize; + let id_ptr = unsafe { Header::get_id_ptr(NonNull::from(&result.header)) }; + assert_eq!(id_addr, id_ptr.as_ptr() as usize); + } + + result } } @@ -132,7 +259,29 @@ impl CoreStage { pub(super) fn with_mut(&self, f: impl FnOnce(*mut Stage) -> R) -> R { self.stage.with_mut(f) } +} +/// Set and clear the task id in the context when the future is executed or +/// dropped, or when the output produced by the future is dropped. +pub(crate) struct TaskIdGuard { + parent_task_id: Option, +} + +impl TaskIdGuard { + fn enter(id: Id) -> Self { + TaskIdGuard { + parent_task_id: context::set_current_task_id(Some(id)), + } + } +} + +impl Drop for TaskIdGuard { + fn drop(&mut self) { + context::set_current_task_id(self.parent_task_id); + } +} + +impl Core { /// Polls the future. /// /// # Safety @@ -148,7 +297,7 @@ impl CoreStage { /// heap. pub(super) fn poll(&self, mut cx: Context<'_>) -> Poll { let res = { - self.stage.with_mut(|ptr| { + self.stage.stage.with_mut(|ptr| { // Safety: The caller ensures mutual exclusion to the field. let future = match unsafe { &mut *ptr } { Stage::Running(future) => future, @@ -158,6 +307,7 @@ impl CoreStage { // Safety: The caller ensures the future is pinned. let future = unsafe { Pin::new_unchecked(future) }; + let _guard = TaskIdGuard::enter(self.task_id); future.poll(&mut cx) }) }; @@ -201,7 +351,7 @@ impl CoreStage { pub(super) fn take_output(&self) -> super::Result { use std::mem; - self.stage.with_mut(|ptr| { + self.stage.stage.with_mut(|ptr| { // Safety:: the caller ensures mutual exclusion to the field. match mem::replace(unsafe { &mut *ptr }, Stage::Consumed) { Stage::Finished(output) => output, @@ -211,19 +361,16 @@ impl CoreStage { } unsafe fn set_stage(&self, stage: Stage) { - self.stage.with_mut(|ptr| *ptr = stage) - } -} - -cfg_rt_multi_thread! { - impl Header { - pub(super) unsafe fn set_next(&self, next: Option>) { - self.queue_next.with_mut(|ptr| *ptr = next); - } + let _guard = TaskIdGuard::enter(self.task_id); + self.stage.stage.with_mut(|ptr| *ptr = stage) } } impl Header { + pub(super) unsafe fn set_next(&self, next: Option>) { + self.queue_next.with_mut(|ptr| *ptr = next); + } + // safety: The caller must guarantee exclusive access to this field, and // must ensure that the id is either 0 or the id of the OwnedTasks // containing this task. @@ -236,6 +383,62 @@ impl Header { // the safety requirements on `set_owner_id`. unsafe { self.owner_id.with(|ptr| *ptr) } } + + /// Gets a pointer to the `Trailer` of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + pub(super) unsafe fn get_trailer(me: NonNull
) -> NonNull { + let offset = me.as_ref().vtable.trailer_offset; + let trailer = me.as_ptr().cast::().add(offset).cast::(); + NonNull::new_unchecked(trailer) + } + + /// Gets a pointer to the scheduler of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + /// + /// The generic type S must be set to the correct scheduler type for this + /// task. + pub(super) unsafe fn get_scheduler(me: NonNull
) -> NonNull { + let offset = me.as_ref().vtable.scheduler_offset; + let scheduler = me.as_ptr().cast::().add(offset).cast::(); + NonNull::new_unchecked(scheduler) + } + + /// Gets a pointer to the id of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + pub(super) unsafe fn get_id_ptr(me: NonNull
) -> NonNull { + let offset = me.as_ref().vtable.id_offset; + let id = me.as_ptr().cast::().add(offset).cast::(); + NonNull::new_unchecked(id) + } + + /// Gets the id of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + pub(super) unsafe fn get_id(me: NonNull
) -> Id { + let ptr = Header::get_id_ptr(me).as_ptr(); + *ptr + } + + /// Gets the tracing id of the task containing this `Header`. + /// + /// # Safety + /// + /// The provided raw pointer must point at the header of a task. + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(super) unsafe fn get_tracing_id(me: &NonNull
) -> Option<&tracing::Id> { + me.as_ref().tracing_id.as_ref() + } } impl Trailer { diff --git a/third_party/rust/tokio/src/runtime/task/error.rs b/third_party/rust/tokio/src/runtime/task/error.rs index 1a8129b2b6f0..f7ead77b7cc4 100644 --- a/third_party/rust/tokio/src/runtime/task/error.rs +++ b/third_party/rust/tokio/src/runtime/task/error.rs @@ -2,12 +2,13 @@ use std::any::Any; use std::fmt; use std::io; +use super::Id; use crate::util::SyncWrapper; - cfg_rt! { /// Task failed to execute to completion. pub struct JoinError { repr: Repr, + id: Id, } } @@ -17,15 +18,17 @@ enum Repr { } impl JoinError { - pub(crate) fn cancelled() -> JoinError { + pub(crate) fn cancelled(id: Id) -> JoinError { JoinError { repr: Repr::Cancelled, + id, } } - pub(crate) fn panic(err: Box) -> JoinError { + pub(crate) fn panic(id: Id, err: Box) -> JoinError { JoinError { repr: Repr::Panic(SyncWrapper::new(err)), + id, } } @@ -79,6 +82,7 @@ impl JoinError { /// } /// } /// ``` + #[track_caller] pub fn into_panic(self) -> Box { self.try_into_panic() .expect("`JoinError` reason is not a panic.") @@ -111,13 +115,28 @@ impl JoinError { _ => Err(self), } } + + /// Returns a [task ID] that identifies the task which errored relative to + /// other currently spawned tasks. + /// + /// **Note**: This is an [unstable API][unstable]. The public API of this type + /// may break in 1.x releases. See [the documentation on unstable + /// features][unstable] for details. + /// + /// [task ID]: crate::task::Id + /// [unstable]: crate#unstable-features + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn id(&self) -> Id { + self.id + } } impl fmt::Display for JoinError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.repr { - Repr::Cancelled => write!(fmt, "cancelled"), - Repr::Panic(_) => write!(fmt, "panic"), + Repr::Cancelled => write!(fmt, "task {} was cancelled", self.id), + Repr::Panic(_) => write!(fmt, "task {} panicked", self.id), } } } @@ -125,8 +144,8 @@ impl fmt::Display for JoinError { impl fmt::Debug for JoinError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.repr { - Repr::Cancelled => write!(fmt, "JoinError::Cancelled"), - Repr::Panic(_) => write!(fmt, "JoinError::Panic(...)"), + Repr::Cancelled => write!(fmt, "JoinError::Cancelled({:?})", self.id), + Repr::Panic(_) => write!(fmt, "JoinError::Panic({:?}, ...)", self.id), } } } diff --git a/third_party/rust/tokio/src/runtime/task/harness.rs b/third_party/rust/tokio/src/runtime/task/harness.rs index 261dccea415e..8e3c3d14fa03 100644 --- a/third_party/rust/tokio/src/runtime/task/harness.rs +++ b/third_party/rust/tokio/src/runtime/task/harness.rs @@ -1,8 +1,8 @@ use crate::future::Future; -use crate::runtime::task::core::{Cell, Core, CoreStage, Header, Trailer}; -use crate::runtime::task::state::Snapshot; +use crate::runtime::task::core::{Cell, Core, Header, Trailer}; +use crate::runtime::task::state::{Snapshot, State}; use crate::runtime::task::waker::waker_ref; -use crate::runtime::task::{JoinError, Notified, Schedule, Task}; +use crate::runtime::task::{JoinError, Notified, RawTask, Schedule, Task}; use std::mem; use std::mem::ManuallyDrop; @@ -31,7 +31,11 @@ where } fn header(&self) -> &Header { - unsafe { &self.cell.as_ref().header } + unsafe { &*self.header_ptr().as_ptr() } + } + + fn state(&self) -> &State { + &self.header().state } fn trailer(&self) -> &Trailer { @@ -43,11 +47,102 @@ where } } +/// Task operations that can be implemented without being generic over the +/// scheduler or task. Only one version of these methods should exist in the +/// final binary. +impl RawTask { + pub(super) fn drop_reference(self) { + if self.state().ref_dec() { + self.dealloc(); + } + } + + /// This call consumes a ref-count and notifies the task. This will create a + /// new Notified and submit it if necessary. + /// + /// The caller does not need to hold a ref-count besides the one that was + /// passed to this call. + pub(super) fn wake_by_val(&self) { + use super::state::TransitionToNotifiedByVal; + + match self.state().transition_to_notified_by_val() { + TransitionToNotifiedByVal::Submit => { + // The caller has given us a ref-count, and the transition has + // created a new ref-count, so we now hold two. We turn the new + // ref-count Notified and pass it to the call to `schedule`. + // + // The old ref-count is retained for now to ensure that the task + // is not dropped during the call to `schedule` if the call + // drops the task it was given. + self.schedule(); + + // Now that we have completed the call to schedule, we can + // release our ref-count. + self.drop_reference(); + } + TransitionToNotifiedByVal::Dealloc => { + self.dealloc(); + } + TransitionToNotifiedByVal::DoNothing => {} + } + } + + /// This call notifies the task. It will not consume any ref-counts, but the + /// caller should hold a ref-count. This will create a new Notified and + /// submit it if necessary. + pub(super) fn wake_by_ref(&self) { + use super::state::TransitionToNotifiedByRef; + + match self.state().transition_to_notified_by_ref() { + TransitionToNotifiedByRef::Submit => { + // The transition above incremented the ref-count for a new task + // and the caller also holds a ref-count. The caller's ref-count + // ensures that the task is not destroyed even if the new task + // is dropped before `schedule` returns. + self.schedule(); + } + TransitionToNotifiedByRef::DoNothing => {} + } + } + + /// Remotely aborts the task. + /// + /// The caller should hold a ref-count, but we do not consume it. + /// + /// This is similar to `shutdown` except that it asks the runtime to perform + /// the shutdown. This is necessary to avoid the shutdown happening in the + /// wrong thread for non-Send tasks. + pub(super) fn remote_abort(&self) { + if self.state().transition_to_notified_and_cancel() { + // The transition has created a new ref-count, which we turn into + // a Notified and pass to the task. + // + // Since the caller holds a ref-count, the task cannot be destroyed + // before the call to `schedule` returns even if the call drops the + // `Notified` internally. + self.schedule(); + } + } + + /// Try to set the waker notified when the task is complete. Returns true if + /// the task has already completed. If this call returns false, then the + /// waker will not be notified. + pub(super) fn try_set_join_waker(&self, waker: &Waker) -> bool { + can_read_output(self.header(), self.trailer(), waker) + } +} + impl Harness where T: Future, S: Schedule, { + pub(super) fn drop_reference(self) { + if self.state().ref_dec() { + self.dealloc(); + } + } + /// Polls the inner future. A ref-count is consumed. /// /// All necessary state checks and transitions are performed. @@ -95,33 +190,32 @@ where fn poll_inner(&self) -> PollFuture { use super::state::{TransitionToIdle, TransitionToRunning}; - match self.header().state.transition_to_running() { + match self.state().transition_to_running() { TransitionToRunning::Success => { let header_ptr = self.header_ptr(); let waker_ref = waker_ref::(&header_ptr); - let cx = Context::from_waker(&*waker_ref); - let res = poll_future(&self.core().stage, cx); + let cx = Context::from_waker(&waker_ref); + let res = poll_future(self.core(), cx); if res == Poll::Ready(()) { // The future completed. Move on to complete the task. return PollFuture::Complete; } - match self.header().state.transition_to_idle() { + match self.state().transition_to_idle() { TransitionToIdle::Ok => PollFuture::Done, TransitionToIdle::OkNotified => PollFuture::Notified, TransitionToIdle::OkDealloc => PollFuture::Dealloc, TransitionToIdle::Cancelled => { // The transition to idle failed because the task was // cancelled during the poll. - - cancel_task(&self.core().stage); + cancel_task(self.core()); PollFuture::Complete } } } TransitionToRunning::Cancelled => { - cancel_task(&self.core().stage); + cancel_task(self.core()); PollFuture::Complete } TransitionToRunning::Failed => PollFuture::Done, @@ -136,7 +230,7 @@ where /// there is nothing further to do. When the task completes running, it will /// notice the `CANCELLED` bit and finalize the task. pub(super) fn shutdown(self) { - if !self.header().state.transition_to_shutdown() { + if !self.state().transition_to_shutdown() { // The task is concurrently running. No further work needed. self.drop_reference(); return; @@ -144,7 +238,7 @@ where // By transitioning the lifecycle to `Running`, we have permission to // drop the future. - cancel_task(&self.core().stage); + cancel_task(self.core()); self.complete(); } @@ -155,6 +249,19 @@ where // Check causality self.core().stage.with_mut(drop); + // Safety: The caller of this method just transitioned our ref-count to + // zero, so it is our responsibility to release the allocation. + // + // We don't hold any references into the allocation at this point, but + // it is possible for another thread to still hold a `&State` into the + // allocation if that other thread has decremented its last ref-count, + // but has not yet returned from the relevant method on `State`. + // + // However, the `State` type consists of just an `AtomicUsize`, and an + // `AtomicUsize` wraps the entirety of its contents in an `UnsafeCell`. + // As explained in the documentation for `UnsafeCell`, such references + // are allowed to be dangling after their last use, even if the + // reference has not yet gone out of scope. unsafe { drop(Box::from_raw(self.cell.as_ptr())); } @@ -165,21 +272,14 @@ where /// Read the task output into `dst`. pub(super) fn try_read_output(self, dst: &mut Poll>, waker: &Waker) { if can_read_output(self.header(), self.trailer(), waker) { - *dst = Poll::Ready(self.core().stage.take_output()); + *dst = Poll::Ready(self.core().take_output()); } } - /// Try to set the waker notified when the task is complete. Returns true if - /// the task has already completed. If this call returns false, then the - /// waker will not be notified. - pub(super) fn try_set_join_waker(self, waker: &Waker) -> bool { - can_read_output(self.header(), self.trailer(), waker) - } - pub(super) fn drop_join_handle_slow(self) { // Try to unset `JOIN_INTEREST`. This must be done as a first step in // case the task concurrently completed. - if self.header().state.unset_join_interested().is_err() { + if self.state().unset_join_interested().is_err() { // It is our responsibility to drop the output. This is critical as // the task output may not be `Send` and as such must remain with // the scheduler or `JoinHandle`. i.e. if the output remains in the @@ -190,7 +290,7 @@ where // they are dropping the `JoinHandle`, we assume they are not // interested in the panic and swallow it. let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| { - self.core().stage.drop_future_or_output(); + self.core().drop_future_or_output(); })); } @@ -198,92 +298,6 @@ where self.drop_reference(); } - /// Remotely aborts the task. - /// - /// The caller should hold a ref-count, but we do not consume it. - /// - /// This is similar to `shutdown` except that it asks the runtime to perform - /// the shutdown. This is necessary to avoid the shutdown happening in the - /// wrong thread for non-Send tasks. - pub(super) fn remote_abort(self) { - if self.header().state.transition_to_notified_and_cancel() { - // The transition has created a new ref-count, which we turn into - // a Notified and pass to the task. - // - // Since the caller holds a ref-count, the task cannot be destroyed - // before the call to `schedule` returns even if the call drops the - // `Notified` internally. - self.core() - .scheduler - .schedule(Notified(self.get_new_task())); - } - } - - // ===== waker behavior ===== - - /// This call consumes a ref-count and notifies the task. This will create a - /// new Notified and submit it if necessary. - /// - /// The caller does not need to hold a ref-count besides the one that was - /// passed to this call. - pub(super) fn wake_by_val(self) { - use super::state::TransitionToNotifiedByVal; - - match self.header().state.transition_to_notified_by_val() { - TransitionToNotifiedByVal::Submit => { - // The caller has given us a ref-count, and the transition has - // created a new ref-count, so we now hold two. We turn the new - // ref-count Notified and pass it to the call to `schedule`. - // - // The old ref-count is retained for now to ensure that the task - // is not dropped during the call to `schedule` if the call - // drops the task it was given. - self.core() - .scheduler - .schedule(Notified(self.get_new_task())); - - // Now that we have completed the call to schedule, we can - // release our ref-count. - self.drop_reference(); - } - TransitionToNotifiedByVal::Dealloc => { - self.dealloc(); - } - TransitionToNotifiedByVal::DoNothing => {} - } - } - - /// This call notifies the task. It will not consume any ref-counts, but the - /// caller should hold a ref-count. This will create a new Notified and - /// submit it if necessary. - pub(super) fn wake_by_ref(&self) { - use super::state::TransitionToNotifiedByRef; - - match self.header().state.transition_to_notified_by_ref() { - TransitionToNotifiedByRef::Submit => { - // The transition above incremented the ref-count for a new task - // and the caller also holds a ref-count. The caller's ref-count - // ensures that the task is not destroyed even if the new task - // is dropped before `schedule` returns. - self.core() - .scheduler - .schedule(Notified(self.get_new_task())); - } - TransitionToNotifiedByRef::DoNothing => {} - } - } - - pub(super) fn drop_reference(self) { - if self.header().state.ref_dec() { - self.dealloc(); - } - } - - #[cfg(all(tokio_unstable, feature = "tracing"))] - pub(super) fn id(&self) -> Option<&tracing::Id> { - self.header().id.as_ref() - } - // ====== internal ====== /// Completes the task. This method assumes that the state is RUNNING. @@ -291,7 +305,7 @@ where // The future has completed and its output has been written to the task // stage. We transition from running to complete. - let snapshot = self.header().state.transition_to_complete(); + let snapshot = self.state().transition_to_complete(); // We catch panics here in case dropping the future or waking the // JoinHandle panics. @@ -300,10 +314,11 @@ where // The `JoinHandle` is not interested in the output of // this task. It is our responsibility to drop the // output. - self.core().stage.drop_future_or_output(); - } else if snapshot.has_join_waker() { - // Notify the join handle. The previous transition obtains the - // lock on the waker cell. + self.core().drop_future_or_output(); + } else if snapshot.is_join_waker_set() { + // Notify the waker. Reading the waker field is safe per rule 4 + // in task/mod.rs, since the JOIN_WAKER bit is set and the call + // to transition_to_complete() above set the COMPLETE bit. self.trailer().wake_join(); } })); @@ -311,7 +326,7 @@ where // The task has completed execution and will no longer be scheduled. let num_release = self.release(); - if self.header().state.transition_to_terminal(num_release) { + if self.state().transition_to_terminal(num_release) { self.dealloc(); } } @@ -353,36 +368,30 @@ fn can_read_output(header: &Header, trailer: &Trailer, waker: &Waker) -> bool { debug_assert!(snapshot.is_join_interested()); if !snapshot.is_complete() { - // The waker must be stored in the task struct. - let res = if snapshot.has_join_waker() { - // There already is a waker stored in the struct. If it matches - // the provided waker, then there is no further work to do. - // Otherwise, the waker must be swapped. - let will_wake = unsafe { - // Safety: when `JOIN_INTEREST` is set, only `JOIN_HANDLE` - // may mutate the `waker` field. - trailer.will_wake(waker) - }; + // If the task is not complete, try storing the provided waker in the + // task's waker field. - if will_wake { - // The task is not complete **and** the waker is up to date, - // there is nothing further that needs to be done. + let res = if snapshot.is_join_waker_set() { + // If JOIN_WAKER is set, then JoinHandle has previously stored a + // waker in the waker field per step (iii) of rule 5 in task/mod.rs. + + // Optimization: if the stored waker and the provided waker wake the + // same task, then return without touching the waker field. (Reading + // the waker field below is safe per rule 3 in task/mod.rs.) + if unsafe { trailer.will_wake(waker) } { return false; } - // Unset the `JOIN_WAKER` to gain mutable access to the `waker` - // field then update the field with the new join worker. - // - // This requires two atomic operations, unsetting the bit and - // then resetting it. If the task transitions to complete - // concurrently to either one of those operations, then setting - // the join waker fails and we proceed to reading the task - // output. + // Otherwise swap the stored waker with the provided waker by + // following the rule 5 in task/mod.rs. header .state .unset_waker() .and_then(|snapshot| set_join_waker(header, trailer, waker.clone(), snapshot)) } else { + // If JOIN_WAKER is unset, then JoinHandle has mutable access to the + // waker field per rule 2 in task/mod.rs; therefore, skip step (i) + // of rule 5 and try to store the provided waker in the waker field. set_join_waker(header, trailer, waker.clone(), snapshot) }; @@ -403,7 +412,7 @@ fn set_join_waker( snapshot: Snapshot, ) -> Result { assert!(snapshot.is_join_interested()); - assert!(!snapshot.has_join_waker()); + assert!(!snapshot.is_join_waker_set()); // Safety: Only the `JoinHandle` may set the `waker` field. When // `JOIN_INTEREST` is **not** set, nothing else will touch the field. @@ -432,31 +441,31 @@ enum PollFuture { } /// Cancels the task and store the appropriate error in the stage field. -fn cancel_task(stage: &CoreStage) { +fn cancel_task(core: &Core) { // Drop the future from a panic guard. let res = panic::catch_unwind(panic::AssertUnwindSafe(|| { - stage.drop_future_or_output(); + core.drop_future_or_output(); })); match res { Ok(()) => { - stage.store_output(Err(JoinError::cancelled())); + core.store_output(Err(JoinError::cancelled(core.task_id))); } Err(panic) => { - stage.store_output(Err(JoinError::panic(panic))); + core.store_output(Err(JoinError::panic(core.task_id, panic))); } } } /// Polls the future. If the future completes, the output is written to the /// stage field. -fn poll_future(core: &CoreStage, cx: Context<'_>) -> Poll<()> { +fn poll_future(core: &Core, cx: Context<'_>) -> Poll<()> { // Poll the future. let output = panic::catch_unwind(panic::AssertUnwindSafe(|| { - struct Guard<'a, T: Future> { - core: &'a CoreStage, + struct Guard<'a, T: Future, S: Schedule> { + core: &'a Core, } - impl<'a, T: Future> Drop for Guard<'a, T> { + impl<'a, T: Future, S: Schedule> Drop for Guard<'a, T, S> { fn drop(&mut self) { // If the future panics on poll, we drop it inside the panic // guard. @@ -473,13 +482,20 @@ fn poll_future(core: &CoreStage, cx: Context<'_>) -> Poll<()> { let output = match output { Ok(Poll::Pending) => return Poll::Pending, Ok(Poll::Ready(output)) => Ok(output), - Err(panic) => Err(JoinError::panic(panic)), + Err(panic) => { + core.scheduler.unhandled_panic(); + Err(JoinError::panic(core.task_id, panic)) + } }; // Catch and ignore panics if the future panics on drop. - let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| { + let res = panic::catch_unwind(panic::AssertUnwindSafe(|| { core.store_output(output); })); + if res.is_err() { + core.scheduler.unhandled_panic(); + } + Poll::Ready(()) } diff --git a/third_party/rust/tokio/src/runtime/task/id.rs b/third_party/rust/tokio/src/runtime/task/id.rs new file mode 100644 index 000000000000..2b0d95c0243e --- /dev/null +++ b/third_party/rust/tokio/src/runtime/task/id.rs @@ -0,0 +1,87 @@ +use crate::runtime::context; + +use std::fmt; + +/// An opaque ID that uniquely identifies a task relative to all other currently +/// running tasks. +/// +/// # Notes +/// +/// - Task IDs are unique relative to other *currently running* tasks. When a +/// task completes, the same ID may be used for another task. +/// - Task IDs are *not* sequential, and do not indicate the order in which +/// tasks are spawned, what runtime a task is spawned on, or any other data. +/// - The task ID of the currently running task can be obtained from inside the +/// task via the [`task::try_id()`](crate::task::try_id()) and +/// [`task::id()`](crate::task::id()) functions and from outside the task via +/// the [`JoinHandle::id()`](crate::task::JoinHandle::id()) function. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [unstable]: crate#unstable-features +#[cfg_attr(docsrs, doc(cfg(all(feature = "rt", tokio_unstable))))] +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] +pub struct Id(u64); + +/// Returns the [`Id`] of the currently running task. +/// +/// # Panics +/// +/// This function panics if called from outside a task. Please note that calls +/// to `block_on` do not have task IDs, so the method will panic if called from +/// within a call to `block_on`. For a version of this function that doesn't +/// panic, see [`task::try_id()`](crate::runtime::task::try_id()). +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [task ID]: crate::task::Id +/// [unstable]: crate#unstable-features +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +#[track_caller] +pub fn id() -> Id { + context::current_task_id().expect("Can't get a task id when not inside a task") +} + +/// Returns the [`Id`] of the currently running task, or `None` if called outside +/// of a task. +/// +/// This function is similar to [`task::id()`](crate::runtime::task::id()), except +/// that it returns `None` rather than panicking if called outside of a task +/// context. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// [task ID]: crate::task::Id +/// [unstable]: crate#unstable-features +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +#[track_caller] +pub fn try_id() -> Option { + context::current_task_id() +} + +impl fmt::Display for Id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Id { + pub(crate) fn next() -> Self { + use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64}; + + static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1); + + Self(NEXT_ID.fetch_add(1, Relaxed)) + } + + pub(crate) fn as_u64(&self) -> u64 { + self.0 + } +} diff --git a/third_party/rust/tokio/src/runtime/task/inject.rs b/third_party/rust/tokio/src/runtime/task/inject.rs deleted file mode 100644 index 1585e13a019a..000000000000 --- a/third_party/rust/tokio/src/runtime/task/inject.rs +++ /dev/null @@ -1,220 +0,0 @@ -//! Inject queue used to send wakeups to a work-stealing scheduler - -use crate::loom::sync::atomic::AtomicUsize; -use crate::loom::sync::Mutex; -use crate::runtime::task; - -use std::marker::PhantomData; -use std::ptr::NonNull; -use std::sync::atomic::Ordering::{Acquire, Release}; - -/// Growable, MPMC queue used to inject new tasks into the scheduler and as an -/// overflow queue when the local, fixed-size, array queue overflows. -pub(crate) struct Inject { - /// Pointers to the head and tail of the queue. - pointers: Mutex, - - /// Number of pending tasks in the queue. This helps prevent unnecessary - /// locking in the hot path. - len: AtomicUsize, - - _p: PhantomData, -} - -struct Pointers { - /// True if the queue is closed. - is_closed: bool, - - /// Linked-list head. - head: Option>, - - /// Linked-list tail. - tail: Option>, -} - -unsafe impl Send for Inject {} -unsafe impl Sync for Inject {} - -impl Inject { - pub(crate) fn new() -> Inject { - Inject { - pointers: Mutex::new(Pointers { - is_closed: false, - head: None, - tail: None, - }), - len: AtomicUsize::new(0), - _p: PhantomData, - } - } - - pub(crate) fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Closes the injection queue, returns `true` if the queue is open when the - /// transition is made. - pub(crate) fn close(&self) -> bool { - let mut p = self.pointers.lock(); - - if p.is_closed { - return false; - } - - p.is_closed = true; - true - } - - pub(crate) fn is_closed(&self) -> bool { - self.pointers.lock().is_closed - } - - pub(crate) fn len(&self) -> usize { - self.len.load(Acquire) - } - - /// Pushes a value into the queue. - /// - /// This does nothing if the queue is closed. - pub(crate) fn push(&self, task: task::Notified) { - // Acquire queue lock - let mut p = self.pointers.lock(); - - if p.is_closed { - return; - } - - // safety: only mutated with the lock held - let len = unsafe { self.len.unsync_load() }; - let task = task.into_raw(); - - // The next pointer should already be null - debug_assert!(get_next(task).is_none()); - - if let Some(tail) = p.tail { - // safety: Holding the Notified for a task guarantees exclusive - // access to the `queue_next` field. - set_next(tail, Some(task)); - } else { - p.head = Some(task); - } - - p.tail = Some(task); - - self.len.store(len + 1, Release); - } - - /// Pushes several values into the queue. - #[inline] - pub(crate) fn push_batch(&self, mut iter: I) - where - I: Iterator>, - { - let first = match iter.next() { - Some(first) => first.into_raw(), - None => return, - }; - - // Link up all the tasks. - let mut prev = first; - let mut counter = 1; - - // We are going to be called with an `std::iter::Chain`, and that - // iterator overrides `for_each` to something that is easier for the - // compiler to optimize than a loop. - iter.for_each(|next| { - let next = next.into_raw(); - - // safety: Holding the Notified for a task guarantees exclusive - // access to the `queue_next` field. - set_next(prev, Some(next)); - prev = next; - counter += 1; - }); - - // Now that the tasks are linked together, insert them into the - // linked list. - self.push_batch_inner(first, prev, counter); - } - - /// Inserts several tasks that have been linked together into the queue. - /// - /// The provided head and tail may be be the same task. In this case, a - /// single task is inserted. - #[inline] - fn push_batch_inner( - &self, - batch_head: NonNull, - batch_tail: NonNull, - num: usize, - ) { - debug_assert!(get_next(batch_tail).is_none()); - - let mut p = self.pointers.lock(); - - if let Some(tail) = p.tail { - set_next(tail, Some(batch_head)); - } else { - p.head = Some(batch_head); - } - - p.tail = Some(batch_tail); - - // Increment the count. - // - // safety: All updates to the len atomic are guarded by the mutex. As - // such, a non-atomic load followed by a store is safe. - let len = unsafe { self.len.unsync_load() }; - - self.len.store(len + num, Release); - } - - pub(crate) fn pop(&self) -> Option> { - // Fast path, if len == 0, then there are no values - if self.is_empty() { - return None; - } - - let mut p = self.pointers.lock(); - - // It is possible to hit null here if another thread popped the last - // task between us checking `len` and acquiring the lock. - let task = p.head?; - - p.head = get_next(task); - - if p.head.is_none() { - p.tail = None; - } - - set_next(task, None); - - // Decrement the count. - // - // safety: All updates to the len atomic are guarded by the mutex. As - // such, a non-atomic load followed by a store is safe. - self.len - .store(unsafe { self.len.unsync_load() } - 1, Release); - - // safety: a `Notified` is pushed into the queue and now it is popped! - Some(unsafe { task::Notified::from_raw(task) }) - } -} - -impl Drop for Inject { - fn drop(&mut self) { - if !std::thread::panicking() { - assert!(self.pop().is_none(), "queue not empty"); - } - } -} - -fn get_next(header: NonNull) -> Option> { - unsafe { header.as_ref().queue_next.with(|ptr| *ptr) } -} - -fn set_next(header: NonNull, val: Option>) { - unsafe { - header.as_ref().set_next(val); - } -} diff --git a/third_party/rust/tokio/src/runtime/task/join.rs b/third_party/rust/tokio/src/runtime/task/join.rs index 8beed2eaacb5..ee39258846a9 100644 --- a/third_party/rust/tokio/src/runtime/task/join.rs +++ b/third_party/rust/tokio/src/runtime/task/join.rs @@ -1,4 +1,4 @@ -use crate::runtime::task::RawTask; +use crate::runtime::task::{Header, RawTask}; use std::fmt; use std::future::Future; @@ -10,8 +10,10 @@ use std::task::{Context, Poll, Waker}; cfg_rt! { /// An owned permission to join on a task (await its termination). /// - /// This can be thought of as the equivalent of [`std::thread::JoinHandle`] for - /// a task rather than a thread. + /// This can be thought of as the equivalent of [`std::thread::JoinHandle`] + /// for a Tokio task rather than a thread. Note that the background task + /// associated with this `JoinHandle` started running immediately when you + /// called spawn, even if you have not yet awaited the `JoinHandle`. /// /// A `JoinHandle` *detaches* the associated task when it is dropped, which /// means that there is no longer any handle to the task, and no way to `join` @@ -20,6 +22,15 @@ cfg_rt! { /// This `struct` is created by the [`task::spawn`] and [`task::spawn_blocking`] /// functions. /// + /// # Cancel safety + /// + /// The `&mut JoinHandle` type is cancel safe. If it is used as the event + /// in a `tokio::select!` statement and some other branch completes first, + /// then it is guaranteed that the output of the task is not lost. + /// + /// If a `JoinHandle` is dropped, then the task continues running in the + /// background and its return value is lost. + /// /// # Examples /// /// Creation from [`task::spawn`]: @@ -143,7 +154,7 @@ cfg_rt! { /// [`std::thread::JoinHandle`]: std::thread::JoinHandle /// [`JoinError`]: crate::task::JoinError pub struct JoinHandle { - raw: Option, + raw: RawTask, _p: PhantomData, } } @@ -157,7 +168,7 @@ impl RefUnwindSafe for JoinHandle {} impl JoinHandle { pub(super) fn new(raw: RawTask) -> JoinHandle { JoinHandle { - raw: Some(raw), + raw, _p: PhantomData, } } @@ -171,45 +182,129 @@ impl JoinHandle { /// ```rust /// use tokio::time; /// - /// #[tokio::main] - /// async fn main() { - /// let mut handles = Vec::new(); + /// # #[tokio::main(flavor = "current_thread", start_paused = true)] + /// # async fn main() { + /// let mut handles = Vec::new(); /// - /// handles.push(tokio::spawn(async { - /// time::sleep(time::Duration::from_secs(10)).await; - /// true - /// })); + /// handles.push(tokio::spawn(async { + /// time::sleep(time::Duration::from_secs(10)).await; + /// true + /// })); /// - /// handles.push(tokio::spawn(async { - /// time::sleep(time::Duration::from_secs(10)).await; - /// false - /// })); + /// handles.push(tokio::spawn(async { + /// time::sleep(time::Duration::from_secs(10)).await; + /// false + /// })); /// - /// for handle in &handles { - /// handle.abort(); - /// } - /// - /// for handle in handles { - /// assert!(handle.await.unwrap_err().is_cancelled()); - /// } + /// for handle in &handles { + /// handle.abort(); /// } + /// + /// for handle in handles { + /// assert!(handle.await.unwrap_err().is_cancelled()); + /// } + /// # } /// ``` /// [cancelled]: method@super::error::JoinError::is_cancelled pub fn abort(&self) { - if let Some(raw) = self.raw { - raw.remote_abort(); - } + self.raw.remote_abort(); + } + + /// Checks if the task associated with this `JoinHandle` has finished. + /// + /// Please note that this method can return `false` even if [`abort`] has been + /// called on the task. This is because the cancellation process may take + /// some time, and this method does not return `true` until it has + /// completed. + /// + /// ```rust + /// use tokio::time; + /// + /// # #[tokio::main(flavor = "current_thread", start_paused = true)] + /// # async fn main() { + /// let handle1 = tokio::spawn(async { + /// // do some stuff here + /// }); + /// let handle2 = tokio::spawn(async { + /// // do some other stuff here + /// time::sleep(time::Duration::from_secs(10)).await; + /// }); + /// // Wait for the task to finish + /// handle2.abort(); + /// time::sleep(time::Duration::from_secs(1)).await; + /// assert!(handle1.is_finished()); + /// assert!(handle2.is_finished()); + /// # } + /// ``` + /// [`abort`]: method@JoinHandle::abort + pub fn is_finished(&self) -> bool { + let state = self.raw.header().state.load(); + state.is_complete() } /// Set the waker that is notified when the task completes. pub(crate) fn set_join_waker(&mut self, waker: &Waker) { - if let Some(raw) = self.raw { - if raw.try_set_join_waker(waker) { - // In this case the task has already completed. We wake the waker immediately. - waker.wake_by_ref(); - } + if self.raw.try_set_join_waker(waker) { + // In this case the task has already completed. We wake the waker immediately. + waker.wake_by_ref(); } } + + /// Returns a new `AbortHandle` that can be used to remotely abort this task. + /// + /// Awaiting a task cancelled by the `AbortHandle` might complete as usual if the task was + /// already completed at the time it was cancelled, but most likely it + /// will fail with a [cancelled] `JoinError`. + /// + /// ```rust + /// use tokio::{time, task}; + /// + /// # #[tokio::main(flavor = "current_thread", start_paused = true)] + /// # async fn main() { + /// let mut handles = Vec::new(); + /// + /// handles.push(tokio::spawn(async { + /// time::sleep(time::Duration::from_secs(10)).await; + /// true + /// })); + /// + /// handles.push(tokio::spawn(async { + /// time::sleep(time::Duration::from_secs(10)).await; + /// false + /// })); + /// + /// let abort_handles: Vec = handles.iter().map(|h| h.abort_handle()).collect(); + /// + /// for handle in abort_handles { + /// handle.abort(); + /// } + /// + /// for handle in handles { + /// assert!(handle.await.unwrap_err().is_cancelled()); + /// } + /// # } + /// ``` + /// [cancelled]: method@super::error::JoinError::is_cancelled + pub fn abort_handle(&self) -> super::AbortHandle { + self.raw.ref_inc(); + super::AbortHandle::new(self.raw) + } + + /// Returns a [task ID] that uniquely identifies this task relative to other + /// currently spawned tasks. + /// + /// **Note**: This is an [unstable API][unstable]. The public API of this type + /// may break in 1.x releases. See [the documentation on unstable + /// features][unstable] for details. + /// + /// [task ID]: crate::task::Id + /// [unstable]: crate#unstable-features + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn id(&self) -> super::Id { + // Safety: The header pointer is valid. + unsafe { Header::get_id(self.raw.header_ptr()) } + } } impl Unpin for JoinHandle {} @@ -218,17 +313,11 @@ impl Future for JoinHandle { type Output = super::Result; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + ready!(crate::trace::trace_leaf(cx)); let mut ret = Poll::Pending; // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); - - // Raw should always be set. If it is not, this is due to polling after - // completion - let raw = self - .raw - .as_ref() - .expect("polling after `JoinHandle` already completed"); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); // Try to read the task output. If the task is not yet complete, the // waker is stored and is notified once the task does complete. @@ -242,7 +331,8 @@ impl Future for JoinHandle { // // The type of `T` must match the task's output type. unsafe { - raw.try_read_output(&mut ret as *mut _ as *mut (), cx.waker()); + self.raw + .try_read_output(&mut ret as *mut _ as *mut (), cx.waker()); } if ret.is_ready() { @@ -255,13 +345,11 @@ impl Future for JoinHandle { impl Drop for JoinHandle { fn drop(&mut self) { - if let Some(raw) = self.raw.take() { - if raw.header().state.drop_join_handle_fast().is_ok() { - return; - } - - raw.drop_join_handle_slow(); + if self.raw.state().drop_join_handle_fast().is_ok() { + return; } + + self.raw.drop_join_handle_slow(); } } @@ -270,6 +358,9 @@ where T: fmt::Debug, { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("JoinHandle").finish() + // Safety: The header pointer is valid. + let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) }; + let id = unsafe { id_ptr.as_ref() }; + fmt.debug_struct("JoinHandle").field("id", id).finish() } } diff --git a/third_party/rust/tokio/src/runtime/task/list.rs b/third_party/rust/tokio/src/runtime/task/list.rs index 7758f8db7aa2..fb7dbdc1d952 100644 --- a/third_party/rust/tokio/src/runtime/task/list.rs +++ b/third_party/rust/tokio/src/runtime/task/list.rs @@ -10,7 +10,7 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; use crate::loom::sync::Mutex; use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; -use crate::util::linked_list::{Link, LinkedList}; +use crate::util::linked_list::{CountedLinkedList, Link, LinkedList}; use std::marker::PhantomData; @@ -54,9 +54,13 @@ cfg_not_has_atomic_u64! { } pub(crate) struct OwnedTasks { - inner: Mutex>, + inner: Mutex>, id: u64, } +struct CountedOwnedTasksInner { + list: CountedLinkedList, as Link>::Target>, + closed: bool, +} pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, id: u64, @@ -70,8 +74,8 @@ struct OwnedTasksInner { impl OwnedTasks { pub(crate) fn new() -> Self { Self { - inner: Mutex::new(OwnedTasksInner { - list: LinkedList::new(), + inner: Mutex::new(CountedOwnedTasksInner { + list: CountedLinkedList::new(), closed: false, }), id: get_next_id(), @@ -84,13 +88,14 @@ impl OwnedTasks { &self, task: T, scheduler: S, + id: super::Id, ) -> (JoinHandle, Option>) where S: Schedule, T: Future + Send + 'static, T::Output: Send + 'static, { - let (task, notified, join) = super::new_task(task, scheduler); + let (task, notified, join) = super::new_task(task, scheduler, id); unsafe { // safety: We just created the task, so we have exclusive access @@ -152,6 +157,10 @@ impl OwnedTasks { } } + pub(crate) fn active_tasks_count(&self) -> usize { + self.inner.lock().list.count() + } + pub(crate) fn remove(&self, task: &Task) -> Option> { let task_id = task.header().get_owner_id(); if task_id == 0 { @@ -163,7 +172,7 @@ impl OwnedTasks { // safety: We just checked that the provided task is not in some other // linked list. - unsafe { self.inner.lock().list.remove(task.header().into()) } + unsafe { self.inner.lock().list.remove(task.header_ptr()) } } pub(crate) fn is_empty(&self) -> bool { @@ -171,6 +180,18 @@ impl OwnedTasks { } } +cfg_taskdump! { + impl OwnedTasks { + /// Locks the tasks, and calls `f` on an iterator over them. + pub(crate) fn for_each(&self, f: F) + where + F: FnMut(&Task) + { + self.inner.lock().list.for_each(f) + } + } +} + impl LocalOwnedTasks { pub(crate) fn new() -> Self { Self { @@ -187,13 +208,14 @@ impl LocalOwnedTasks { &self, task: T, scheduler: S, + id: super::Id, ) -> (JoinHandle, Option>) where S: Schedule, T: Future + 'static, T::Output: 'static, { - let (task, notified, join) = super::new_task(task, scheduler); + let (task, notified, join) = super::new_task(task, scheduler, id); unsafe { // safety: We just created the task, so we have exclusive access @@ -238,7 +260,7 @@ impl LocalOwnedTasks { self.with_inner(|inner| // safety: We just checked that the provided task is not in some // other linked list. - unsafe { inner.list.remove(task.header().into()) }) + unsafe { inner.list.remove(task.header_ptr()) }) } /// Asserts that the given task is owned by this LocalOwnedTasks and convert diff --git a/third_party/rust/tokio/src/runtime/task/mod.rs b/third_party/rust/tokio/src/runtime/task/mod.rs index 2a492dc985d1..932552fb9153 100644 --- a/third_party/rust/tokio/src/runtime/task/mod.rs +++ b/third_party/rust/tokio/src/runtime/task/mod.rs @@ -47,7 +47,8 @@ //! //! * JOIN_INTEREST - Is set to one if there exists a JoinHandle. //! -//! * JOIN_WAKER - Is set to one if the JoinHandle has set a waker. +//! * JOIN_WAKER - Acts as an access control bit for the join handle waker. The +//! protocol for its usage is described below. //! //! The rest of the bits are used for the ref-count. //! @@ -71,10 +72,38 @@ //! a lock for the stage field, and it can be accessed only by the thread //! that set RUNNING to one. //! -//! * If JOIN_WAKER is zero, then the JoinHandle has exclusive access to the -//! join handle waker. If JOIN_WAKER and COMPLETE are both one, then the -//! thread that set COMPLETE to one has exclusive access to the join handle -//! waker. +//! * The waker field may be concurrently accessed by different threads: in one +//! thread the runtime may complete a task and *read* the waker field to +//! invoke the waker, and in another thread the task's JoinHandle may be +//! polled, and if the task hasn't yet completed, the JoinHandle may *write* +//! a waker to the waker field. The JOIN_WAKER bit ensures safe access by +//! multiple threads to the waker field using the following rules: +//! +//! 1. JOIN_WAKER is initialized to zero. +//! +//! 2. If JOIN_WAKER is zero, then the JoinHandle has exclusive (mutable) +//! access to the waker field. +//! +//! 3. If JOIN_WAKER is one, then the JoinHandle has shared (read-only) +//! access to the waker field. +//! +//! 4. If JOIN_WAKER is one and COMPLETE is one, then the runtime has shared +//! (read-only) access to the waker field. +//! +//! 5. If the JoinHandle needs to write to the waker field, then the +//! JoinHandle needs to (i) successfully set JOIN_WAKER to zero if it is +//! not already zero to gain exclusive access to the waker field per rule +//! 2, (ii) write a waker, and (iii) successfully set JOIN_WAKER to one. +//! +//! 6. The JoinHandle can change JOIN_WAKER only if COMPLETE is zero (i.e. +//! the task hasn't yet completed). +//! +//! Rule 6 implies that the steps (i) or (iii) of rule 5 may fail due to a +//! race. If step (i) fails, then the attempt to write a waker is aborted. If +//! step (iii) fails because COMPLETE is set to one by another thread after +//! step (i), then the waker field is cleared. Once COMPLETE is one (i.e. +//! task has completed), the JoinHandle will not modify JOIN_WAKER. After the +//! runtime sets COMPLETE to one, it invokes the waker if there is one. //! //! All other fields are immutable and can be accessed immutably without //! synchronization by anyone. @@ -121,7 +150,7 @@ //! 1. The output is created on the thread that the future was polled on. Since //! only non-Send futures can have non-Send output, the future was polled on //! the thread that the future was spawned from. -//! 2. Since JoinHandle is not Send if Output is not Send, the +//! 2. Since `JoinHandle` is not Send if Output is not Send, the //! JoinHandle is also on the thread that the future was spawned from. //! 3. Thus, the JoinHandle will not move the output across threads when it //! takes or drops the output. @@ -144,32 +173,39 @@ use self::core::Cell; use self::core::Header; mod error; -#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411 pub use self::error::JoinError; mod harness; use self::harness::Harness; -cfg_rt_multi_thread! { - mod inject; - pub(super) use self::inject::Inject; -} +mod id; +#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] +pub use id::{id, try_id, Id}; +#[cfg(feature = "rt")] +mod abort; mod join; -#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411 + +#[cfg(feature = "rt")] +pub use self::abort::AbortHandle; + pub use self::join::JoinHandle; mod list; pub(crate) use self::list::{LocalOwnedTasks, OwnedTasks}; mod raw; -use self::raw::RawTask; +pub(crate) use self::raw::RawTask; mod state; use self::state::State; mod waker; +cfg_taskdump! { + pub(crate) mod trace; +} + use crate::future::Future; use crate::util::linked_list; @@ -234,6 +270,11 @@ pub(crate) trait Schedule: Sync + Sized + 'static { fn yield_now(&self, task: Notified) { self.schedule(task); } + + /// Polling the task resulted in a panic. Should the runtime shutdown? + fn unhandled_panic(&self) { + // By default, do nothing. This maintains the 1.0 behavior. + } } cfg_rt! { @@ -243,14 +284,15 @@ cfg_rt! { /// notification. fn new_task( task: T, - scheduler: S + scheduler: S, + id: Id, ) -> (Task, Notified, JoinHandle) where S: Schedule, T: Future + 'static, T::Output: 'static, { - let raw = RawTask::new::(task, scheduler); + let raw = RawTask::new::(task, scheduler, id); let task = Task { raw, _p: PhantomData, @@ -268,13 +310,13 @@ cfg_rt! { /// only when the task is not going to be stored in an `OwnedTasks` list. /// /// Currently only blocking tasks use this method. - pub(crate) fn unowned(task: T, scheduler: S) -> (UnownedTask, JoinHandle) + pub(crate) fn unowned(task: T, scheduler: S, id: Id) -> (UnownedTask, JoinHandle) where S: Schedule, T: Send + Future + 'static, T::Output: Send + 'static, { - let (task, notified, join) = new_task(task, scheduler); + let (task, notified, join) = new_task(task, scheduler, id); // This transfers the ref-count of task and notified into an UnownedTask. // This is valid because an UnownedTask holds two ref-counts. @@ -290,16 +332,35 @@ cfg_rt! { } impl Task { - unsafe fn from_raw(ptr: NonNull
) -> Task { + unsafe fn new(raw: RawTask) -> Task { Task { - raw: RawTask::from_raw(ptr), + raw, _p: PhantomData, } } + unsafe fn from_raw(ptr: NonNull
) -> Task { + Task::new(RawTask::from_raw(ptr)) + } + + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + ))] + pub(super) fn as_raw(&self) -> RawTask { + self.raw + } + fn header(&self) -> &Header { self.raw.header() } + + fn header_ptr(&self) -> NonNull
{ + self.raw.header_ptr() + } } impl Notified { @@ -308,30 +369,22 @@ impl Notified { } } -cfg_rt_multi_thread! { - impl Notified { - unsafe fn from_raw(ptr: NonNull
) -> Notified { - Notified(Task::from_raw(ptr)) - } +impl Notified { + pub(crate) unsafe fn from_raw(ptr: RawTask) -> Notified { + Notified(Task::new(ptr)) } +} - impl Task { - fn into_raw(self) -> NonNull
{ - let ret = self.raw.header_ptr(); - mem::forget(self); - ret - } - } - - impl Notified { - fn into_raw(self) -> NonNull
{ - self.0.into_raw() - } +impl Notified { + pub(crate) fn into_raw(self) -> RawTask { + let raw = self.0.raw; + mem::forget(self); + raw } } impl Task { - /// Pre-emptively cancels the task as part of the shutdown process. + /// Preemptively cancels the task as part of the shutdown process. pub(crate) fn shutdown(self) { let raw = self.raw; mem::forget(self); @@ -351,7 +404,7 @@ impl LocalNotified { impl UnownedTask { // Used in test of the inject queue. #[cfg(test)] - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg_attr(tokio_wasm, allow(dead_code))] pub(super) fn into_notified(self) -> Notified { Notified(self.into_task()) } @@ -439,7 +492,6 @@ unsafe impl linked_list::Link for Task { } unsafe fn pointers(target: NonNull
) -> NonNull> { - // Not super great as it avoids some of looms checking... - NonNull::from(target.as_ref().owned.with_mut(|ptr| &mut *ptr)) + self::core::Trailer::addr_of_owned(Header::get_trailer(target)) } } diff --git a/third_party/rust/tokio/src/runtime/task/raw.rs b/third_party/rust/tokio/src/runtime/task/raw.rs index 2e4420b5c133..8078859285d4 100644 --- a/third_party/rust/tokio/src/runtime/task/raw.rs +++ b/third_party/rust/tokio/src/runtime/task/raw.rs @@ -1,11 +1,12 @@ use crate::future::Future; -use crate::runtime::task::{Cell, Harness, Header, Schedule, State}; +use crate::runtime::task::core::{Core, Trailer}; +use crate::runtime::task::{Cell, Harness, Header, Id, Schedule, State}; use std::ptr::NonNull; use std::task::{Poll, Waker}; /// Raw task handle -pub(super) struct RawTask { +pub(crate) struct RawTask { ptr: NonNull
, } @@ -13,47 +14,154 @@ pub(super) struct Vtable { /// Polls the future. pub(super) poll: unsafe fn(NonNull
), + /// Schedules the task for execution on the runtime. + pub(super) schedule: unsafe fn(NonNull
), + /// Deallocates the memory. pub(super) dealloc: unsafe fn(NonNull
), /// Reads the task output, if complete. pub(super) try_read_output: unsafe fn(NonNull
, *mut (), &Waker), - /// Try to set the waker notified when the task is complete. Returns true if - /// the task has already completed. If this call returns false, then the - /// waker will not be notified. - pub(super) try_set_join_waker: unsafe fn(NonNull
, &Waker) -> bool, - /// The join handle has been dropped. pub(super) drop_join_handle_slow: unsafe fn(NonNull
), - /// The task is remotely aborted. - pub(super) remote_abort: unsafe fn(NonNull
), + /// An abort handle has been dropped. + pub(super) drop_abort_handle: unsafe fn(NonNull
), /// Scheduler is being shutdown. pub(super) shutdown: unsafe fn(NonNull
), + + /// The number of bytes that the `trailer` field is offset from the header. + pub(super) trailer_offset: usize, + + /// The number of bytes that the `scheduler` field is offset from the header. + pub(super) scheduler_offset: usize, + + /// The number of bytes that the `id` field is offset from the header. + pub(super) id_offset: usize, } /// Get the vtable for the requested `T` and `S` generics. pub(super) fn vtable() -> &'static Vtable { &Vtable { poll: poll::, + schedule: schedule::, dealloc: dealloc::, try_read_output: try_read_output::, - try_set_join_waker: try_set_join_waker::, drop_join_handle_slow: drop_join_handle_slow::, - remote_abort: remote_abort::, + drop_abort_handle: drop_abort_handle::, shutdown: shutdown::, + trailer_offset: OffsetHelper::::TRAILER_OFFSET, + scheduler_offset: OffsetHelper::::SCHEDULER_OFFSET, + id_offset: OffsetHelper::::ID_OFFSET, } } +/// Calling `get_trailer_offset` directly in vtable doesn't work because it +/// prevents the vtable from being promoted to a static reference. +/// +/// See this thread for more info: +/// +struct OffsetHelper(T, S); +impl OffsetHelper { + // Pass `size_of`/`align_of` as arguments rather than calling them directly + // inside `get_trailer_offset` because trait bounds on generic parameters + // of const fn are unstable on our MSRV. + const TRAILER_OFFSET: usize = get_trailer_offset( + std::mem::size_of::
(), + std::mem::size_of::>(), + std::mem::align_of::>(), + std::mem::align_of::(), + ); + + // The `scheduler` is the first field of `Core`, so it has the same + // offset as `Core`. + const SCHEDULER_OFFSET: usize = get_core_offset( + std::mem::size_of::
(), + std::mem::align_of::>(), + ); + + const ID_OFFSET: usize = get_id_offset( + std::mem::size_of::
(), + std::mem::align_of::>(), + std::mem::size_of::(), + std::mem::align_of::(), + ); +} + +/// Compute the offset of the `Trailer` field in `Cell` using the +/// `#[repr(C)]` algorithm. +/// +/// Pseudo-code for the `#[repr(C)]` algorithm can be found here: +/// +const fn get_trailer_offset( + header_size: usize, + core_size: usize, + core_align: usize, + trailer_align: usize, +) -> usize { + let mut offset = header_size; + + let core_misalign = offset % core_align; + if core_misalign > 0 { + offset += core_align - core_misalign; + } + offset += core_size; + + let trailer_misalign = offset % trailer_align; + if trailer_misalign > 0 { + offset += trailer_align - trailer_misalign; + } + + offset +} + +/// Compute the offset of the `Core` field in `Cell` using the +/// `#[repr(C)]` algorithm. +/// +/// Pseudo-code for the `#[repr(C)]` algorithm can be found here: +/// +const fn get_core_offset(header_size: usize, core_align: usize) -> usize { + let mut offset = header_size; + + let core_misalign = offset % core_align; + if core_misalign > 0 { + offset += core_align - core_misalign; + } + + offset +} + +/// Compute the offset of the `Id` field in `Cell` using the +/// `#[repr(C)]` algorithm. +/// +/// Pseudo-code for the `#[repr(C)]` algorithm can be found here: +/// +const fn get_id_offset( + header_size: usize, + core_align: usize, + scheduler_size: usize, + id_align: usize, +) -> usize { + let mut offset = get_core_offset(header_size, core_align); + offset += scheduler_size; + + let id_misalign = offset % id_align; + if id_misalign > 0 { + offset += id_align - id_misalign; + } + + offset +} + impl RawTask { - pub(super) fn new(task: T, scheduler: S) -> RawTask + pub(super) fn new(task: T, scheduler: S, id: Id) -> RawTask where T: Future, S: Schedule, { - let ptr = Box::into_raw(Cell::<_, S>::new(task, scheduler, State::new())); + let ptr = Box::into_raw(Cell::<_, S>::new(task, scheduler, State::new(), id)); let ptr = unsafe { NonNull::new_unchecked(ptr as *mut Header) }; RawTask { ptr } @@ -67,19 +175,36 @@ impl RawTask { self.ptr } - /// Returns a reference to the task's meta structure. - /// - /// Safe as `Header` is `Sync`. + pub(super) fn trailer_ptr(&self) -> NonNull { + unsafe { Header::get_trailer(self.ptr) } + } + + /// Returns a reference to the task's header. pub(super) fn header(&self) -> &Header { unsafe { self.ptr.as_ref() } } + /// Returns a reference to the task's trailer. + pub(super) fn trailer(&self) -> &Trailer { + unsafe { &*self.trailer_ptr().as_ptr() } + } + + /// Returns a reference to the task's state. + pub(super) fn state(&self) -> &State { + &self.header().state + } + /// Safety: mutual exclusion is required to call this function. - pub(super) fn poll(self) { + pub(crate) fn poll(self) { let vtable = self.header().vtable; unsafe { (vtable.poll)(self.ptr) } } + pub(super) fn schedule(self) { + let vtable = self.header().vtable; + unsafe { (vtable.schedule)(self.ptr) } + } + pub(super) fn dealloc(self) { let vtable = self.header().vtable; unsafe { @@ -94,24 +219,47 @@ impl RawTask { (vtable.try_read_output)(self.ptr, dst, waker); } - pub(super) fn try_set_join_waker(self, waker: &Waker) -> bool { - let vtable = self.header().vtable; - unsafe { (vtable.try_set_join_waker)(self.ptr, waker) } - } - pub(super) fn drop_join_handle_slow(self) { let vtable = self.header().vtable; unsafe { (vtable.drop_join_handle_slow)(self.ptr) } } + pub(super) fn drop_abort_handle(self) { + let vtable = self.header().vtable; + unsafe { (vtable.drop_abort_handle)(self.ptr) } + } + pub(super) fn shutdown(self) { let vtable = self.header().vtable; unsafe { (vtable.shutdown)(self.ptr) } } - pub(super) fn remote_abort(self) { - let vtable = self.header().vtable; - unsafe { (vtable.remote_abort)(self.ptr) } + /// Increment the task's reference count. + /// + /// Currently, this is used only when creating an `AbortHandle`. + pub(super) fn ref_inc(self) { + self.header().state.ref_inc(); + } + + /// Get the queue-next pointer + /// + /// This is for usage by the injection queue + /// + /// Safety: make sure only one queue uses this and access is synchronized. + pub(crate) unsafe fn get_queue_next(self) -> Option { + self.header() + .queue_next + .with(|ptr| *ptr) + .map(|p| RawTask::from_raw(p)) + } + + /// Sets the queue-next pointer + /// + /// This is for usage by the injection queue + /// + /// Safety: make sure only one queue uses this and access is synchronized. + pub(crate) unsafe fn set_queue_next(self, val: Option) { + self.header().set_next(val.map(|task| task.ptr)); } } @@ -128,6 +276,15 @@ unsafe fn poll(ptr: NonNull
) { harness.poll(); } +unsafe fn schedule(ptr: NonNull
) { + use crate::runtime::task::{Notified, Task}; + + let scheduler = Header::get_scheduler::(ptr); + scheduler + .as_ref() + .schedule(Notified(Task::from_raw(ptr.cast()))); +} + unsafe fn dealloc(ptr: NonNull
) { let harness = Harness::::from_raw(ptr); harness.dealloc(); @@ -144,19 +301,14 @@ unsafe fn try_read_output( harness.try_read_output(out, waker); } -unsafe fn try_set_join_waker(ptr: NonNull
, waker: &Waker) -> bool { - let harness = Harness::::from_raw(ptr); - harness.try_set_join_waker(waker) -} - unsafe fn drop_join_handle_slow(ptr: NonNull
) { let harness = Harness::::from_raw(ptr); harness.drop_join_handle_slow() } -unsafe fn remote_abort(ptr: NonNull
) { +unsafe fn drop_abort_handle(ptr: NonNull
) { let harness = Harness::::from_raw(ptr); - harness.remote_abort() + harness.drop_reference(); } unsafe fn shutdown(ptr: NonNull
) { diff --git a/third_party/rust/tokio/src/runtime/task/state.rs b/third_party/rust/tokio/src/runtime/task/state.rs index c2d5b28eac28..12f544918137 100644 --- a/third_party/rust/tokio/src/runtime/task/state.rs +++ b/third_party/rust/tokio/src/runtime/task/state.rs @@ -88,7 +88,7 @@ pub(super) enum TransitionToNotifiedByVal { } #[must_use] -pub(super) enum TransitionToNotifiedByRef { +pub(crate) enum TransitionToNotifiedByRef { DoNothing, Submit, } @@ -270,6 +270,22 @@ impl State { }) } + /// Transitions the state to `NOTIFIED`, unconditionally increasing the ref count. + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") + ))] + pub(super) fn transition_to_notified_for_tracing(&self) { + self.fetch_update_action(|mut snapshot| { + snapshot.set_notified(); + snapshot.ref_inc(); + ((), Some(snapshot)) + }); + } + /// Sets the cancelled bit and transitions the state to `NOTIFIED` if idle. /// /// Returns `true` if the task needs to be submitted to the pool for @@ -378,7 +394,7 @@ impl State { pub(super) fn set_join_waker(&self) -> UpdateResult { self.fetch_update(|curr| { assert!(curr.is_join_interested()); - assert!(!curr.has_join_waker()); + assert!(!curr.is_join_waker_set()); if curr.is_complete() { return None; @@ -398,7 +414,7 @@ impl State { pub(super) fn unset_waker(&self) -> UpdateResult { self.fetch_update(|curr| { assert!(curr.is_join_interested()); - assert!(curr.has_join_waker()); + assert!(curr.is_join_waker_set()); if curr.is_complete() { return None; @@ -546,7 +562,7 @@ impl Snapshot { self.0 &= !JOIN_INTEREST } - pub(super) fn has_join_waker(self) -> bool { + pub(super) fn is_join_waker_set(self) -> bool { self.0 & JOIN_WAKER == JOIN_WAKER } @@ -588,7 +604,7 @@ impl fmt::Debug for Snapshot { .field("is_notified", &self.is_notified()) .field("is_cancelled", &self.is_cancelled()) .field("is_join_interested", &self.is_join_interested()) - .field("has_join_waker", &self.has_join_waker()) + .field("is_join_waker_set", &self.is_join_waker_set()) .field("ref_count", &self.ref_count()) .finish() } diff --git a/third_party/rust/tokio/src/runtime/task/trace/mod.rs b/third_party/rust/tokio/src/runtime/task/trace/mod.rs new file mode 100644 index 000000000000..543b7eee98ed --- /dev/null +++ b/third_party/rust/tokio/src/runtime/task/trace/mod.rs @@ -0,0 +1,330 @@ +use crate::loom::sync::Arc; +use crate::runtime::context; +use crate::runtime::scheduler::{self, current_thread, Inject}; + +use backtrace::BacktraceFrame; +use std::cell::Cell; +use std::collections::VecDeque; +use std::ffi::c_void; +use std::fmt; +use std::future::Future; +use std::pin::Pin; +use std::ptr::{self, NonNull}; +use std::task::{self, Poll}; + +mod symbol; +mod tree; + +use symbol::Symbol; +use tree::Tree; + +use super::{Notified, OwnedTasks}; + +type Backtrace = Vec; +type SymbolTrace = Vec; + +/// The ambiant backtracing context. +pub(crate) struct Context { + /// The address of [`Trace::root`] establishes an upper unwinding bound on + /// the backtraces in `Trace`. + active_frame: Cell>>, + /// The place to stash backtraces. + collector: Cell>, +} + +/// A [`Frame`] in an intrusive, doubly-linked tree of [`Frame`]s. +struct Frame { + /// The location associated with this frame. + inner_addr: *const c_void, + + /// The parent frame, if any. + parent: Option>, +} + +/// An tree execution trace. +/// +/// Traces are captured with [`Trace::capture`], rooted with [`Trace::root`] +/// and leaved with [`trace_leaf`]. +#[derive(Clone, Debug)] +pub(crate) struct Trace { + // The linear backtraces that comprise this trace. These linear traces can + // be re-knitted into a tree. + backtraces: Vec, +} + +pin_project_lite::pin_project! { + #[derive(Debug, Clone)] + #[must_use = "futures do nothing unless you `.await` or poll them"] + pub(crate) struct Root { + #[pin] + future: T, + } +} + +const FAIL_NO_THREAD_LOCAL: &str = "The Tokio thread-local has been destroyed \ + as part of shutting down the current \ + thread, so collecting a taskdump is not \ + possible."; + +impl Context { + pub(crate) const fn new() -> Self { + Context { + active_frame: Cell::new(None), + collector: Cell::new(None), + } + } + + /// SAFETY: Callers of this function must ensure that trace frames always + /// form a valid linked list. + unsafe fn try_with_current(f: F) -> Option + where + F: FnOnce(&Self) -> R, + { + crate::runtime::context::with_trace(f) + } + + unsafe fn with_current_frame(f: F) -> R + where + F: FnOnce(&Cell>>) -> R, + { + Self::try_with_current(|context| f(&context.active_frame)).expect(FAIL_NO_THREAD_LOCAL) + } + + fn with_current_collector(f: F) -> R + where + F: FnOnce(&Cell>) -> R, + { + // SAFETY: This call can only access the collector field, so it cannot + // break the trace frame linked list. + unsafe { + Self::try_with_current(|context| f(&context.collector)).expect(FAIL_NO_THREAD_LOCAL) + } + } +} + +impl Trace { + /// Invokes `f`, returning both its result and the collection of backtraces + /// captured at each sub-invocation of [`trace_leaf`]. + #[inline(never)] + pub(crate) fn capture(f: F) -> (R, Trace) + where + F: FnOnce() -> R, + { + let collector = Trace { backtraces: vec![] }; + + let previous = Context::with_current_collector(|current| current.replace(Some(collector))); + + let result = f(); + + let collector = + Context::with_current_collector(|current| current.replace(previous)).unwrap(); + + (result, collector) + } + + /// The root of a trace. + #[inline(never)] + pub(crate) fn root(future: F) -> Root { + Root { future } + } +} + +/// If this is a sub-invocation of [`Trace::capture`], capture a backtrace. +/// +/// The captured backtrace will be returned by [`Trace::capture`]. +/// +/// Invoking this function does nothing when it is not a sub-invocation +/// [`Trace::capture`]. +// This function is marked `#[inline(never)]` to ensure that it gets a distinct `Frame` in the +// backtrace, below which frames should not be included in the backtrace (since they reflect the +// internal implementation details of this crate). +#[inline(never)] +pub(crate) fn trace_leaf(cx: &mut task::Context<'_>) -> Poll<()> { + // Safety: We don't manipulate the current context's active frame. + let did_trace = unsafe { + Context::try_with_current(|context_cell| { + if let Some(mut collector) = context_cell.collector.take() { + let mut frames = vec![]; + let mut above_leaf = false; + + if let Some(active_frame) = context_cell.active_frame.get() { + let active_frame = active_frame.as_ref(); + + backtrace::trace(|frame| { + let below_root = !ptr::eq(frame.symbol_address(), active_frame.inner_addr); + + // only capture frames above `Trace::leaf` and below + // `Trace::root`. + if above_leaf && below_root { + frames.push(frame.to_owned().into()); + } + + if ptr::eq(frame.symbol_address(), trace_leaf as *const _) { + above_leaf = true; + } + + // only continue unwinding if we're below `Trace::root` + below_root + }); + } + collector.backtraces.push(frames); + context_cell.collector.set(Some(collector)); + true + } else { + false + } + }) + .unwrap_or(false) + }; + + if did_trace { + // Use the same logic that `yield_now` uses to send out wakeups after + // the task yields. + context::with_scheduler(|scheduler| { + if let Some(scheduler) = scheduler { + match scheduler { + scheduler::Context::CurrentThread(s) => s.defer.defer(cx.waker()), + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + scheduler::Context::MultiThread(s) => s.defer.defer(cx.waker()), + } + } + }); + + Poll::Pending + } else { + Poll::Ready(()) + } +} + +impl fmt::Display for Trace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Tree::from_trace(self.clone()).fmt(f) + } +} + +fn defer R, R>(f: F) -> impl Drop { + use std::mem::ManuallyDrop; + + struct Defer R, R>(ManuallyDrop); + + impl R, R> Drop for Defer { + #[inline(always)] + fn drop(&mut self) { + unsafe { + ManuallyDrop::take(&mut self.0)(); + } + } + } + + Defer(ManuallyDrop::new(f)) +} + +impl Future for Root { + type Output = T::Output; + + #[inline(never)] + fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll { + // SAFETY: The context's current frame is restored to its original state + // before `frame` is dropped. + unsafe { + let mut frame = Frame { + inner_addr: Self::poll as *const c_void, + parent: None, + }; + + Context::with_current_frame(|current| { + frame.parent = current.take(); + current.set(Some(NonNull::from(&frame))); + }); + + let _restore = defer(|| { + Context::with_current_frame(|current| { + current.set(frame.parent); + }); + }); + + let this = self.project(); + this.future.poll(cx) + } + } +} + +/// Trace and poll all tasks of the current_thread runtime. +pub(in crate::runtime) fn trace_current_thread( + owned: &OwnedTasks>, + local: &mut VecDeque>>, + injection: &Inject>, +) -> Vec { + // clear the local and injection queues + local.clear(); + + while let Some(task) = injection.pop() { + drop(task); + } + + // notify each task + let mut tasks = vec![]; + owned.for_each(|task| { + // set the notified bit + task.as_raw().state().transition_to_notified_for_tracing(); + // store the raw tasks into a vec + tasks.push(task.as_raw()); + }); + + tasks + .into_iter() + .map(|task| { + let ((), trace) = Trace::capture(|| task.poll()); + trace + }) + .collect() +} + +cfg_rt_multi_thread! { + use crate::loom::sync::Mutex; + use crate::runtime::scheduler::multi_thread; + use crate::runtime::scheduler::multi_thread::Synced; + use crate::runtime::scheduler::inject::Shared; + + /// Trace and poll all tasks of the current_thread runtime. + /// + /// ## Safety + /// + /// Must be called with the same `synced` that `injection` was created with. + pub(in crate::runtime) unsafe fn trace_multi_thread( + owned: &OwnedTasks>, + local: &mut multi_thread::queue::Local>, + synced: &Mutex, + injection: &Shared>, + ) -> Vec { + // clear the local queue + while let Some(notified) = local.pop() { + drop(notified); + } + + // clear the injection queue + let mut synced = synced.lock(); + while let Some(notified) = injection.pop(&mut synced.inject) { + drop(notified); + } + + drop(synced); + + // notify each task + let mut traces = vec![]; + owned.for_each(|task| { + // set the notified bit + task.as_raw().state().transition_to_notified_for_tracing(); + + // trace the task + let ((), trace) = Trace::capture(|| task.as_raw().poll()); + traces.push(trace); + + // reschedule the task + let _ = task.as_raw().state().transition_to_notified_by_ref(); + task.as_raw().schedule(); + }); + + traces + } +} diff --git a/third_party/rust/tokio/src/runtime/task/trace/symbol.rs b/third_party/rust/tokio/src/runtime/task/trace/symbol.rs new file mode 100644 index 000000000000..49d7ba37f738 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/task/trace/symbol.rs @@ -0,0 +1,92 @@ +use backtrace::BacktraceSymbol; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::ptr; + +/// A symbol in a backtrace. +/// +/// This wrapper type serves two purposes. The first is that it provides a +/// representation of a symbol that can be inserted into hashmaps and hashsets; +/// the [`backtrace`] crate does not define [`Hash`], [`PartialEq`], or [`Eq`] +/// on [`BacktraceSymbol`], and recommends that users define their own wrapper +/// which implements these traits. +/// +/// Second, this wrapper includes a `parent_hash` field that uniquely +/// identifies this symbol's position in its trace. Otherwise, e.g., our code +/// would not be able to distinguish between recursive calls of a function at +/// different depths. +#[derive(Clone)] +pub(super) struct Symbol { + pub(super) symbol: BacktraceSymbol, + pub(super) parent_hash: u64, +} + +impl Hash for Symbol { + fn hash(&self, state: &mut H) + where + H: Hasher, + { + if let Some(name) = self.symbol.name() { + name.as_bytes().hash(state); + } + + if let Some(addr) = self.symbol.addr() { + ptr::hash(addr, state); + } + + self.symbol.filename().hash(state); + self.symbol.lineno().hash(state); + self.symbol.colno().hash(state); + self.parent_hash.hash(state); + } +} + +impl PartialEq for Symbol { + fn eq(&self, other: &Self) -> bool { + (self.parent_hash == other.parent_hash) + && match (self.symbol.name(), other.symbol.name()) { + (None, None) => true, + (Some(lhs_name), Some(rhs_name)) => lhs_name.as_bytes() == rhs_name.as_bytes(), + _ => false, + } + && match (self.symbol.addr(), other.symbol.addr()) { + (None, None) => true, + (Some(lhs_addr), Some(rhs_addr)) => ptr::eq(lhs_addr, rhs_addr), + _ => false, + } + && (self.symbol.filename() == other.symbol.filename()) + && (self.symbol.lineno() == other.symbol.lineno()) + && (self.symbol.colno() == other.symbol.colno()) + } +} + +impl Eq for Symbol {} + +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(name) = self.symbol.name() { + let name = name.to_string(); + let name = if let Some((name, _)) = name.rsplit_once("::") { + name + } else { + &name + }; + fmt::Display::fmt(&name, f)?; + } + + if let Some(filename) = self.symbol.filename() { + f.write_str(" at ")?; + filename.to_string_lossy().fmt(f)?; + if let Some(lineno) = self.symbol.lineno() { + f.write_str(":")?; + fmt::Display::fmt(&lineno, f)?; + if let Some(colno) = self.symbol.colno() { + f.write_str(":")?; + fmt::Display::fmt(&colno, f)?; + } + } + } + + Ok(()) + } +} diff --git a/third_party/rust/tokio/src/runtime/task/trace/tree.rs b/third_party/rust/tokio/src/runtime/task/trace/tree.rs new file mode 100644 index 000000000000..7e6f8efeca95 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/task/trace/tree.rs @@ -0,0 +1,126 @@ +use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; +use std::fmt; +use std::hash::{Hash, Hasher}; + +use super::{Backtrace, Symbol, SymbolTrace, Trace}; + +/// An adjacency list representation of an execution tree. +/// +/// This tree provides a convenient intermediate representation for formatting +/// [`Trace`] as a tree. +pub(super) struct Tree { + /// The roots of the trees. + /// + /// There should only be one root, but the code is robust to multiple roots. + roots: HashSet, + + /// The adjacency list of symbols in the execution tree(s). + edges: HashMap>, +} + +impl Tree { + /// Constructs a [`Tree`] from [`Trace`] + pub(super) fn from_trace(trace: Trace) -> Self { + let mut roots: HashSet = HashSet::default(); + let mut edges: HashMap> = HashMap::default(); + + for trace in trace.backtraces { + let trace = to_symboltrace(trace); + + if let Some(first) = trace.first() { + roots.insert(first.to_owned()); + } + + let mut trace = trace.into_iter().peekable(); + while let Some(frame) = trace.next() { + let subframes = edges.entry(frame).or_default(); + if let Some(subframe) = trace.peek() { + subframes.insert(subframe.clone()); + } + } + } + + Tree { roots, edges } + } + + /// Produces the sub-symbols of a given symbol. + fn consequences(&self, frame: &Symbol) -> Option> { + Some(self.edges.get(frame)?.iter()) + } + + /// Format this [`Tree`] as a textual tree. + fn display( + &self, + f: &mut W, + root: &Symbol, + is_last: bool, + prefix: &str, + ) -> fmt::Result { + let root_fmt = format!("{}", root); + + let current; + let next; + + if is_last { + current = format!("{prefix}└╼\u{a0}{root_fmt}"); + next = format!("{}\u{a0}\u{a0}\u{a0}", prefix); + } else { + current = format!("{prefix}├╼\u{a0}{root_fmt}"); + next = format!("{}│\u{a0}\u{a0}", prefix); + } + + write!(f, "{}", { + let mut current = current.chars(); + current.next().unwrap(); + current.next().unwrap(); + ¤t.as_str() + })?; + + if let Some(consequences) = self.consequences(root) { + let len = consequences.len(); + for (i, consequence) in consequences.enumerate() { + let is_last = i == len - 1; + writeln!(f)?; + self.display(f, consequence, is_last, &next)?; + } + } + + Ok(()) + } +} + +impl fmt::Display for Tree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for root in &self.roots { + self.display(f, root, true, " ")?; + } + Ok(()) + } +} + +/// Resolve a sequence of [`backtrace::BacktraceFrame`]s into a sequence of +/// [`Symbol`]s. +fn to_symboltrace(backtrace: Backtrace) -> SymbolTrace { + // Resolve the backtrace frames to symbols. + let backtrace: Backtrace = { + let mut backtrace = backtrace::Backtrace::from(backtrace); + backtrace.resolve(); + backtrace.into() + }; + + // Accumulate the symbols in descending order into `symboltrace`. + let mut symboltrace: SymbolTrace = vec![]; + let mut state = DefaultHasher::new(); + for frame in backtrace.into_iter().rev() { + for symbol in frame.symbols().iter().rev() { + let symbol = Symbol { + symbol: symbol.clone(), + parent_hash: state.finish(), + }; + symbol.hash(&mut state); + symboltrace.push(symbol); + } + } + + symboltrace +} diff --git a/third_party/rust/tokio/src/runtime/task/waker.rs b/third_party/rust/tokio/src/runtime/task/waker.rs index 74a29f4a8476..b5f5ace9ecee 100644 --- a/third_party/rust/tokio/src/runtime/task/waker.rs +++ b/third_party/rust/tokio/src/runtime/task/waker.rs @@ -1,6 +1,5 @@ use crate::future::Future; -use crate::runtime::task::harness::Harness; -use crate::runtime::task::{Header, Schedule}; +use crate::runtime::task::{Header, RawTask, Schedule}; use std::marker::PhantomData; use std::mem::ManuallyDrop; @@ -13,7 +12,7 @@ pub(super) struct WakerRef<'a, S: 'static> { _p: PhantomData<(&'a Header, S)>, } -/// Returns a `WakerRef` which avoids having to pre-emptively increase the +/// Returns a `WakerRef` which avoids having to preemptively increase the /// refcount if there is no need to do so. pub(super) fn waker_ref(header: &NonNull
) -> WakerRef<'_, S> where @@ -28,7 +27,7 @@ where // point and not an *owned* waker, we must ensure that `drop` is never // called on this waker instance. This is done by wrapping it with // `ManuallyDrop` and then never calling drop. - let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker::(*header))) }; + let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker(*header))) }; WakerRef { waker, @@ -46,8 +45,8 @@ impl ops::Deref for WakerRef<'_, S> { cfg_trace! { macro_rules! trace { - ($harness:expr, $op:expr) => { - if let Some(id) = $harness.id() { + ($header:expr, $op:expr) => { + if let Some(id) = Header::get_tracing_id(&$header) { tracing::trace!( target: "tokio::task::waker", op = $op, @@ -60,71 +59,46 @@ cfg_trace! { cfg_not_trace! { macro_rules! trace { - ($harness:expr, $op:expr) => { + ($header:expr, $op:expr) => { // noop - let _ = &$harness; + let _ = &$header; } } } -unsafe fn clone_waker(ptr: *const ()) -> RawWaker -where - T: Future, - S: Schedule, -{ - let header = ptr as *const Header; - let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::::from_raw(ptr); - trace!(harness, "waker.clone"); - (*header).state.ref_inc(); - raw_waker::(ptr) +unsafe fn clone_waker(ptr: *const ()) -> RawWaker { + let header = NonNull::new_unchecked(ptr as *mut Header); + trace!(header, "waker.clone"); + header.as_ref().state.ref_inc(); + raw_waker(header) } -unsafe fn drop_waker(ptr: *const ()) -where - T: Future, - S: Schedule, -{ +unsafe fn drop_waker(ptr: *const ()) { let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::::from_raw(ptr); - trace!(harness, "waker.drop"); - harness.drop_reference(); + trace!(ptr, "waker.drop"); + let raw = RawTask::from_raw(ptr); + raw.drop_reference(); } -unsafe fn wake_by_val(ptr: *const ()) -where - T: Future, - S: Schedule, -{ +unsafe fn wake_by_val(ptr: *const ()) { let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::::from_raw(ptr); - trace!(harness, "waker.wake"); - harness.wake_by_val(); + trace!(ptr, "waker.wake"); + let raw = RawTask::from_raw(ptr); + raw.wake_by_val(); } // Wake without consuming the waker -unsafe fn wake_by_ref(ptr: *const ()) -where - T: Future, - S: Schedule, -{ +unsafe fn wake_by_ref(ptr: *const ()) { let ptr = NonNull::new_unchecked(ptr as *mut Header); - let harness = Harness::::from_raw(ptr); - trace!(harness, "waker.wake_by_ref"); - harness.wake_by_ref(); + trace!(ptr, "waker.wake_by_ref"); + let raw = RawTask::from_raw(ptr); + raw.wake_by_ref(); } -fn raw_waker(header: NonNull
) -> RawWaker -where - T: Future, - S: Schedule, -{ +static WAKER_VTABLE: RawWakerVTable = + RawWakerVTable::new(clone_waker, wake_by_val, wake_by_ref, drop_waker); + +fn raw_waker(header: NonNull
) -> RawWaker { let ptr = header.as_ptr() as *const (); - let vtable = &RawWakerVTable::new( - clone_waker::, - wake_by_val::, - wake_by_ref::, - drop_waker::, - ); - RawWaker::new(ptr, vtable) + RawWaker::new(ptr, &WAKER_VTABLE) } diff --git a/third_party/rust/tokio/src/runtime/tests/inject.rs b/third_party/rust/tokio/src/runtime/tests/inject.rs new file mode 100644 index 000000000000..ccead5e024a9 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/tests/inject.rs @@ -0,0 +1,54 @@ +use crate::runtime::scheduler::inject; + +#[test] +fn push_and_pop() { + const N: usize = 2; + + let (inject, mut synced) = inject::Shared::new(); + + for i in 0..N { + assert_eq!(inject.len(), i); + let (task, _) = super::unowned(async {}); + unsafe { inject.push(&mut synced, task) }; + } + + for i in 0..N { + assert_eq!(inject.len(), N - i); + assert!(unsafe { inject.pop(&mut synced) }.is_some()); + } + + println!("--------------"); + + assert!(unsafe { inject.pop(&mut synced) }.is_none()); +} + +#[test] +fn push_batch_and_pop() { + let (inject, mut inject_synced) = inject::Shared::new(); + + unsafe { + inject.push_batch( + &mut inject_synced, + (0..10).map(|_| super::unowned(async {}).0), + ); + + assert_eq!(5, inject.pop_n(&mut inject_synced, 5).count()); + assert_eq!(5, inject.pop_n(&mut inject_synced, 5).count()); + assert_eq!(0, inject.pop_n(&mut inject_synced, 5).count()); + } +} + +#[test] +fn pop_n_drains_on_drop() { + let (inject, mut inject_synced) = inject::Shared::new(); + + unsafe { + inject.push_batch( + &mut inject_synced, + (0..10).map(|_| super::unowned(async {}).0), + ); + let _ = inject.pop_n(&mut inject_synced, 10); + + assert_eq!(inject.len(), 0); + } +} diff --git a/third_party/rust/tokio/src/runtime/tests/loom_blocking.rs b/third_party/rust/tokio/src/runtime/tests/loom_blocking.rs index 89de85e4362f..5c4aeae39c5c 100644 --- a/third_party/rust/tokio/src/runtime/tests/loom_blocking.rs +++ b/third_party/rust/tokio/src/runtime/tests/loom_blocking.rs @@ -73,6 +73,27 @@ fn spawn_mandatory_blocking_should_run_even_when_shutting_down_from_other_thread }); } +#[test] +fn spawn_blocking_when_paused() { + use std::time::Duration; + loom::model(|| { + let rt = crate::runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .unwrap(); + let handle = rt.handle(); + let _enter = handle.enter(); + let a = crate::task::spawn_blocking(|| {}); + let b = crate::task::spawn_blocking(|| {}); + rt.block_on(crate::time::timeout(Duration::from_millis(1), async move { + a.await.expect("blocking task should finish"); + b.await.expect("blocking task should finish"); + })) + .expect("timeout should not trigger"); + }); +} + fn mk_runtime(num_threads: usize) -> Runtime { runtime::Builder::new_multi_thread() .worker_threads(num_threads) diff --git a/third_party/rust/tokio/src/runtime/tests/loom_basic_scheduler.rs b/third_party/rust/tokio/src/runtime/tests/loom_current_thread_scheduler.rs similarity index 100% rename from third_party/rust/tokio/src/runtime/tests/loom_basic_scheduler.rs rename to third_party/rust/tokio/src/runtime/tests/loom_current_thread_scheduler.rs diff --git a/third_party/rust/tokio/src/runtime/tests/loom_join_set.rs b/third_party/rust/tokio/src/runtime/tests/loom_join_set.rs index e87ddb014069..bd343876a26c 100644 --- a/third_party/rust/tokio/src/runtime/tests/loom_join_set.rs +++ b/third_party/rust/tokio/src/runtime/tests/loom_join_set.rs @@ -16,13 +16,13 @@ fn test_join_set() { assert_eq!(set.len(), 1); set.spawn(async { () }); assert_eq!(set.len(), 2); - let () = set.join_one().await.unwrap().unwrap(); + let () = set.join_next().await.unwrap().unwrap(); assert_eq!(set.len(), 1); set.spawn(async { () }); assert_eq!(set.len(), 2); - let () = set.join_one().await.unwrap().unwrap(); + let () = set.join_next().await.unwrap().unwrap(); assert_eq!(set.len(), 1); - let () = set.join_one().await.unwrap().unwrap(); + let () = set.join_next().await.unwrap().unwrap(); assert_eq!(set.len(), 0); set.spawn(async { () }); assert_eq!(set.len(), 1); @@ -60,16 +60,16 @@ fn abort_all_during_completion() { set.spawn(async { () }); set.abort_all(); - match set.join_one().await { - Ok(Some(())) => complete_happened.store(true, SeqCst), - Err(err) if err.is_cancelled() => cancel_happened.store(true, SeqCst), - Err(err) => panic!("fail: {}", err), - Ok(None) => { + match set.join_next().await { + Some(Ok(())) => complete_happened.store(true, SeqCst), + Some(Err(err)) if err.is_cancelled() => cancel_happened.store(true, SeqCst), + Some(Err(err)) => panic!("fail: {}", err), + None => { unreachable!("Aborting the task does not remove it from the JoinSet.") } } - assert!(matches!(set.join_one().await, Ok(None))); + assert!(matches!(set.join_next().await, None)); }); drop(set); diff --git a/third_party/rust/tokio/src/runtime/tests/loom_pool.rs b/third_party/rust/tokio/src/runtime/tests/loom_pool.rs index b3ecd431240f..fb42e1eb40bc 100644 --- a/third_party/rust/tokio/src/runtime/tests/loom_pool.rs +++ b/third_party/rust/tokio/src/runtime/tests/loom_pool.rs @@ -323,8 +323,7 @@ mod group_d { // Spawn a task let c2 = c1.clone(); pool.spawn(track(async move { - gated().await; - gated().await; + multi_gated().await; if 1 == c1.fetch_add(1, Relaxed) { done_tx1.assert_send(()); @@ -333,8 +332,7 @@ mod group_d { // Spawn a second task pool.spawn(track(async move { - gated().await; - gated().await; + multi_gated().await; if 1 == c2.fetch_add(1, Relaxed) { done_tx2.assert_send(()); @@ -349,14 +347,12 @@ mod group_d { fn mk_pool(num_threads: usize) -> Runtime { runtime::Builder::new_multi_thread() .worker_threads(num_threads) + // Set the intervals to avoid tuning logic + .event_interval(2) .build() .unwrap() } -fn gated() -> impl Future { - gated2(false) -} - fn gated2(thread: bool) -> impl Future { use loom::thread; use std::sync::Arc; @@ -394,6 +390,38 @@ fn gated2(thread: bool) -> impl Future { }) } +async fn multi_gated() { + struct Gate { + waker: loom::future::AtomicWaker, + count: AtomicUsize, + } + + let gate = Arc::new(Gate { + waker: loom::future::AtomicWaker::new(), + count: AtomicUsize::new(0), + }); + + { + let gate = gate.clone(); + spawn(track(async move { + for i in 1..3 { + gate.count.store(i, SeqCst); + gate.waker.wake(); + } + })); + } + + poll_fn(move |cx| { + if gate.count.load(SeqCst) < 2 { + gate.waker.register_by_ref(cx.waker()); + Poll::Pending + } else { + Poll::Ready(()) + } + }) + .await; +} + fn track(f: T) -> Track { Track { inner: f, diff --git a/third_party/rust/tokio/src/runtime/tests/loom_queue.rs b/third_party/rust/tokio/src/runtime/tests/loom_queue.rs index b5f78d7ebe75..b60e039b9a67 100644 --- a/third_party/rust/tokio/src/runtime/tests/loom_queue.rs +++ b/third_party/rust/tokio/src/runtime/tests/loom_queue.rs @@ -1,23 +1,27 @@ -use crate::runtime::blocking::NoopSchedule; -use crate::runtime::task::Inject; -use crate::runtime::{queue, MetricsBatch}; +use crate::runtime::scheduler::multi_thread::{queue, Stats}; +use crate::runtime::tests::NoopSchedule; use loom::thread; +use std::cell::RefCell; + +fn new_stats() -> Stats { + Stats::new(&crate::runtime::WorkerMetrics::new()) +} #[test] fn basic() { loom::model(|| { let (steal, mut local) = queue::local(); - let inject = Inject::new(); - let mut metrics = MetricsBatch::new(); + let inject = RefCell::new(vec![]); + let mut stats = new_stats(); let th = thread::spawn(move || { - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); let (_, mut local) = queue::local(); let mut n = 0; for _ in 0..3 { - if steal.steal_into(&mut local, &mut metrics).is_some() { + if steal.steal_into(&mut local, &mut stats).is_some() { n += 1; } @@ -34,7 +38,7 @@ fn basic() { for _ in 0..2 { for _ in 0..2 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); } if local.pop().is_some() { @@ -43,16 +47,14 @@ fn basic() { // Push another task let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); while local.pop().is_some() { n += 1; } } - while inject.pop().is_some() { - n += 1; - } + n += inject.borrow_mut().drain(..).count(); n += th.join().unwrap(); @@ -64,15 +66,15 @@ fn basic() { fn steal_overflow() { loom::model(|| { let (steal, mut local) = queue::local(); - let inject = Inject::new(); - let mut metrics = MetricsBatch::new(); + let inject = RefCell::new(vec![]); + let mut stats = new_stats(); let th = thread::spawn(move || { - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); let (_, mut local) = queue::local(); let mut n = 0; - if steal.steal_into(&mut local, &mut metrics).is_some() { + if steal.steal_into(&mut local, &mut stats).is_some() { n += 1; } @@ -87,7 +89,7 @@ fn steal_overflow() { // push a task, pop a task let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); if local.pop().is_some() { n += 1; @@ -95,7 +97,7 @@ fn steal_overflow() { for _ in 0..6 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); } n += th.join().unwrap(); @@ -104,9 +106,7 @@ fn steal_overflow() { n += 1; } - while inject.pop().is_some() { - n += 1; - } + n += inject.borrow_mut().drain(..).count(); assert_eq!(7, n); }); @@ -117,10 +117,10 @@ fn multi_stealer() { const NUM_TASKS: usize = 5; fn steal_tasks(steal: queue::Steal) -> usize { - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); let (_, mut local) = queue::local(); - if steal.steal_into(&mut local, &mut metrics).is_none() { + if steal.steal_into(&mut local, &mut stats).is_none() { return 0; } @@ -135,13 +135,13 @@ fn multi_stealer() { loom::model(|| { let (steal, mut local) = queue::local(); - let inject = Inject::new(); - let mut metrics = MetricsBatch::new(); + let inject = RefCell::new(vec![]); + let mut stats = new_stats(); // Push work for _ in 0..NUM_TASKS { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); } let th1 = { @@ -157,9 +157,7 @@ fn multi_stealer() { n += 1; } - while inject.pop().is_some() { - n += 1; - } + n += inject.borrow_mut().drain(..).count(); n += th1.join().unwrap(); n += th2.join().unwrap(); @@ -171,25 +169,25 @@ fn multi_stealer() { #[test] fn chained_steal() { loom::model(|| { - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); let (s1, mut l1) = queue::local(); let (s2, mut l2) = queue::local(); - let inject = Inject::new(); + let inject = RefCell::new(vec![]); // Load up some tasks for _ in 0..4 { let (task, _) = super::unowned(async {}); - l1.push_back(task, &inject, &mut metrics); + l1.push_back_or_overflow(task, &inject, &mut stats); let (task, _) = super::unowned(async {}); - l2.push_back(task, &inject, &mut metrics); + l2.push_back_or_overflow(task, &inject, &mut stats); } // Spawn a task to steal from **our** queue let th = thread::spawn(move || { - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); let (_, mut local) = queue::local(); - s1.steal_into(&mut local, &mut metrics); + s1.steal_into(&mut local, &mut stats); while local.pop().is_some() {} }); @@ -197,12 +195,11 @@ fn chained_steal() { // Drain our tasks, then attempt to steal while l1.pop().is_some() {} - s2.steal_into(&mut l1, &mut metrics); + s2.steal_into(&mut l1, &mut stats); th.join().unwrap(); while l1.pop().is_some() {} while l2.pop().is_some() {} - while inject.pop().is_some() {} }); } diff --git a/third_party/rust/tokio/src/runtime/tests/loom_yield.rs b/third_party/rust/tokio/src/runtime/tests/loom_yield.rs new file mode 100644 index 000000000000..ba506e5a4083 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/tests/loom_yield.rs @@ -0,0 +1,37 @@ +use crate::runtime::park; +use crate::runtime::tests::loom_oneshot as oneshot; +use crate::runtime::{self, Runtime}; + +#[test] +fn yield_calls_park_before_scheduling_again() { + // Don't need to check all permutations + let mut loom = loom::model::Builder::default(); + loom.max_permutations = Some(1); + loom.check(|| { + let rt = mk_runtime(2); + let (tx, rx) = oneshot::channel::<()>(); + + rt.spawn(async { + let tid = loom::thread::current().id(); + let park_count = park::current_thread_park_count(); + + crate::task::yield_now().await; + + if tid == loom::thread::current().id() { + let new_park_count = park::current_thread_park_count(); + assert_eq!(park_count + 1, new_park_count); + } + + tx.send(()); + }); + + rx.recv(); + }); +} + +fn mk_runtime(num_threads: usize) -> Runtime { + runtime::Builder::new_multi_thread() + .worker_threads(num_threads) + .build() + .unwrap() +} diff --git a/third_party/rust/tokio/src/runtime/tests/mod.rs b/third_party/rust/tokio/src/runtime/tests/mod.rs index 4b49698a86a6..b12a76e268ed 100644 --- a/third_party/rust/tokio/src/runtime/tests/mod.rs +++ b/third_party/rust/tokio/src/runtime/tests/mod.rs @@ -1,8 +1,30 @@ +// Enable dead_code / unreachable_pub here. It has been disabled in lib.rs for +// other code when running loom tests. +#![cfg_attr(loom, warn(dead_code, unreachable_pub))] + +use self::noop_scheduler::NoopSchedule; use self::unowned_wrapper::unowned; +mod noop_scheduler { + use crate::runtime::task::{self, Task}; + + /// `task::Schedule` implementation that does nothing, for testing. + pub(crate) struct NoopSchedule; + + impl task::Schedule for NoopSchedule { + fn release(&self, _task: &Task) -> Option> { + None + } + + fn schedule(&self, _task: task::Notified) { + unreachable!(); + } + } +} + mod unowned_wrapper { - use crate::runtime::blocking::NoopSchedule; - use crate::runtime::task::{JoinHandle, Notified}; + use crate::runtime::task::{Id, JoinHandle, Notified}; + use crate::runtime::tests::NoopSchedule; #[cfg(all(tokio_unstable, feature = "tracing"))] pub(crate) fn unowned(task: T) -> (Notified, JoinHandle) @@ -13,7 +35,7 @@ mod unowned_wrapper { use tracing::Instrument; let span = tracing::trace_span!("test_span"); let task = task.instrument(span); - let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule); + let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule, Id::next()); (task.into_notified(), handle) } @@ -23,23 +45,29 @@ mod unowned_wrapper { T: std::future::Future + Send + 'static, T::Output: Send + 'static, { - let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule); + let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule, Id::next()); (task.into_notified(), handle) } } cfg_loom! { - mod loom_basic_scheduler; mod loom_blocking; + mod loom_current_thread_scheduler; mod loom_local; mod loom_oneshot; mod loom_pool; mod loom_queue; mod loom_shutdown_join; mod loom_join_set; + mod loom_yield; + + // Make sure debug assertions are enabled + #[cfg(not(debug_assertions))] + compiler_error!("these tests require debug assertions to be enabled"); } cfg_not_loom! { + mod inject; mod queue; #[cfg(not(miri))] diff --git a/third_party/rust/tokio/src/runtime/tests/queue.rs b/third_party/rust/tokio/src/runtime/tests/queue.rs index 0fd1e0c6d9e7..5df92b7a2915 100644 --- a/third_party/rust/tokio/src/runtime/tests/queue.rs +++ b/third_party/rust/tokio/src/runtime/tests/queue.rs @@ -1,18 +1,18 @@ -use crate::runtime::queue; -use crate::runtime::task::{self, Inject, Schedule, Task}; -use crate::runtime::MetricsBatch; +use crate::runtime::scheduler::multi_thread::{queue, Stats}; +use crate::runtime::task::{self, Schedule, Task}; +use std::cell::RefCell; use std::thread; use std::time::Duration; #[allow(unused)] macro_rules! assert_metrics { - ($metrics:ident, $field:ident == $v:expr) => {{ + ($stats:ident, $field:ident == $v:expr) => {{ use crate::runtime::WorkerMetrics; use std::sync::atomic::Ordering::Relaxed; let worker = WorkerMetrics::new(); - $metrics.submit(&worker); + $stats.submit(&worker); let expect = $v; let actual = worker.$field.load(Relaxed); @@ -21,46 +21,87 @@ macro_rules! assert_metrics { }}; } +fn new_stats() -> Stats { + use crate::runtime::WorkerMetrics; + Stats::new(&WorkerMetrics::new()) +} + #[test] -fn fits_256() { +fn fits_256_one_at_a_time() { let (_, mut local) = queue::local(); - let inject = Inject::new(); - let mut metrics = MetricsBatch::new(); + let inject = RefCell::new(vec![]); + let mut stats = new_stats(); for _ in 0..256 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); } cfg_metrics! { - assert_metrics!(metrics, overflow_count == 0); + assert_metrics!(stats, overflow_count == 0); } - assert!(inject.pop().is_none()); + assert!(inject.borrow_mut().pop().is_none()); while local.pop().is_some() {} } +#[test] +fn fits_256_all_at_once() { + let (_, mut local) = queue::local(); + + let mut tasks = (0..256) + .map(|_| super::unowned(async {}).0) + .collect::>(); + local.push_back(tasks.drain(..)); + + let mut i = 0; + while local.pop().is_some() { + i += 1; + } + + assert_eq!(i, 256); +} + +#[test] +fn fits_256_all_in_chunks() { + let (_, mut local) = queue::local(); + + let mut tasks = (0..256) + .map(|_| super::unowned(async {}).0) + .collect::>(); + + local.push_back(tasks.drain(..10)); + local.push_back(tasks.drain(..100)); + local.push_back(tasks.drain(..46)); + local.push_back(tasks.drain(..100)); + + let mut i = 0; + while local.pop().is_some() { + i += 1; + } + + assert_eq!(i, 256); +} + #[test] fn overflow() { let (_, mut local) = queue::local(); - let inject = Inject::new(); - let mut metrics = MetricsBatch::new(); + let inject = RefCell::new(vec![]); + let mut stats = new_stats(); for _ in 0..257 { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); } cfg_metrics! { - assert_metrics!(metrics, overflow_count == 1); + assert_metrics!(stats, overflow_count == 1); } let mut n = 0; - while inject.pop().is_some() { - n += 1; - } + n += inject.borrow_mut().drain(..).count(); while local.pop().is_some() { n += 1; @@ -71,21 +112,21 @@ fn overflow() { #[test] fn steal_batch() { - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); let (steal1, mut local1) = queue::local(); let (_, mut local2) = queue::local(); - let inject = Inject::new(); + let inject = RefCell::new(vec![]); for _ in 0..4 { let (task, _) = super::unowned(async {}); - local1.push_back(task, &inject, &mut metrics); + local1.push_back_or_overflow(task, &inject, &mut stats); } - assert!(steal1.steal_into(&mut local2, &mut metrics).is_some()); + assert!(steal1.steal_into(&mut local2, &mut stats).is_some()); cfg_metrics! { - assert_metrics!(metrics, steal_count == 2); + assert_metrics!(stats, steal_count == 2); } for _ in 0..1 { @@ -111,25 +152,25 @@ const fn normal_or_miri(normal: usize, miri: usize) -> usize { #[test] fn stress1() { - const NUM_ITER: usize = 1; + const NUM_ITER: usize = 5; const NUM_STEAL: usize = normal_or_miri(1_000, 10); const NUM_LOCAL: usize = normal_or_miri(1_000, 10); const NUM_PUSH: usize = normal_or_miri(500, 10); const NUM_POP: usize = normal_or_miri(250, 10); - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); for _ in 0..NUM_ITER { let (steal, mut local) = queue::local(); - let inject = Inject::new(); + let inject = RefCell::new(vec![]); let th = thread::spawn(move || { - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); let (_, mut local) = queue::local(); let mut n = 0; for _ in 0..NUM_STEAL { - if steal.steal_into(&mut local, &mut metrics).is_some() { + if steal.steal_into(&mut local, &mut stats).is_some() { n += 1; } @@ -141,7 +182,7 @@ fn stress1() { } cfg_metrics! { - assert_metrics!(metrics, steal_count == n as _); + assert_metrics!(stats, steal_count == n as _); } n @@ -152,7 +193,7 @@ fn stress1() { for _ in 0..NUM_LOCAL { for _ in 0..NUM_PUSH { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); } for _ in 0..NUM_POP { @@ -164,9 +205,7 @@ fn stress1() { } } - while inject.pop().is_some() { - n += 1; - } + n += inject.borrow_mut().drain(..).count(); n += th.join().unwrap(); @@ -180,14 +219,14 @@ fn stress2() { const NUM_TASKS: usize = normal_or_miri(1_000_000, 50); const NUM_STEAL: usize = normal_or_miri(1_000, 10); - let mut metrics = MetricsBatch::new(); + let mut stats = new_stats(); for _ in 0..NUM_ITER { let (steal, mut local) = queue::local(); - let inject = Inject::new(); + let inject = RefCell::new(vec![]); let th = thread::spawn(move || { - let mut stats = MetricsBatch::new(); + let mut stats = new_stats(); let (_, mut local) = queue::local(); let mut n = 0; @@ -210,15 +249,13 @@ fn stress2() { for i in 0..NUM_TASKS { let (task, _) = super::unowned(async {}); - local.push_back(task, &inject, &mut metrics); + local.push_back_or_overflow(task, &inject, &mut stats); if i % 128 == 0 && local.pop().is_some() { num_pop += 1; } - while inject.pop().is_some() { - num_pop += 1; - } + num_pop += inject.borrow_mut().drain(..).count(); } num_pop += th.join().unwrap(); @@ -227,9 +264,7 @@ fn stress2() { num_pop += 1; } - while inject.pop().is_some() { - num_pop += 1; - } + num_pop += inject.borrow_mut().drain(..).count(); assert_eq!(num_pop, NUM_TASKS); } diff --git a/third_party/rust/tokio/src/runtime/tests/task.rs b/third_party/rust/tokio/src/runtime/tests/task.rs index 04e1b56e7774..a79c0f50d150 100644 --- a/third_party/rust/tokio/src/runtime/tests/task.rs +++ b/third_party/rust/tokio/src/runtime/tests/task.rs @@ -1,5 +1,5 @@ -use crate::runtime::blocking::NoopSchedule; -use crate::runtime::task::{self, unowned, JoinHandle, OwnedTasks, Schedule, Task}; +use crate::runtime::task::{self, unowned, Id, JoinHandle, OwnedTasks, Schedule, Task}; +use crate::runtime::tests::NoopSchedule; use crate::util::TryLock; use std::collections::VecDeque; @@ -55,6 +55,7 @@ fn create_drop1() { unreachable!() }, NoopSchedule, + Id::next(), ); drop(notified); handle.assert_not_dropped(); @@ -71,6 +72,7 @@ fn create_drop2() { unreachable!() }, NoopSchedule, + Id::next(), ); drop(join); handle.assert_not_dropped(); @@ -78,6 +80,46 @@ fn create_drop2() { handle.assert_dropped(); } +#[test] +fn drop_abort_handle1() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + Id::next(), + ); + let abort = join.abort_handle(); + drop(join); + handle.assert_not_dropped(); + drop(notified); + handle.assert_not_dropped(); + drop(abort); + handle.assert_dropped(); +} + +#[test] +fn drop_abort_handle2() { + let (ad, handle) = AssertDrop::new(); + let (notified, join) = unowned( + async { + drop(ad); + unreachable!() + }, + NoopSchedule, + Id::next(), + ); + let abort = join.abort_handle(); + drop(notified); + handle.assert_not_dropped(); + drop(abort); + handle.assert_not_dropped(); + drop(join); + handle.assert_dropped(); +} + // Shutting down through Notified works #[test] fn create_shutdown1() { @@ -88,6 +130,7 @@ fn create_shutdown1() { unreachable!() }, NoopSchedule, + Id::next(), ); drop(join); handle.assert_not_dropped(); @@ -104,6 +147,7 @@ fn create_shutdown2() { unreachable!() }, NoopSchedule, + Id::next(), ); handle.assert_not_dropped(); notified.shutdown(); @@ -113,7 +157,7 @@ fn create_shutdown2() { #[test] fn unowned_poll() { - let (task, _) = unowned(async {}, NoopSchedule); + let (task, _) = unowned(async {}, NoopSchedule, Id::next()); task.run(); } @@ -228,7 +272,7 @@ impl Runtime { T: 'static + Send + Future, T::Output: 'static + Send, { - let (handle, notified) = self.0.owned.bind(future, self.clone()); + let (handle, notified) = self.0.owned.bind(future, self.clone(), Id::next()); if let Some(notified) = notified { self.schedule(notified); diff --git a/third_party/rust/tokio/src/runtime/tests/task_combinations.rs b/third_party/rust/tokio/src/runtime/tests/task_combinations.rs index 76ce2330c2cf..73a20d976006 100644 --- a/third_party/rust/tokio/src/runtime/tests/task_combinations.rs +++ b/third_party/rust/tokio/src/runtime/tests/task_combinations.rs @@ -1,8 +1,10 @@ +use std::fmt; use std::future::Future; use std::panic; use std::pin::Pin; use std::task::{Context, Poll}; +use crate::runtime::task::AbortHandle; use crate::runtime::Builder; use crate::sync::oneshot; use crate::task::JoinHandle; @@ -56,6 +58,12 @@ enum CombiAbort { AbortedAfterConsumeOutput = 4, } +#[derive(Copy, Clone, Debug, PartialEq)] +enum CombiAbortSource { + JoinHandle, + AbortHandle, +} + #[test] fn test_combinations() { let mut rt = &[ @@ -90,6 +98,13 @@ fn test_combinations() { CombiAbort::AbortedAfterFinish, CombiAbort::AbortedAfterConsumeOutput, ]; + let ah = [ + None, + Some(CombiJoinHandle::DropImmediately), + Some(CombiJoinHandle::DropFirstPoll), + Some(CombiJoinHandle::DropAfterNoConsume), + Some(CombiJoinHandle::DropAfterConsume), + ]; for rt in rt.iter().copied() { for ls in ls.iter().copied() { @@ -98,7 +113,34 @@ fn test_combinations() { for ji in ji.iter().copied() { for jh in jh.iter().copied() { for abort in abort.iter().copied() { - test_combination(rt, ls, task, output, ji, jh, abort); + // abort via join handle --- abort handles + // may be dropped at any point + for ah in ah.iter().copied() { + test_combination( + rt, + ls, + task, + output, + ji, + jh, + ah, + abort, + CombiAbortSource::JoinHandle, + ); + } + // if aborting via AbortHandle, it will + // never be dropped. + test_combination( + rt, + ls, + task, + output, + ji, + jh, + None, + abort, + CombiAbortSource::AbortHandle, + ); } } } @@ -108,6 +150,9 @@ fn test_combinations() { } } +fn is_debug(_: &T) {} + +#[allow(clippy::too_many_arguments)] fn test_combination( rt: CombiRuntime, ls: CombiLocalSet, @@ -115,12 +160,24 @@ fn test_combination( output: CombiOutput, ji: CombiJoinInterest, jh: CombiJoinHandle, + ah: Option, abort: CombiAbort, + abort_src: CombiAbortSource, ) { - if (jh as usize) < (abort as usize) { - // drop before abort not possible - return; + match (abort_src, ah) { + (CombiAbortSource::JoinHandle, _) if (jh as usize) < (abort as usize) => { + // join handle dropped prior to abort + return; + } + (CombiAbortSource::AbortHandle, Some(_)) => { + // abort handle dropped, we can't abort through the + // abort handle + return; + } + + _ => {} } + if (task == CombiTask::PanicOnDrop) && (output == CombiOutput::PanicOnDrop) { // this causes double panic return; @@ -130,7 +187,15 @@ fn test_combination( return; } - println!("Runtime {:?}, LocalSet {:?}, Task {:?}, Output {:?}, JoinInterest {:?}, JoinHandle {:?}, Abort {:?}", rt, ls, task, output, ji, jh, abort); + is_debug(&rt); + is_debug(&ls); + is_debug(&task); + is_debug(&output); + is_debug(&ji); + is_debug(&jh); + is_debug(&ah); + is_debug(&abort); + is_debug(&abort_src); // A runtime optionally with a LocalSet struct Rt { @@ -282,8 +347,24 @@ fn test_combination( ); } + // If we are either aborting the task via an abort handle, or dropping via + // an abort handle, do that now. + let mut abort_handle = if ah.is_some() || abort_src == CombiAbortSource::AbortHandle { + handle.as_ref().map(JoinHandle::abort_handle) + } else { + None + }; + + let do_abort = |abort_handle: &mut Option, + join_handle: Option<&mut JoinHandle<_>>| { + match abort_src { + CombiAbortSource::AbortHandle => abort_handle.take().unwrap().abort(), + CombiAbortSource::JoinHandle => join_handle.unwrap().abort(), + } + }; + if abort == CombiAbort::AbortedImmediately { - handle.as_mut().unwrap().abort(); + do_abort(&mut abort_handle, handle.as_mut()); aborted = true; } if jh == CombiJoinHandle::DropImmediately { @@ -301,12 +382,15 @@ fn test_combination( } if abort == CombiAbort::AbortedFirstPoll { - handle.as_mut().unwrap().abort(); + do_abort(&mut abort_handle, handle.as_mut()); aborted = true; } if jh == CombiJoinHandle::DropFirstPoll { drop(handle.take().unwrap()); } + if ah == Some(CombiJoinHandle::DropFirstPoll) { + drop(abort_handle.take().unwrap()); + } // Signal the future that it can return now let _ = on_complete.send(()); @@ -318,23 +402,42 @@ fn test_combination( if abort == CombiAbort::AbortedAfterFinish { // Don't set aborted to true here as the task already finished - handle.as_mut().unwrap().abort(); + do_abort(&mut abort_handle, handle.as_mut()); } if jh == CombiJoinHandle::DropAfterNoConsume { - // The runtime will usually have dropped every ref-count at this point, - // in which case dropping the JoinHandle drops the output. - // - // (But it might race and still hold a ref-count) - let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + if ah == Some(CombiJoinHandle::DropAfterNoConsume) { drop(handle.take().unwrap()); - })); - if panic.is_err() { - assert!( - (output == CombiOutput::PanicOnDrop) - && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) - && !aborted, - "Dropping JoinHandle shouldn't panic here" - ); + // The runtime will usually have dropped every ref-count at this point, + // in which case dropping the AbortHandle drops the output. + // + // (But it might race and still hold a ref-count) + let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + drop(abort_handle.take().unwrap()); + })); + if panic.is_err() { + assert!( + (output == CombiOutput::PanicOnDrop) + && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) + && !aborted, + "Dropping AbortHandle shouldn't panic here" + ); + } + } else { + // The runtime will usually have dropped every ref-count at this point, + // in which case dropping the JoinHandle drops the output. + // + // (But it might race and still hold a ref-count) + let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| { + drop(handle.take().unwrap()); + })); + if panic.is_err() { + assert!( + (output == CombiOutput::PanicOnDrop) + && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop)) + && !aborted, + "Dropping JoinHandle shouldn't panic here" + ); + } } } @@ -362,11 +465,15 @@ fn test_combination( _ => unreachable!(), } - let handle = handle.take().unwrap(); + let mut handle = handle.take().unwrap(); if abort == CombiAbort::AbortedAfterConsumeOutput { - handle.abort(); + do_abort(&mut abort_handle, Some(&mut handle)); } drop(handle); + + if ah == Some(CombiJoinHandle::DropAfterConsume) { + drop(abort_handle.take()); + } } // The output should have been dropped now. Check whether the output diff --git a/third_party/rust/tokio/src/runtime/thread_id.rs b/third_party/rust/tokio/src/runtime/thread_id.rs new file mode 100644 index 000000000000..ef3928979634 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/thread_id.rs @@ -0,0 +1,31 @@ +use std::num::NonZeroU64; + +#[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)] +pub(crate) struct ThreadId(NonZeroU64); + +impl ThreadId { + pub(crate) fn next() -> Self { + use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64}; + + static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(0); + + let mut last = NEXT_ID.load(Relaxed); + loop { + let id = match last.checked_add(1) { + Some(id) => id, + None => exhausted(), + }; + + match NEXT_ID.compare_exchange_weak(last, id, Relaxed, Relaxed) { + Ok(_) => return ThreadId(NonZeroU64::new(id).unwrap()), + Err(id) => last = id, + } + } + } +} + +#[cold] +#[allow(dead_code)] +fn exhausted() -> ! { + panic!("failed to generate unique thread ID: bitspace exhausted") +} diff --git a/third_party/rust/tokio/src/runtime/thread_pool/mod.rs b/third_party/rust/tokio/src/runtime/thread_pool/mod.rs deleted file mode 100644 index d3f46517cb05..000000000000 --- a/third_party/rust/tokio/src/runtime/thread_pool/mod.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Threadpool - -mod idle; -use self::idle::Idle; - -mod worker; -pub(crate) use worker::Launch; - -pub(crate) use worker::block_in_place; - -use crate::loom::sync::Arc; -use crate::runtime::task::JoinHandle; -use crate::runtime::{Callback, Parker}; - -use std::fmt; -use std::future::Future; - -/// Work-stealing based thread pool for executing futures. -pub(crate) struct ThreadPool { - spawner: Spawner, -} - -/// Submits futures to the associated thread pool for execution. -/// -/// A `Spawner` instance is a handle to a single thread pool that allows the owner -/// of the handle to spawn futures onto the thread pool. -/// -/// The `Spawner` handle is *only* used for spawning new futures. It does not -/// impact the lifecycle of the thread pool in any way. The thread pool may -/// shut down while there are outstanding `Spawner` instances. -/// -/// `Spawner` instances are obtained by calling [`ThreadPool::spawner`]. -/// -/// [`ThreadPool::spawner`]: method@ThreadPool::spawner -#[derive(Clone)] -pub(crate) struct Spawner { - shared: Arc, -} - -// ===== impl ThreadPool ===== - -impl ThreadPool { - pub(crate) fn new( - size: usize, - parker: Parker, - before_park: Option, - after_unpark: Option, - ) -> (ThreadPool, Launch) { - let (shared, launch) = worker::create(size, parker, before_park, after_unpark); - let spawner = Spawner { shared }; - let thread_pool = ThreadPool { spawner }; - - (thread_pool, launch) - } - - /// Returns reference to `Spawner`. - /// - /// The `Spawner` handle can be cloned and enables spawning tasks from other - /// threads. - pub(crate) fn spawner(&self) -> &Spawner { - &self.spawner - } - - /// Blocks the current thread waiting for the future to complete. - /// - /// The future will execute on the current thread, but all spawned tasks - /// will be executed on the thread pool. - pub(crate) fn block_on(&self, future: F) -> F::Output - where - F: Future, - { - let mut enter = crate::runtime::enter(true); - enter.block_on(future).expect("failed to park thread") - } -} - -impl fmt::Debug for ThreadPool { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("ThreadPool").finish() - } -} - -impl Drop for ThreadPool { - fn drop(&mut self) { - self.spawner.shutdown(); - } -} - -// ==== impl Spawner ===== - -impl Spawner { - /// Spawns a future onto the thread pool - pub(crate) fn spawn(&self, future: F) -> JoinHandle - where - F: crate::future::Future + Send + 'static, - F::Output: Send + 'static, - { - worker::Shared::bind_new_task(&self.shared, future) - } - - pub(crate) fn shutdown(&mut self) { - self.shared.close(); - } -} - -cfg_metrics! { - use crate::runtime::{SchedulerMetrics, WorkerMetrics}; - - impl Spawner { - pub(crate) fn num_workers(&self) -> usize { - self.shared.worker_metrics.len() - } - - pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics { - &self.shared.scheduler_metrics - } - - pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics { - &self.shared.worker_metrics[worker] - } - - pub(crate) fn injection_queue_depth(&self) -> usize { - self.shared.injection_queue_depth() - } - - pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize { - self.shared.worker_local_queue_depth(worker) - } - } -} - -impl fmt::Debug for Spawner { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("Spawner").finish() - } -} diff --git a/third_party/rust/tokio/src/runtime/thread_pool/worker.rs b/third_party/rust/tokio/src/runtime/thread_pool/worker.rs deleted file mode 100644 index 7e4989701e5b..000000000000 --- a/third_party/rust/tokio/src/runtime/thread_pool/worker.rs +++ /dev/null @@ -1,848 +0,0 @@ -//! A scheduler is initialized with a fixed number of workers. Each worker is -//! driven by a thread. Each worker has a "core" which contains data such as the -//! run queue and other state. When `block_in_place` is called, the worker's -//! "core" is handed off to a new thread allowing the scheduler to continue to -//! make progress while the originating thread blocks. -//! -//! # Shutdown -//! -//! Shutting down the runtime involves the following steps: -//! -//! 1. The Shared::close method is called. This closes the inject queue and -//! OwnedTasks instance and wakes up all worker threads. -//! -//! 2. Each worker thread observes the close signal next time it runs -//! Core::maintenance by checking whether the inject queue is closed. -//! The Core::is_shutdown flag is set to true. -//! -//! 3. The worker thread calls `pre_shutdown` in parallel. Here, the worker -//! will keep removing tasks from OwnedTasks until it is empty. No new -//! tasks can be pushed to the OwnedTasks during or after this step as it -//! was closed in step 1. -//! -//! 5. The workers call Shared::shutdown to enter the single-threaded phase of -//! shutdown. These calls will push their core to Shared::shutdown_cores, -//! and the last thread to push its core will finish the shutdown procedure. -//! -//! 6. The local run queue of each core is emptied, then the inject queue is -//! emptied. -//! -//! At this point, shutdown has completed. It is not possible for any of the -//! collections to contain any tasks at this point, as each collection was -//! closed first, then emptied afterwards. -//! -//! ## Spawns during shutdown -//! -//! When spawning tasks during shutdown, there are two cases: -//! -//! * The spawner observes the OwnedTasks being open, and the inject queue is -//! closed. -//! * The spawner observes the OwnedTasks being closed and doesn't check the -//! inject queue. -//! -//! The first case can only happen if the OwnedTasks::bind call happens before -//! or during step 1 of shutdown. In this case, the runtime will clean up the -//! task in step 3 of shutdown. -//! -//! In the latter case, the task was not spawned and the task is immediately -//! cancelled by the spawner. -//! -//! The correctness of shutdown requires both the inject queue and OwnedTasks -//! collection to have a closed bit. With a close bit on only the inject queue, -//! spawning could run in to a situation where a task is successfully bound long -//! after the runtime has shut down. With a close bit on only the OwnedTasks, -//! the first spawning situation could result in the notification being pushed -//! to the inject queue after step 6 of shutdown, which would leave a task in -//! the inject queue indefinitely. This would be a ref-count cycle and a memory -//! leak. - -use crate::coop; -use crate::future::Future; -use crate::loom::rand::seed; -use crate::loom::sync::{Arc, Mutex}; -use crate::park::{Park, Unpark}; -use crate::runtime; -use crate::runtime::enter::EnterContext; -use crate::runtime::park::{Parker, Unparker}; -use crate::runtime::task::{Inject, JoinHandle, OwnedTasks}; -use crate::runtime::thread_pool::Idle; -use crate::runtime::{queue, task, Callback, MetricsBatch, SchedulerMetrics, WorkerMetrics}; -use crate::util::atomic_cell::AtomicCell; -use crate::util::FastRand; - -use std::cell::RefCell; -use std::time::Duration; - -/// A scheduler worker -pub(super) struct Worker { - /// Reference to shared state - shared: Arc, - - /// Index holding this worker's remote state - index: usize, - - /// Used to hand-off a worker's core to another thread. - core: AtomicCell, -} - -/// Core data -struct Core { - /// Used to schedule bookkeeping tasks every so often. - tick: u8, - - /// When a task is scheduled from a worker, it is stored in this slot. The - /// worker will check this slot for a task **before** checking the run - /// queue. This effectively results in the **last** scheduled task to be run - /// next (LIFO). This is an optimization for message passing patterns and - /// helps to reduce latency. - lifo_slot: Option, - - /// The worker-local run queue. - run_queue: queue::Local>, - - /// True if the worker is currently searching for more work. Searching - /// involves attempting to steal from other workers. - is_searching: bool, - - /// True if the scheduler is being shutdown - is_shutdown: bool, - - /// Parker - /// - /// Stored in an `Option` as the parker is added / removed to make the - /// borrow checker happy. - park: Option, - - /// Batching metrics so they can be submitted to RuntimeMetrics. - metrics: MetricsBatch, - - /// Fast random number generator. - rand: FastRand, -} - -/// State shared across all workers -pub(super) struct Shared { - /// Per-worker remote state. All other workers have access to this and is - /// how they communicate between each other. - remotes: Box<[Remote]>, - - /// Submits work to the scheduler while **not** currently on a worker thread. - inject: Inject>, - - /// Coordinates idle workers - idle: Idle, - - /// Collection of all active tasks spawned onto this executor. - owned: OwnedTasks>, - - /// Cores that have observed the shutdown signal - /// - /// The core is **not** placed back in the worker to avoid it from being - /// stolen by a thread that was spawned as part of `block_in_place`. - #[allow(clippy::vec_box)] // we're moving an already-boxed value - shutdown_cores: Mutex>>, - - /// Callback for a worker parking itself - before_park: Option, - /// Callback for a worker unparking itself - after_unpark: Option, - - /// Collects metrics from the runtime. - pub(super) scheduler_metrics: SchedulerMetrics, - - pub(super) worker_metrics: Box<[WorkerMetrics]>, -} - -/// Used to communicate with a worker from other threads. -struct Remote { - /// Steals tasks from this worker. - steal: queue::Steal>, - - /// Unparks the associated worker thread - unpark: Unparker, -} - -/// Thread-local context -struct Context { - /// Worker - worker: Arc, - - /// Core data - core: RefCell>>, -} - -/// Starts the workers -pub(crate) struct Launch(Vec>); - -/// Running a task may consume the core. If the core is still available when -/// running the task completes, it is returned. Otherwise, the worker will need -/// to stop processing. -type RunResult = Result, ()>; - -/// A task handle -type Task = task::Task>; - -/// A notified task handle -type Notified = task::Notified>; - -// Tracks thread-local state -scoped_thread_local!(static CURRENT: Context); - -pub(super) fn create( - size: usize, - park: Parker, - before_park: Option, - after_unpark: Option, -) -> (Arc, Launch) { - let mut cores = vec![]; - let mut remotes = vec![]; - let mut worker_metrics = vec![]; - - // Create the local queues - for _ in 0..size { - let (steal, run_queue) = queue::local(); - - let park = park.clone(); - let unpark = park.unpark(); - - cores.push(Box::new(Core { - tick: 0, - lifo_slot: None, - run_queue, - is_searching: false, - is_shutdown: false, - park: Some(park), - metrics: MetricsBatch::new(), - rand: FastRand::new(seed()), - })); - - remotes.push(Remote { steal, unpark }); - worker_metrics.push(WorkerMetrics::new()); - } - - let shared = Arc::new(Shared { - remotes: remotes.into_boxed_slice(), - inject: Inject::new(), - idle: Idle::new(size), - owned: OwnedTasks::new(), - shutdown_cores: Mutex::new(vec![]), - before_park, - after_unpark, - scheduler_metrics: SchedulerMetrics::new(), - worker_metrics: worker_metrics.into_boxed_slice(), - }); - - let mut launch = Launch(vec![]); - - for (index, core) in cores.drain(..).enumerate() { - launch.0.push(Arc::new(Worker { - shared: shared.clone(), - index, - core: AtomicCell::new(Some(core)), - })); - } - - (shared, launch) -} - -pub(crate) fn block_in_place(f: F) -> R -where - F: FnOnce() -> R, -{ - // Try to steal the worker core back - struct Reset(coop::Budget); - - impl Drop for Reset { - fn drop(&mut self) { - CURRENT.with(|maybe_cx| { - if let Some(cx) = maybe_cx { - let core = cx.worker.core.take(); - let mut cx_core = cx.core.borrow_mut(); - assert!(cx_core.is_none()); - *cx_core = core; - - // Reset the task budget as we are re-entering the - // runtime. - coop::set(self.0); - } - }); - } - } - - let mut had_entered = false; - - CURRENT.with(|maybe_cx| { - match (crate::runtime::enter::context(), maybe_cx.is_some()) { - (EnterContext::Entered { .. }, true) => { - // We are on a thread pool runtime thread, so we just need to - // set up blocking. - had_entered = true; - } - (EnterContext::Entered { allow_blocking }, false) => { - // We are on an executor, but _not_ on the thread pool. That is - // _only_ okay if we are in a thread pool runtime's block_on - // method: - if allow_blocking { - had_entered = true; - return; - } else { - // This probably means we are on the basic_scheduler or in a - // LocalSet, where it is _not_ okay to block. - panic!("can call blocking only when running on the multi-threaded runtime"); - } - } - (EnterContext::NotEntered, true) => { - // This is a nested call to block_in_place (we already exited). - // All the necessary setup has already been done. - return; - } - (EnterContext::NotEntered, false) => { - // We are outside of the tokio runtime, so blocking is fine. - // We can also skip all of the thread pool blocking setup steps. - return; - } - } - - let cx = maybe_cx.expect("no .is_some() == false cases above should lead here"); - - // Get the worker core. If none is set, then blocking is fine! - let core = match cx.core.borrow_mut().take() { - Some(core) => core, - None => return, - }; - - // The parker should be set here - assert!(core.park.is_some()); - - // In order to block, the core must be sent to another thread for - // execution. - // - // First, move the core back into the worker's shared core slot. - cx.worker.core.set(core); - - // Next, clone the worker handle and send it to a new thread for - // processing. - // - // Once the blocking task is done executing, we will attempt to - // steal the core back. - let worker = cx.worker.clone(); - runtime::spawn_blocking(move || run(worker)); - }); - - if had_entered { - // Unset the current task's budget. Blocking sections are not - // constrained by task budgets. - let _reset = Reset(coop::stop()); - - crate::runtime::enter::exit(f) - } else { - f() - } -} - -/// After how many ticks is the global queue polled. This helps to ensure -/// fairness. -/// -/// The number is fairly arbitrary. I believe this value was copied from golang. -const GLOBAL_POLL_INTERVAL: u8 = 61; - -impl Launch { - pub(crate) fn launch(mut self) { - for worker in self.0.drain(..) { - runtime::spawn_blocking(move || run(worker)); - } - } -} - -fn run(worker: Arc) { - // Acquire a core. If this fails, then another thread is running this - // worker and there is nothing further to do. - let core = match worker.core.take() { - Some(core) => core, - None => return, - }; - - // Set the worker context. - let cx = Context { - worker, - core: RefCell::new(None), - }; - - let _enter = crate::runtime::enter(true); - - CURRENT.set(&cx, || { - // This should always be an error. It only returns a `Result` to support - // using `?` to short circuit. - assert!(cx.run(core).is_err()); - }); -} - -impl Context { - fn run(&self, mut core: Box) -> RunResult { - while !core.is_shutdown { - // Increment the tick - core.tick(); - - // Run maintenance, if needed - core = self.maintenance(core); - - // First, check work available to the current worker. - if let Some(task) = core.next_task(&self.worker) { - core = self.run_task(task, core)?; - continue; - } - - // There is no more **local** work to process, try to steal work - // from other workers. - if let Some(task) = core.steal_work(&self.worker) { - core = self.run_task(task, core)?; - } else { - // Wait for work - core = self.park(core); - } - } - - core.pre_shutdown(&self.worker); - - // Signal shutdown - self.worker.shared.shutdown(core); - Err(()) - } - - fn run_task(&self, task: Notified, mut core: Box) -> RunResult { - let task = self.worker.shared.owned.assert_owner(task); - - // Make sure the worker is not in the **searching** state. This enables - // another idle worker to try to steal work. - core.transition_from_searching(&self.worker); - - // Make the core available to the runtime context - core.metrics.incr_poll_count(); - *self.core.borrow_mut() = Some(core); - - // Run the task - coop::budget(|| { - task.run(); - - // As long as there is budget remaining and a task exists in the - // `lifo_slot`, then keep running. - loop { - // Check if we still have the core. If not, the core was stolen - // by another worker. - let mut core = match self.core.borrow_mut().take() { - Some(core) => core, - None => return Err(()), - }; - - // Check for a task in the LIFO slot - let task = match core.lifo_slot.take() { - Some(task) => task, - None => return Ok(core), - }; - - if coop::has_budget_remaining() { - // Run the LIFO task, then loop - core.metrics.incr_poll_count(); - *self.core.borrow_mut() = Some(core); - let task = self.worker.shared.owned.assert_owner(task); - task.run(); - } else { - // Not enough budget left to run the LIFO task, push it to - // the back of the queue and return. - core.run_queue - .push_back(task, self.worker.inject(), &mut core.metrics); - return Ok(core); - } - } - }) - } - - fn maintenance(&self, mut core: Box) -> Box { - if core.tick % GLOBAL_POLL_INTERVAL == 0 { - // Call `park` with a 0 timeout. This enables the I/O driver, timer, ... - // to run without actually putting the thread to sleep. - core = self.park_timeout(core, Some(Duration::from_millis(0))); - - // Run regularly scheduled maintenance - core.maintenance(&self.worker); - } - - core - } - - fn park(&self, mut core: Box) -> Box { - if let Some(f) = &self.worker.shared.before_park { - f(); - } - - if core.transition_to_parked(&self.worker) { - while !core.is_shutdown { - core.metrics.about_to_park(); - core = self.park_timeout(core, None); - core.metrics.returned_from_park(); - - // Run regularly scheduled maintenance - core.maintenance(&self.worker); - - if core.transition_from_parked(&self.worker) { - break; - } - } - } - - if let Some(f) = &self.worker.shared.after_unpark { - f(); - } - core - } - - fn park_timeout(&self, mut core: Box, duration: Option) -> Box { - // Take the parker out of core - let mut park = core.park.take().expect("park missing"); - - // Store `core` in context - *self.core.borrow_mut() = Some(core); - - // Park thread - if let Some(timeout) = duration { - park.park_timeout(timeout).expect("park failed"); - } else { - park.park().expect("park failed"); - } - - // Remove `core` from context - core = self.core.borrow_mut().take().expect("core missing"); - - // Place `park` back in `core` - core.park = Some(park); - - // If there are tasks available to steal, but this worker is not - // looking for tasks to steal, notify another worker. - if !core.is_searching && core.run_queue.is_stealable() { - self.worker.shared.notify_parked(); - } - - core - } -} - -impl Core { - /// Increment the tick - fn tick(&mut self) { - self.tick = self.tick.wrapping_add(1); - } - - /// Return the next notified task available to this worker. - fn next_task(&mut self, worker: &Worker) -> Option { - if self.tick % GLOBAL_POLL_INTERVAL == 0 { - worker.inject().pop().or_else(|| self.next_local_task()) - } else { - self.next_local_task().or_else(|| worker.inject().pop()) - } - } - - fn next_local_task(&mut self) -> Option { - self.lifo_slot.take().or_else(|| self.run_queue.pop()) - } - - fn steal_work(&mut self, worker: &Worker) -> Option { - if !self.transition_to_searching(worker) { - return None; - } - - let num = worker.shared.remotes.len(); - // Start from a random worker - let start = self.rand.fastrand_n(num as u32) as usize; - - for i in 0..num { - let i = (start + i) % num; - - // Don't steal from ourself! We know we don't have work. - if i == worker.index { - continue; - } - - let target = &worker.shared.remotes[i]; - if let Some(task) = target - .steal - .steal_into(&mut self.run_queue, &mut self.metrics) - { - return Some(task); - } - } - - // Fallback on checking the global queue - worker.shared.inject.pop() - } - - fn transition_to_searching(&mut self, worker: &Worker) -> bool { - if !self.is_searching { - self.is_searching = worker.shared.idle.transition_worker_to_searching(); - } - - self.is_searching - } - - fn transition_from_searching(&mut self, worker: &Worker) { - if !self.is_searching { - return; - } - - self.is_searching = false; - worker.shared.transition_worker_from_searching(); - } - - /// Prepares the worker state for parking. - /// - /// Returns true if the transition happend, false if there is work to do first. - fn transition_to_parked(&mut self, worker: &Worker) -> bool { - // Workers should not park if they have work to do - if self.lifo_slot.is_some() || self.run_queue.has_tasks() { - return false; - } - - // When the final worker transitions **out** of searching to parked, it - // must check all the queues one last time in case work materialized - // between the last work scan and transitioning out of searching. - let is_last_searcher = worker - .shared - .idle - .transition_worker_to_parked(worker.index, self.is_searching); - - // The worker is no longer searching. Setting this is the local cache - // only. - self.is_searching = false; - - if is_last_searcher { - worker.shared.notify_if_work_pending(); - } - - true - } - - /// Returns `true` if the transition happened. - fn transition_from_parked(&mut self, worker: &Worker) -> bool { - // If a task is in the lifo slot, then we must unpark regardless of - // being notified - if self.lifo_slot.is_some() { - // When a worker wakes, it should only transition to the "searching" - // state when the wake originates from another worker *or* a new task - // is pushed. We do *not* want the worker to transition to "searching" - // when it wakes when the I/O driver receives new events. - self.is_searching = !worker.shared.idle.unpark_worker_by_id(worker.index); - return true; - } - - if worker.shared.idle.is_parked(worker.index) { - return false; - } - - // When unparked, the worker is in the searching state. - self.is_searching = true; - true - } - - /// Runs maintenance work such as checking the pool's state. - fn maintenance(&mut self, worker: &Worker) { - self.metrics - .submit(&worker.shared.worker_metrics[worker.index]); - - if !self.is_shutdown { - // Check if the scheduler has been shutdown - self.is_shutdown = worker.inject().is_closed(); - } - } - - /// Signals all tasks to shut down, and waits for them to complete. Must run - /// before we enter the single-threaded phase of shutdown processing. - fn pre_shutdown(&mut self, worker: &Worker) { - // Signal to all tasks to shut down. - worker.shared.owned.close_and_shutdown_all(); - - self.metrics - .submit(&worker.shared.worker_metrics[worker.index]); - } - - /// Shuts down the core. - fn shutdown(&mut self) { - // Take the core - let mut park = self.park.take().expect("park missing"); - - // Drain the queue - while self.next_local_task().is_some() {} - - park.shutdown(); - } -} - -impl Worker { - /// Returns a reference to the scheduler's injection queue. - fn inject(&self) -> &Inject> { - &self.shared.inject - } -} - -impl task::Schedule for Arc { - fn release(&self, task: &Task) -> Option { - self.owned.remove(task) - } - - fn schedule(&self, task: Notified) { - (**self).schedule(task, false); - } - - fn yield_now(&self, task: Notified) { - (**self).schedule(task, true); - } -} - -impl Shared { - pub(super) fn bind_new_task(me: &Arc, future: T) -> JoinHandle - where - T: Future + Send + 'static, - T::Output: Send + 'static, - { - let (handle, notified) = me.owned.bind(future, me.clone()); - - if let Some(notified) = notified { - me.schedule(notified, false); - } - - handle - } - - pub(super) fn schedule(&self, task: Notified, is_yield: bool) { - CURRENT.with(|maybe_cx| { - if let Some(cx) = maybe_cx { - // Make sure the task is part of the **current** scheduler. - if self.ptr_eq(&cx.worker.shared) { - // And the current thread still holds a core - if let Some(core) = cx.core.borrow_mut().as_mut() { - self.schedule_local(core, task, is_yield); - return; - } - } - } - - // Otherwise, use the inject queue. - self.inject.push(task); - self.scheduler_metrics.inc_remote_schedule_count(); - self.notify_parked(); - }) - } - - fn schedule_local(&self, core: &mut Core, task: Notified, is_yield: bool) { - core.metrics.inc_local_schedule_count(); - - // Spawning from the worker thread. If scheduling a "yield" then the - // task must always be pushed to the back of the queue, enabling other - // tasks to be executed. If **not** a yield, then there is more - // flexibility and the task may go to the front of the queue. - let should_notify = if is_yield { - core.run_queue - .push_back(task, &self.inject, &mut core.metrics); - true - } else { - // Push to the LIFO slot - let prev = core.lifo_slot.take(); - let ret = prev.is_some(); - - if let Some(prev) = prev { - core.run_queue - .push_back(prev, &self.inject, &mut core.metrics); - } - - core.lifo_slot = Some(task); - - ret - }; - - // Only notify if not currently parked. If `park` is `None`, then the - // scheduling is from a resource driver. As notifications often come in - // batches, the notification is delayed until the park is complete. - if should_notify && core.park.is_some() { - self.notify_parked(); - } - } - - pub(super) fn close(&self) { - if self.inject.close() { - self.notify_all(); - } - } - - fn notify_parked(&self) { - if let Some(index) = self.idle.worker_to_notify() { - self.remotes[index].unpark.unpark(); - } - } - - fn notify_all(&self) { - for remote in &self.remotes[..] { - remote.unpark.unpark(); - } - } - - fn notify_if_work_pending(&self) { - for remote in &self.remotes[..] { - if !remote.steal.is_empty() { - self.notify_parked(); - return; - } - } - - if !self.inject.is_empty() { - self.notify_parked(); - } - } - - fn transition_worker_from_searching(&self) { - if self.idle.transition_worker_from_searching() { - // We are the final searching worker. Because work was found, we - // need to notify another worker. - self.notify_parked(); - } - } - - /// Signals that a worker has observed the shutdown signal and has replaced - /// its core back into its handle. - /// - /// If all workers have reached this point, the final cleanup is performed. - fn shutdown(&self, core: Box) { - let mut cores = self.shutdown_cores.lock(); - cores.push(core); - - if cores.len() != self.remotes.len() { - return; - } - - debug_assert!(self.owned.is_empty()); - - for mut core in cores.drain(..) { - core.shutdown(); - } - - // Drain the injection queue - // - // We already shut down every task, so we can simply drop the tasks. - while let Some(task) = self.inject.pop() { - drop(task); - } - } - - fn ptr_eq(&self, other: &Shared) -> bool { - std::ptr::eq(self, other) - } -} - -cfg_metrics! { - impl Shared { - pub(super) fn injection_queue_depth(&self) -> usize { - self.inject.len() - } - - pub(super) fn worker_local_queue_depth(&self, worker: usize) -> usize { - self.remotes[worker].steal.len() - } - } -} diff --git a/third_party/rust/tokio/src/time/driver/entry.rs b/third_party/rust/tokio/src/runtime/time/entry.rs similarity index 87% rename from third_party/rust/tokio/src/time/driver/entry.rs rename to third_party/rust/tokio/src/runtime/time/entry.rs index f0ea898e1202..798d3c11eb85 100644 --- a/third_party/rust/tokio/src/time/driver/entry.rs +++ b/third_party/rust/tokio/src/runtime/time/entry.rs @@ -36,7 +36,7 @@ //! point than it was originally scheduled for. //! //! This is accomplished by lazily rescheduling timers. That is, we update the -//! state field field with the true expiration of the timer from the holder of +//! state field with the true expiration of the timer from the holder of //! the [`TimerEntry`]. When the driver services timers (ie, whenever it's //! walking lists of timers), it checks this "true when" value, and reschedules //! based on it. @@ -58,12 +58,11 @@ use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::AtomicU64; use crate::loom::sync::atomic::Ordering; +use crate::runtime::scheduler; use crate::sync::AtomicWaker; use crate::time::Instant; use crate::util::linked_list; -use super::Handle; - use std::cell::UnsafeCell as StdUnsafeCell; use std::task::{Context, Poll, Waker}; use std::{marker::PhantomPinned, pin::Pin, ptr::NonNull}; @@ -73,6 +72,10 @@ type TimerResult = Result<(), crate::time::error::Error>; const STATE_DEREGISTERED: u64 = u64::MAX; const STATE_PENDING_FIRE: u64 = STATE_DEREGISTERED - 1; const STATE_MIN_VALUE: u64 = STATE_PENDING_FIRE; +/// The largest safe integer to use for ticks. +/// +/// This value should be updated if any other signal values are added above. +pub(super) const MAX_SAFE_MILLIS_DURATION: u64 = u64::MAX - 2; /// This structure holds the current shared state of the timer - its scheduled /// time (if registered), or otherwise the result of the timer completing, as @@ -95,7 +98,7 @@ pub(super) struct StateCell { /// without holding the driver lock is undefined behavior. result: UnsafeCell, /// The currently-registered waker - waker: CachePadded, + waker: AtomicWaker, } impl Default for StateCell { @@ -115,7 +118,7 @@ impl StateCell { Self { state: AtomicU64::new(STATE_DEREGISTERED), result: UnsafeCell::new(Ok(())), - waker: CachePadded(AtomicWaker::new()), + waker: AtomicWaker::new(), } } @@ -127,7 +130,7 @@ impl StateCell { fn when(&self) -> Option { let cur_state = self.state.load(Ordering::Relaxed); - if cur_state == u64::MAX { + if cur_state == STATE_DEREGISTERED { None } else { Some(cur_state) @@ -140,7 +143,7 @@ impl StateCell { // We must register first. This ensures that either `fire` will // observe the new waker, or we will observe a racing fire to have set // the state, or both. - self.waker.0.register_by_ref(waker); + self.waker.register_by_ref(waker); self.read_state() } @@ -173,7 +176,12 @@ impl StateCell { let mut cur_state = self.state.load(Ordering::Relaxed); loop { - debug_assert!(cur_state < STATE_MIN_VALUE); + // improve the error message for things like + // https://github.com/tokio-rs/tokio/issues/3675 + assert!( + cur_state < STATE_MIN_VALUE, + "mark_pending called when the timer entry is in an invalid state" + ); if cur_state > not_after { break Err(cur_state); @@ -223,7 +231,7 @@ impl StateCell { self.state.store(STATE_DEREGISTERED, Ordering::Release); - self.waker.0.take_waker() + self.waker.take_waker() } /// Marks the timer as registered (poll will return None) and sets the @@ -283,10 +291,10 @@ impl StateCell { /// timer. As this participates in intrusive data structures, it must be pinned /// before polling. #[derive(Debug)] -pub(super) struct TimerEntry { - /// Arc reference to the driver. We can only free the driver after +pub(crate) struct TimerEntry { + /// Arc reference to the runtime handle. We can only free the driver after /// deregistering everything from their respective timer wheels. - driver: Handle, + driver: scheduler::Handle, /// Shared inner structure; this is part of an intrusive linked list, and /// therefore other references can exist to it while mutable references to /// Entry exist. @@ -294,9 +302,11 @@ pub(super) struct TimerEntry { /// This is manipulated only under the inner mutex. TODO: Can we use loom /// cells for this? inner: StdUnsafeCell, - /// Initial deadline for the timer. This is used to register on the first + /// Deadline for the timer. This is used to register on the first /// poll, as we can't register prior to being pinned. - initial_deadline: Option, + deadline: Instant, + /// Whether the deadline has been registered. + registered: bool, /// Ensure the type is !Unpin _m: std::marker::PhantomPinned, } @@ -325,11 +335,20 @@ pub(super) type EntryList = crate::util::linked_list::LinkedList, + /// A link within the doubly-linked list of timers on a particular level and + /// slot. Valid only if state is equal to Registered. + /// + /// Only accessed under the entry lock. + pointers: linked_list::Pointers, + + /// The expiration time for which this entry is currently registered. + /// Generally owned by the driver, but is accessed by the entry when not + /// registered. + cached_when: AtomicU64, + + /// The true expiration time. Set by the timer future, read by the driver. + true_when: AtomicU64, /// Current state. This records whether the timer entry is currently under /// the ownership of the driver, and if not, its current state (not @@ -339,11 +358,34 @@ pub(crate) struct TimerShared { _p: PhantomPinned, } +unsafe impl Send for TimerShared {} +unsafe impl Sync for TimerShared {} + +impl std::fmt::Debug for TimerShared { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TimerShared") + .field("when", &self.true_when.load(Ordering::Relaxed)) + .field("cached_when", &self.cached_when.load(Ordering::Relaxed)) + .field("state", &self.state) + .finish() + } +} + +generate_addr_of_methods! { + impl<> TimerShared { + unsafe fn addr_of_pointers(self: NonNull) -> NonNull> { + &self.pointers + } + } +} + impl TimerShared { pub(super) fn new() -> Self { Self { + cached_when: AtomicU64::new(0), + true_when: AtomicU64::new(0), + pointers: linked_list::Pointers::new(), state: StateCell::default(), - driver_state: CachePadded(TimerSharedPadded::new()), _p: PhantomPinned, } } @@ -351,7 +393,7 @@ impl TimerShared { /// Gets the cached time-of-expiration value. pub(super) fn cached_when(&self) -> u64 { // Cached-when is only accessed under the driver lock, so we can use relaxed - self.driver_state.0.cached_when.load(Ordering::Relaxed) + self.cached_when.load(Ordering::Relaxed) } /// Gets the true time-of-expiration value, and copies it into the cached @@ -362,10 +404,7 @@ impl TimerShared { pub(super) unsafe fn sync_when(&self) -> u64 { let true_when = self.true_when(); - self.driver_state - .0 - .cached_when - .store(true_when, Ordering::Relaxed); + self.cached_when.store(true_when, Ordering::Relaxed); true_when } @@ -375,10 +414,7 @@ impl TimerShared { /// SAFETY: Must be called with the driver lock held, and when this entry is /// not in any timer wheel lists. unsafe fn set_cached_when(&self, when: u64) { - self.driver_state - .0 - .cached_when - .store(when, Ordering::Relaxed); + self.cached_when.store(when, Ordering::Relaxed); } /// Returns the true time-of-expiration value, with relaxed memory ordering. @@ -393,7 +429,7 @@ impl TimerShared { /// in the timer wheel. pub(super) unsafe fn set_expiration(&self, t: u64) { self.state.set_expiration(t); - self.driver_state.0.cached_when.store(t, Ordering::Relaxed); + self.cached_when.store(t, Ordering::Relaxed); } /// Sets the true time-of-expiration only if it is after the current. @@ -417,49 +453,6 @@ impl TimerShared { } } -/// Additional shared state between the driver and the timer which is cache -/// padded. This contains the information that the driver thread accesses most -/// frequently to minimize contention. In particular, we move it away from the -/// waker, as the waker is updated on every poll. -#[repr(C)] // required by `link_list::Link` impl -struct TimerSharedPadded { - /// A link within the doubly-linked list of timers on a particular level and - /// slot. Valid only if state is equal to Registered. - /// - /// Only accessed under the entry lock. - pointers: linked_list::Pointers, - - /// The expiration time for which this entry is currently registered. - /// Generally owned by the driver, but is accessed by the entry when not - /// registered. - cached_when: AtomicU64, - - /// The true expiration time. Set by the timer future, read by the driver. - true_when: AtomicU64, -} - -impl std::fmt::Debug for TimerSharedPadded { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TimerSharedPadded") - .field("when", &self.true_when.load(Ordering::Relaxed)) - .field("cached_when", &self.cached_when.load(Ordering::Relaxed)) - .finish() - } -} - -impl TimerSharedPadded { - fn new() -> Self { - Self { - cached_when: AtomicU64::new(0), - true_when: AtomicU64::new(0), - pointers: linked_list::Pointers::new(), - } - } -} - -unsafe impl Send for TimerShared {} -unsafe impl Sync for TimerShared {} - unsafe impl linked_list::Link for TimerShared { type Handle = TimerHandle; @@ -476,20 +469,25 @@ unsafe impl linked_list::Link for TimerShared { unsafe fn pointers( target: NonNull, ) -> NonNull> { - target.cast() + TimerShared::addr_of_pointers(target) } } // ===== impl Entry ===== impl TimerEntry { - pub(crate) fn new(handle: &Handle, deadline: Instant) -> Self { + #[track_caller] + pub(crate) fn new(handle: &scheduler::Handle, deadline: Instant) -> Self { + // Panic if the time driver is not enabled + let _ = handle.driver().time(); + let driver = handle.clone(); Self { driver, inner: StdUnsafeCell::new(TimerShared::new()), - initial_deadline: Some(deadline), + deadline, + registered: false, _m: std::marker::PhantomPinned, } } @@ -498,8 +496,12 @@ impl TimerEntry { unsafe { &*self.inner.get() } } + pub(crate) fn deadline(&self) -> Instant { + self.deadline + } + pub(crate) fn is_elapsed(&self) -> bool { - !self.inner().state.might_be_registered() && self.initial_deadline.is_none() + !self.inner().state.might_be_registered() && self.registered } /// Cancels and deregisters the timer. This operation is irreversible. @@ -526,20 +528,24 @@ impl TimerEntry { // driver did so far and happens-before everything the driver does in // the future. While we have the lock held, we also go ahead and // deregister the entry if necessary. - unsafe { self.driver.clear_entry(NonNull::from(self.inner())) }; + unsafe { self.driver().clear_entry(NonNull::from(self.inner())) }; } - pub(crate) fn reset(mut self: Pin<&mut Self>, new_time: Instant) { - unsafe { self.as_mut().get_unchecked_mut() }.initial_deadline = None; + pub(crate) fn reset(mut self: Pin<&mut Self>, new_time: Instant, reregister: bool) { + unsafe { self.as_mut().get_unchecked_mut() }.deadline = new_time; + unsafe { self.as_mut().get_unchecked_mut() }.registered = reregister; - let tick = self.driver.time_source().deadline_to_tick(new_time); + let tick = self.driver().time_source().deadline_to_tick(new_time); if self.inner().extend_expiration(tick).is_ok() { return; } - unsafe { - self.driver.reregister(tick, self.inner().into()); + if reregister { + unsafe { + self.driver() + .reregister(&self.driver.driver().io, tick, self.inner().into()); + } } } @@ -547,18 +553,28 @@ impl TimerEntry { mut self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll> { - if self.driver.is_shutdown() { + if self.driver().is_shutdown() { panic!("{}", crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR); } - if let Some(deadline) = self.initial_deadline { - self.as_mut().reset(deadline); + if !self.registered { + let deadline = self.deadline; + self.as_mut().reset(deadline, true); } let this = unsafe { self.get_unchecked_mut() }; this.inner().state.poll(cx.waker()) } + + pub(crate) fn driver(&self) -> &super::Handle { + self.driver.driver().time() + } + + #[cfg(all(tokio_unstable, feature = "tracing"))] + pub(crate) fn clock(&self) -> &super::Clock { + self.driver.driver().clock() + } } impl TimerHandle { @@ -626,8 +642,3 @@ impl Drop for TimerEntry { unsafe { Pin::new_unchecked(self) }.as_mut().cancel() } } - -#[cfg_attr(target_arch = "x86_64", repr(align(128)))] -#[cfg_attr(not(target_arch = "x86_64"), repr(align(64)))] -#[derive(Debug, Default)] -struct CachePadded(T); diff --git a/third_party/rust/tokio/src/runtime/time/handle.rs b/third_party/rust/tokio/src/runtime/time/handle.rs new file mode 100644 index 000000000000..fce791d998c9 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/time/handle.rs @@ -0,0 +1,62 @@ +use crate::runtime::time::TimeSource; +use std::fmt; + +/// Handle to time driver instance. +pub(crate) struct Handle { + pub(super) time_source: TimeSource, + pub(super) inner: super::Inner, +} + +impl Handle { + /// Returns the time source associated with this handle. + pub(crate) fn time_source(&self) -> &TimeSource { + &self.time_source + } + + /// Checks whether the driver has been shutdown. + pub(super) fn is_shutdown(&self) -> bool { + self.inner.is_shutdown() + } + + /// Track that the driver is being unparked + pub(crate) fn unpark(&self) { + #[cfg(feature = "test-util")] + self.inner + .did_wake + .store(true, std::sync::atomic::Ordering::SeqCst); + } +} + +cfg_not_rt! { + impl Handle { + /// Tries to get a handle to the current timer. + /// + /// # Panics + /// + /// This function panics if there is no current timer set. + /// + /// It can be triggered when [`Builder::enable_time`] or + /// [`Builder::enable_all`] are not included in the builder. + /// + /// It can also panic whenever a timer is created outside of a + /// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic, + /// since the function is executed outside of the runtime. + /// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic. + /// And this is because wrapping the function on an async makes it lazy, + /// and so gets executed inside the runtime successfully without + /// panicking. + /// + /// [`Builder::enable_time`]: crate::runtime::Builder::enable_time + /// [`Builder::enable_all`]: crate::runtime::Builder::enable_all + #[track_caller] + pub(crate) fn current() -> Self { + panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) + } + } +} + +impl fmt::Debug for Handle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Handle") + } +} diff --git a/third_party/rust/tokio/src/time/driver/mod.rs b/third_party/rust/tokio/src/runtime/time/mod.rs similarity index 66% rename from third_party/rust/tokio/src/time/driver/mod.rs rename to third_party/rust/tokio/src/runtime/time/mod.rs index 997187747934..423ad79ab917 100644 --- a/third_party/rust/tokio/src/time/driver/mod.rs +++ b/third_party/rust/tokio/src/runtime/time/mod.rs @@ -7,22 +7,23 @@ //! Time driver. mod entry; -pub(self) use self::entry::{EntryList, TimerEntry, TimerHandle, TimerShared}; +pub(crate) use entry::TimerEntry; +use entry::{EntryList, TimerHandle, TimerShared, MAX_SAFE_MILLIS_DURATION}; mod handle; pub(crate) use self::handle::Handle; +mod source; +pub(crate) use source::TimeSource; + mod wheel; -pub(super) mod sleep; - use crate::loom::sync::atomic::{AtomicBool, Ordering}; -use crate::loom::sync::{Arc, Mutex}; -use crate::park::{Park, Unpark}; +use crate::loom::sync::Mutex; +use crate::runtime::driver::{self, IoHandle, IoStack}; use crate::time::error::Error; -use crate::time::{Clock, Duration, Instant}; +use crate::time::{Clock, Duration}; -use std::convert::TryInto; use std::fmt; use std::{num::NonZeroU64, ptr::NonNull, task::Waker}; @@ -82,63 +83,9 @@ use std::{num::NonZeroU64, ptr::NonNull, task::Waker}; /// [timeout]: crate::time::Timeout /// [interval]: crate::time::Interval #[derive(Debug)] -pub(crate) struct Driver { - /// Timing backend in use. - time_source: ClockTime, - - /// Shared state. - handle: Handle, - +pub(crate) struct Driver { /// Parker to delegate to. - park: P, - - // When `true`, a call to `park_timeout` should immediately return and time - // should not advance. One reason for this to be `true` is if the task - // passed to `Runtime::block_on` called `task::yield_now()`. - // - // While it may look racy, it only has any effect when the clock is paused - // and pausing the clock is restricted to a single-threaded runtime. - #[cfg(feature = "test-util")] - did_wake: Arc, -} - -/// A structure which handles conversion from Instants to u64 timestamps. -#[derive(Debug, Clone)] -pub(self) struct ClockTime { - clock: super::clock::Clock, - start_time: Instant, -} - -impl ClockTime { - pub(self) fn new(clock: Clock) -> Self { - Self { - start_time: clock.now(), - clock, - } - } - - pub(self) fn deadline_to_tick(&self, t: Instant) -> u64 { - // Round up to the end of a ms - self.instant_to_tick(t + Duration::from_nanos(999_999)) - } - - pub(self) fn instant_to_tick(&self, t: Instant) -> u64 { - // round up - let dur: Duration = t - .checked_duration_since(self.start_time) - .unwrap_or_else(|| Duration::from_secs(0)); - let ms = dur.as_millis(); - - ms.try_into().unwrap_or(u64::MAX) - } - - pub(self) fn tick_to_duration(&self, t: u64) -> Duration { - Duration::from_millis(t) - } - - pub(self) fn now(&self) -> u64 { - self.instant_to_tick(self.clock.now()) - } + park: IoStack, } /// Timer state shared between `Driver`, `Handle`, and `Registration`. @@ -148,13 +95,19 @@ struct Inner { /// True if the driver is being shutdown. pub(super) is_shutdown: AtomicBool, + + // When `true`, a call to `park_timeout` should immediately return and time + // should not advance. One reason for this to be `true` is if the task + // passed to `Runtime::block_on` called `task::yield_now()`. + // + // While it may look racy, it only has any effect when the clock is paused + // and pausing the clock is restricted to a single-threaded runtime. + #[cfg(feature = "test-util")] + did_wake: AtomicBool, } /// Time state shared which must be protected by a `Mutex` struct InnerState { - /// Timing backend in use. - time_source: ClockTime, - /// The last published timer `elapsed` value. elapsed: u64, @@ -163,49 +116,67 @@ struct InnerState { /// Timer wheel. wheel: wheel::Wheel, - - /// Unparker that can be used to wake the time driver. - unpark: Box, } // ===== impl Driver ===== -impl

Driver

-where - P: Park + 'static, -{ +impl Driver { /// Creates a new `Driver` instance that uses `park` to block the current /// thread and `time_source` to get the current time and convert to ticks. /// /// Specifying the source of time is useful when testing. - pub(crate) fn new(park: P, clock: Clock) -> Driver

{ - let time_source = ClockTime::new(clock); + pub(crate) fn new(park: IoStack, clock: &Clock) -> (Driver, Handle) { + let time_source = TimeSource::new(clock); - let inner = Inner::new(time_source.clone(), Box::new(park.unpark())); - - Driver { + let handle = Handle { time_source, - handle: Handle::new(Arc::new(inner)), - park, - #[cfg(feature = "test-util")] - did_wake: Arc::new(AtomicBool::new(false)), + inner: Inner { + state: Mutex::new(InnerState { + elapsed: 0, + next_wake: None, + wheel: wheel::Wheel::new(), + }), + is_shutdown: AtomicBool::new(false), + + #[cfg(feature = "test-util")] + did_wake: AtomicBool::new(false), + }, + }; + + let driver = Driver { park }; + + (driver, handle) + } + + pub(crate) fn park(&mut self, handle: &driver::Handle) { + self.park_internal(handle, None) + } + + pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) { + self.park_internal(handle, Some(duration)) + } + + pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) { + let handle = rt_handle.time(); + + if handle.is_shutdown() { + return; } + + handle.inner.is_shutdown.store(true, Ordering::SeqCst); + + // Advance time forward to the end of time. + + handle.process_at_time(u64::MAX); + + self.park.shutdown(rt_handle); } - /// Returns a handle to the timer. - /// - /// The `Handle` is how `Sleep` instances are created. The `Sleep` instances - /// can either be created directly or the `Handle` instance can be passed to - /// `with_default`, setting the timer as the default timer for the execution - /// context. - pub(crate) fn handle(&self) -> Handle { - self.handle.clone() - } + fn park_internal(&mut self, rt_handle: &driver::Handle, limit: Option) { + let handle = rt_handle.time(); + let mut lock = handle.inner.state.lock(); - fn park_internal(&mut self, limit: Option) -> Result<(), P::Error> { - let mut lock = self.handle.get().state.lock(); - - assert!(!self.handle.is_shutdown()); + assert!(!handle.is_shutdown()); let next_wake = lock.wheel.next_expiration_time(); lock.next_wake = @@ -215,75 +186,72 @@ where match next_wake { Some(when) => { - let now = self.time_source.now(); + let now = handle.time_source.now(rt_handle.clock()); // Note that we effectively round up to 1ms here - this avoids // very short-duration microsecond-resolution sleeps that the OS // might treat as zero-length. - let mut duration = self.time_source.tick_to_duration(when.saturating_sub(now)); + let mut duration = handle + .time_source + .tick_to_duration(when.saturating_sub(now)); if duration > Duration::from_millis(0) { if let Some(limit) = limit { duration = std::cmp::min(limit, duration); } - self.park_timeout(duration)?; + self.park_thread_timeout(rt_handle, duration); } else { - self.park.park_timeout(Duration::from_secs(0))?; + self.park.park_timeout(rt_handle, Duration::from_secs(0)); } } None => { if let Some(duration) = limit { - self.park_timeout(duration)?; + self.park_thread_timeout(rt_handle, duration); } else { - self.park.park()?; + self.park.park(rt_handle); } } } // Process pending timers after waking up - self.handle.process(); - - Ok(()) + handle.process(rt_handle.clock()); } cfg_test_util! { - fn park_timeout(&mut self, duration: Duration) -> Result<(), P::Error> { - let clock = &self.time_source.clock; + fn park_thread_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) { + let handle = rt_handle.time(); + let clock = rt_handle.clock(); - if clock.is_paused() { - self.park.park_timeout(Duration::from_secs(0))?; + if clock.can_auto_advance() { + self.park.park_timeout(rt_handle, Duration::from_secs(0)); // If the time driver was woken, then the park completed // before the "duration" elapsed (usually caused by a // yield in `Runtime::block_on`). In this case, we don't // advance the clock. - if !self.did_wake() { + if !handle.did_wake() { // Simulate advancing time - clock.advance(duration); + if let Err(msg) = clock.advance(duration) { + panic!("{}", msg); + } } } else { - self.park.park_timeout(duration)?; + self.park.park_timeout(rt_handle, duration); } - - Ok(()) - } - - fn did_wake(&self) -> bool { - self.did_wake.swap(false, Ordering::SeqCst) } } cfg_not_test_util! { - fn park_timeout(&mut self, duration: Duration) -> Result<(), P::Error> { - self.park.park_timeout(duration) + fn park_thread_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) { + self.park.park_timeout(rt_handle, duration); } } } impl Handle { /// Runs timer related logic, and returns the next wakeup time - pub(self) fn process(&self) { - let now = self.time_source().now(); + pub(self) fn process(&self, clock: &Clock) { + let now = self.time_source().now(clock); self.process_at_time(now) } @@ -292,7 +260,7 @@ impl Handle { let mut waker_list: [Option; 32] = Default::default(); let mut waker_idx = 0; - let mut lock = self.get().lock(); + let mut lock = self.inner.lock(); if now < lock.elapsed { // Time went backwards! This normally shouldn't happen as the Rust language @@ -323,7 +291,7 @@ impl Handle { waker_idx = 0; - lock = self.get().lock(); + lock = self.inner.lock(); } } } @@ -354,7 +322,7 @@ impl Handle { /// `add_entry` must not be called concurrently. pub(self) unsafe fn clear_entry(&self, entry: NonNull) { unsafe { - let mut lock = self.get().lock(); + let mut lock = self.inner.lock(); if entry.as_ref().might_be_registered() { lock.wheel.remove(entry); @@ -370,9 +338,14 @@ impl Handle { /// driver. No other threads are allowed to concurrently manipulate the /// timer at all (the current thread should hold an exclusive reference to /// the `TimerEntry`) - pub(self) unsafe fn reregister(&self, new_tick: u64, entry: NonNull) { + pub(self) unsafe fn reregister( + &self, + unpark: &IoHandle, + new_tick: u64, + entry: NonNull, + ) { let waker = unsafe { - let mut lock = self.get().lock(); + let mut lock = self.inner.lock(); // We may have raced with a firing/deregistration, so check before // deregistering. @@ -398,12 +371,12 @@ impl Handle { .map(|next_wake| when < next_wake.get()) .unwrap_or(true) { - lock.unpark.unpark(); + unpark.unpark(); } None } - Err((entry, super::error::InsertError::Elapsed)) => unsafe { + Err((entry, crate::time::error::InsertError::Elapsed)) => unsafe { entry.fire(Ok(())) }, } @@ -419,94 +392,17 @@ impl Handle { waker.wake(); } } -} -impl

Park for Driver

-where - P: Park + 'static, -{ - type Unpark = TimerUnpark

; - type Error = P::Error; - - fn unpark(&self) -> Self::Unpark { - TimerUnpark::new(self) - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.park_internal(None) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.park_internal(Some(duration)) - } - - fn shutdown(&mut self) { - if self.handle.is_shutdown() { - return; + cfg_test_util! { + fn did_wake(&self) -> bool { + self.inner.did_wake.swap(false, Ordering::SeqCst) } - - self.handle.get().is_shutdown.store(true, Ordering::SeqCst); - - // Advance time forward to the end of time. - - self.handle.process_at_time(u64::MAX); - - self.park.shutdown(); - } -} - -impl

Drop for Driver

-where - P: Park + 'static, -{ - fn drop(&mut self) { - self.shutdown(); - } -} - -pub(crate) struct TimerUnpark { - inner: P::Unpark, - - #[cfg(feature = "test-util")] - did_wake: Arc, -} - -impl TimerUnpark

{ - fn new(driver: &Driver

) -> TimerUnpark

{ - TimerUnpark { - inner: driver.park.unpark(), - - #[cfg(feature = "test-util")] - did_wake: driver.did_wake.clone(), - } - } -} - -impl Unpark for TimerUnpark

{ - fn unpark(&self) { - #[cfg(feature = "test-util")] - self.did_wake.store(true, Ordering::SeqCst); - - self.inner.unpark(); } } // ===== impl Inner ===== impl Inner { - pub(self) fn new(time_source: ClockTime, unpark: Box) -> Self { - Inner { - state: Mutex::new(InnerState { - time_source, - elapsed: 0, - next_wake: None, - unpark, - wheel: wheel::Wheel::new(), - }), - is_shutdown: AtomicBool::new(false), - } - } - /// Locks the driver's inner structure pub(super) fn lock(&self) -> crate::loom::sync::MutexGuard<'_, InnerState> { self.state.lock() diff --git a/third_party/rust/tokio/src/runtime/time/source.rs b/third_party/rust/tokio/src/runtime/time/source.rs new file mode 100644 index 000000000000..4647bc412238 --- /dev/null +++ b/third_party/rust/tokio/src/runtime/time/source.rs @@ -0,0 +1,39 @@ +use super::MAX_SAFE_MILLIS_DURATION; +use crate::time::{Clock, Duration, Instant}; + +/// A structure which handles conversion from Instants to u64 timestamps. +#[derive(Debug)] +pub(crate) struct TimeSource { + start_time: Instant, +} + +impl TimeSource { + pub(crate) fn new(clock: &Clock) -> Self { + Self { + start_time: clock.now(), + } + } + + pub(crate) fn deadline_to_tick(&self, t: Instant) -> u64 { + // Round up to the end of a ms + self.instant_to_tick(t + Duration::from_nanos(999_999)) + } + + pub(crate) fn instant_to_tick(&self, t: Instant) -> u64 { + // round up + let dur: Duration = t + .checked_duration_since(self.start_time) + .unwrap_or_else(|| Duration::from_secs(0)); + let ms = dur.as_millis(); + + ms.try_into().unwrap_or(MAX_SAFE_MILLIS_DURATION) + } + + pub(crate) fn tick_to_duration(&self, t: u64) -> Duration { + Duration::from_millis(t) + } + + pub(crate) fn now(&self, clock: &Clock) -> u64 { + self.instant_to_tick(clock.now()) + } +} diff --git a/third_party/rust/tokio/src/time/driver/tests/mod.rs b/third_party/rust/tokio/src/runtime/time/tests/mod.rs similarity index 53% rename from third_party/rust/tokio/src/time/driver/tests/mod.rs rename to third_party/rust/tokio/src/runtime/time/tests/mod.rs index 3ac8c756437c..155d99a348f3 100644 --- a/third_party/rust/tokio/src/time/driver/tests/mod.rs +++ b/third_party/rust/tokio/src/runtime/time/tests/mod.rs @@ -1,26 +1,15 @@ +#![cfg(not(tokio_wasi))] + use std::{task::Context, time::Duration}; #[cfg(not(loom))] use futures::task::noop_waker_ref; +use crate::loom::sync::atomic::{AtomicBool, Ordering}; use crate::loom::sync::Arc; use crate::loom::thread; -use crate::{ - loom::sync::atomic::{AtomicBool, Ordering}, - park::Unpark, -}; -use super::{Handle, TimerEntry}; - -struct MockUnpark {} -impl Unpark for MockUnpark { - fn unpark(&self) {} -} -impl MockUnpark { - fn mock() -> Box { - Box::new(Self {}) - } -} +use super::TimerEntry; fn block_on(f: impl std::future::Future) -> T { #[cfg(loom)] @@ -43,18 +32,26 @@ fn model(f: impl Fn() + Send + Sync + 'static) { f(); } +fn rt(start_paused: bool) -> crate::runtime::Runtime { + crate::runtime::Builder::new_current_thread() + .enable_time() + .start_paused(start_paused) + .build() + .unwrap() +} + #[test] fn single_timer() { model(|| { - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1)); + let entry = TimerEntry::new( + &handle_.inner, + handle_.inner.driver().clock().now() + Duration::from_secs(1), + ); pin!(entry); block_on(futures::future::poll_fn(|cx| { @@ -65,10 +62,13 @@ fn single_timer() { thread::yield_now(); + let time = handle.inner.driver().time(); + let clock = handle.inner.driver().clock(); + // This may or may not return Some (depending on how it races with the // thread). If it does return None, however, the timer should complete // synchronously. - handle.process_at_time(time_source.now() + 2_000_000_000); + time.process_at_time(time.time_source().now(clock) + 2_000_000_000); jh.join().unwrap(); }) @@ -77,15 +77,15 @@ fn single_timer() { #[test] fn drop_timer() { model(|| { - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1)); + let entry = TimerEntry::new( + &handle_.inner, + handle_.inner.driver().clock().now() + Duration::from_secs(1), + ); pin!(entry); let _ = entry @@ -98,8 +98,11 @@ fn drop_timer() { thread::yield_now(); + let time = handle.inner.driver().time(); + let clock = handle.inner.driver().clock(); + // advance 2s in the future. - handle.process_at_time(time_source.now() + 2_000_000_000); + time.process_at_time(time.time_source().now(clock) + 2_000_000_000); jh.join().unwrap(); }) @@ -108,15 +111,15 @@ fn drop_timer() { #[test] fn change_waker() { model(|| { - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1)); + let entry = TimerEntry::new( + &handle_.inner, + handle_.inner.driver().clock().now() + Duration::from_secs(1), + ); pin!(entry); let _ = entry @@ -131,8 +134,11 @@ fn change_waker() { thread::yield_now(); + let time = handle.inner.driver().time(); + let clock = handle.inner.driver().clock(); + // advance 2s - handle.process_at_time(time_source.now() + 2_000_000_000); + time.process_at_time(time.time_source().now(clock) + 2_000_000_000); jh.join().unwrap(); }) @@ -143,25 +149,22 @@ fn reset_future() { model(|| { let finished_early = Arc::new(AtomicBool::new(false)); - let clock = crate::time::clock::Clock::new(true, false); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source.clone(), MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(false); + let handle = rt.handle(); let handle_ = handle.clone(); let finished_early_ = finished_early.clone(); - let start = clock.now(); + let start = handle.inner.driver().clock().now(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(&handle_, start + Duration::from_secs(1)); + let entry = TimerEntry::new(&handle_.inner, start + Duration::from_secs(1)); pin!(entry); let _ = entry .as_mut() .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref())); - entry.as_mut().reset(start + Duration::from_secs(2)); + entry.as_mut().reset(start + Duration::from_secs(2), true); // shouldn't complete before 2s block_on(futures::future::poll_fn(|cx| { @@ -174,12 +177,22 @@ fn reset_future() { thread::yield_now(); + let handle = handle.inner.driver().time(); + // This may or may not return a wakeup time. - handle.process_at_time(time_source.instant_to_tick(start + Duration::from_millis(1500))); + handle.process_at_time( + handle + .time_source() + .instant_to_tick(start + Duration::from_millis(1500)), + ); assert!(!finished_early.load(Ordering::Relaxed)); - handle.process_at_time(time_source.instant_to_tick(start + Duration::from_millis(2500))); + handle.process_at_time( + handle + .time_source() + .instant_to_tick(start + Duration::from_millis(2500)), + ); jh.join().unwrap(); @@ -199,20 +212,15 @@ fn normal_or_miri(normal: T, miri: T) -> T { #[test] #[cfg(not(loom))] fn poll_process_levels() { - let clock = crate::time::clock::Clock::new(true, false); - clock.pause(); - - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source, MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); + let rt = rt(true); + let handle = rt.handle(); let mut entries = vec![]; for i in 0..normal_or_miri(1024, 64) { let mut entry = Box::pin(TimerEntry::new( - &handle, - clock.now() + Duration::from_millis(i), + &handle.inner, + handle.inner.driver().clock().now() + Duration::from_millis(i), )); let _ = entry @@ -223,7 +231,8 @@ fn poll_process_levels() { } for t in 1..normal_or_miri(1024, 64) { - handle.process_at_time(t as u64); + handle.inner.driver().time().process_at_time(t as u64); + for (deadline, future) in entries.iter_mut().enumerate() { let mut context = Context::from_waker(noop_waker_ref()); if deadline <= t { @@ -240,62 +249,19 @@ fn poll_process_levels() { fn poll_process_levels_targeted() { let mut context = Context::from_waker(noop_waker_ref()); - let clock = crate::time::clock::Clock::new(true, false); - clock.pause(); + let rt = rt(true); + let handle = rt.handle(); - let time_source = super::ClockTime::new(clock.clone()); - - let inner = super::Inner::new(time_source, MockUnpark::mock()); - let handle = Handle::new(Arc::new(inner)); - - let e1 = TimerEntry::new(&handle, clock.now() + Duration::from_millis(193)); + let e1 = TimerEntry::new( + &handle.inner, + handle.inner.driver().clock().now() + Duration::from_millis(193), + ); pin!(e1); + let handle = handle.inner.driver().time(); + handle.process_at_time(62); assert!(e1.as_mut().poll_elapsed(&mut context).is_pending()); handle.process_at_time(192); handle.process_at_time(192); } - -/* -#[test] -fn balanced_incr_and_decr() { - const OPS: usize = 5; - - fn incr(inner: Arc) { - for _ in 0..OPS { - inner.increment().expect("increment should not have failed"); - thread::yield_now(); - } - } - - fn decr(inner: Arc) { - let mut ops_performed = 0; - while ops_performed < OPS { - if inner.num(Ordering::Relaxed) > 0 { - ops_performed += 1; - inner.decrement(); - } - thread::yield_now(); - } - } - - loom::model(|| { - let unpark = Box::new(MockUnpark); - let instant = Instant::now(); - - let inner = Arc::new(Inner::new(instant, unpark)); - - let incr_inner = inner.clone(); - let decr_inner = inner.clone(); - - let incr_hndle = thread::spawn(move || incr(incr_inner)); - let decr_hndle = thread::spawn(move || decr(decr_inner)); - - incr_hndle.join().expect("should never fail"); - decr_hndle.join().expect("should never fail"); - - assert_eq!(inner.num(Ordering::SeqCst), 0); - }) -} -*/ diff --git a/third_party/rust/tokio/src/time/driver/wheel/level.rs b/third_party/rust/tokio/src/runtime/time/wheel/level.rs similarity index 98% rename from third_party/rust/tokio/src/time/driver/wheel/level.rs rename to third_party/rust/tokio/src/runtime/time/wheel/level.rs index 878754177b9c..7e48ff5c57d0 100644 --- a/third_party/rust/tokio/src/time/driver/wheel/level.rs +++ b/third_party/rust/tokio/src/runtime/time/wheel/level.rs @@ -1,6 +1,4 @@ -use crate::time::driver::TimerHandle; - -use crate::time::driver::{EntryList, TimerShared}; +use crate::runtime::time::{EntryList, TimerHandle, TimerShared}; use std::{fmt, ptr::NonNull}; diff --git a/third_party/rust/tokio/src/time/driver/wheel/mod.rs b/third_party/rust/tokio/src/runtime/time/wheel/mod.rs similarity index 95% rename from third_party/rust/tokio/src/time/driver/wheel/mod.rs rename to third_party/rust/tokio/src/runtime/time/wheel/mod.rs index f088f2cfd66b..bf13b7b24155 100644 --- a/third_party/rust/tokio/src/time/driver/wheel/mod.rs +++ b/third_party/rust/tokio/src/runtime/time/wheel/mod.rs @@ -1,4 +1,4 @@ -use crate::time::driver::{TimerHandle, TimerShared}; +use crate::runtime::time::{TimerHandle, TimerShared}; use crate::time::error::InsertError; mod level; @@ -148,23 +148,13 @@ impl Wheel { return Some(handle); } - // under what circumstances is poll.expiration Some vs. None? - let expiration = self.next_expiration().and_then(|expiration| { - if expiration.deadline > now { - None - } else { - Some(expiration) - } - }); - - match expiration { - Some(ref expiration) if expiration.deadline > now => return None, - Some(ref expiration) => { + match self.next_expiration() { + Some(ref expiration) if expiration.deadline <= now => { self.process_expiration(expiration); self.set_elapsed(expiration.deadline); } - None => { + _ => { // in this case the poll did not indicate an expiration // _and_ we were not able to find a next expiration in // the current list of timers. advance to the poll's diff --git a/third_party/rust/tokio/src/signal/mod.rs b/third_party/rust/tokio/src/signal/mod.rs index 882218a0f6d6..3aacc60efc3d 100644 --- a/third_party/rust/tokio/src/signal/mod.rs +++ b/third_party/rust/tokio/src/signal/mod.rs @@ -48,7 +48,7 @@ use std::task::{Context, Poll}; mod ctrl_c; pub use ctrl_c::ctrl_c; -mod registry; +pub(crate) mod registry; mod os { #[cfg(unix)] diff --git a/third_party/rust/tokio/src/signal/registry.rs b/third_party/rust/tokio/src/signal/registry.rs index 6d8eb9e7487b..48e98c832faf 100644 --- a/third_party/rust/tokio/src/signal/registry.rs +++ b/third_party/rust/tokio/src/signal/registry.rs @@ -1,12 +1,10 @@ #![allow(clippy::unit_arg)] use crate::signal::os::{OsExtraData, OsStorage}; - use crate::sync::watch; +use crate::util::once_cell::OnceCell; -use once_cell::sync::Lazy; use std::ops; -use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; pub(crate) type EventId = usize; @@ -152,19 +150,25 @@ impl Globals { } } -pub(crate) fn globals() -> Pin<&'static Globals> +fn globals_init() -> Globals where OsExtraData: 'static + Send + Sync + Init, OsStorage: 'static + Send + Sync + Init, { - static GLOBALS: Lazy>> = Lazy::new(|| { - Box::pin(Globals { - extra: OsExtraData::init(), - registry: Registry::new(OsStorage::init()), - }) - }); + Globals { + extra: OsExtraData::init(), + registry: Registry::new(OsStorage::init()), + } +} - GLOBALS.as_ref() +pub(crate) fn globals() -> &'static Globals +where + OsExtraData: 'static + Send + Sync + Init, + OsStorage: 'static + Send + Sync + Init, +{ + static GLOBALS: OnceCell = OnceCell::new(); + + GLOBALS.get(globals_init) } #[cfg(all(test, not(loom)))] @@ -237,7 +241,7 @@ mod tests { #[test] fn record_invalid_event_does_nothing() { let registry = Registry::new(vec![EventInfo::default()]); - registry.record_event(42); + registry.record_event(1302); } #[test] diff --git a/third_party/rust/tokio/src/signal/reusable_box.rs b/third_party/rust/tokio/src/signal/reusable_box.rs index 02f32474b163..796fa210b034 100644 --- a/third_party/rust/tokio/src/signal/reusable_box.rs +++ b/third_party/rust/tokio/src/signal/reusable_box.rs @@ -151,7 +151,6 @@ impl fmt::Debug for ReusableBoxFuture { } #[cfg(test)] -#[cfg(not(miri))] // Miri breaks when you use Pin<&mut dyn Future> mod test { use super::ReusableBoxFuture; use futures::future::FutureExt; diff --git a/third_party/rust/tokio/src/signal/unix.rs b/third_party/rust/tokio/src/signal/unix.rs index 86ea9a93ee68..ae5c13085e9f 100644 --- a/third_party/rust/tokio/src/signal/unix.rs +++ b/third_party/rust/tokio/src/signal/unix.rs @@ -6,29 +6,31 @@ #![cfg(unix)] #![cfg_attr(docsrs, doc(cfg(all(unix, feature = "signal"))))] +use crate::runtime::scheduler; +use crate::runtime::signal::Handle; use crate::signal::registry::{globals, EventId, EventInfo, Globals, Init, Storage}; use crate::signal::RxFuture; use crate::sync::watch; use mio::net::UnixStream; use std::io::{self, Error, ErrorKind, Write}; -use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Once; use std::task::{Context, Poll}; -pub(crate) mod driver; -use self::driver::Handle; - pub(crate) type OsStorage = Vec; -// Number of different unix signals -// (FreeBSD has 33) -const SIGNUM: usize = 33; - impl Init for OsStorage { fn init() -> Self { - (0..SIGNUM).map(|_| SignalInfo::default()).collect() + // There are reliable signals ranging from 1 to 33 available on every Unix platform. + #[cfg(not(target_os = "linux"))] + let possible = 0..=33; + + // On Linux, there are additional real-time signals available. + #[cfg(target_os = "linux")] + let possible = 0..=libc::SIGRTMAX(); + + possible.map(|_| SignalInfo::default()).collect() } } @@ -48,7 +50,7 @@ impl Storage for OsStorage { #[derive(Debug)] pub(crate) struct OsExtraData { sender: UnixStream, - receiver: UnixStream, + pub(crate) receiver: UnixStream, } impl Init for OsExtraData { @@ -60,7 +62,7 @@ impl Init for OsExtraData { } /// Represents the specific kind of signal to listen for. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] pub struct SignalKind(libc::c_int); impl SignalKind { @@ -80,15 +82,26 @@ impl SignalKind { // unlikely to change to other types, but technically libc can change this // in the future minor version. // See https://github.com/tokio-rs/tokio/issues/3767 for more. - pub fn from_raw(signum: std::os::raw::c_int) -> Self { + pub const fn from_raw(signum: std::os::raw::c_int) -> Self { Self(signum as libc::c_int) } + /// Get the signal's numeric value. + /// + /// ```rust + /// # use tokio::signal::unix::SignalKind; + /// let kind = SignalKind::interrupt(); + /// assert_eq!(kind.as_raw_value(), libc::SIGINT); + /// ``` + pub const fn as_raw_value(&self) -> std::os::raw::c_int { + self.0 + } + /// Represents the SIGALRM signal. /// /// On Unix systems this signal is sent when a real-time timer has expired. /// By default, the process is terminated by this signal. - pub fn alarm() -> Self { + pub const fn alarm() -> Self { Self(libc::SIGALRM) } @@ -96,7 +109,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent when the status of a child process /// has changed. By default, this signal is ignored. - pub fn child() -> Self { + pub const fn child() -> Self { Self(libc::SIGCHLD) } @@ -104,7 +117,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent when the terminal is disconnected. /// By default, the process is terminated by this signal. - pub fn hangup() -> Self { + pub const fn hangup() -> Self { Self(libc::SIGHUP) } @@ -119,7 +132,7 @@ impl SignalKind { target_os = "netbsd", target_os = "openbsd" ))] - pub fn info() -> Self { + pub const fn info() -> Self { Self(libc::SIGINFO) } @@ -127,7 +140,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent to interrupt a program. /// By default, the process is terminated by this signal. - pub fn interrupt() -> Self { + pub const fn interrupt() -> Self { Self(libc::SIGINT) } @@ -135,7 +148,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent when I/O operations are possible /// on some file descriptor. By default, this signal is ignored. - pub fn io() -> Self { + pub const fn io() -> Self { Self(libc::SIGIO) } @@ -144,7 +157,7 @@ impl SignalKind { /// On Unix systems this signal is sent when the process attempts to write /// to a pipe which has no reader. By default, the process is terminated by /// this signal. - pub fn pipe() -> Self { + pub const fn pipe() -> Self { Self(libc::SIGPIPE) } @@ -153,7 +166,7 @@ impl SignalKind { /// On Unix systems this signal is sent to issue a shutdown of the /// process, after which the OS will dump the process core. /// By default, the process is terminated by this signal. - pub fn quit() -> Self { + pub const fn quit() -> Self { Self(libc::SIGQUIT) } @@ -161,7 +174,7 @@ impl SignalKind { /// /// On Unix systems this signal is sent to issue a shutdown of the /// process. By default, the process is terminated by this signal. - pub fn terminate() -> Self { + pub const fn terminate() -> Self { Self(libc::SIGTERM) } @@ -169,7 +182,7 @@ impl SignalKind { /// /// On Unix systems this is a user defined signal. /// By default, the process is terminated by this signal. - pub fn user_defined1() -> Self { + pub const fn user_defined1() -> Self { Self(libc::SIGUSR1) } @@ -177,7 +190,7 @@ impl SignalKind { /// /// On Unix systems this is a user defined signal. /// By default, the process is terminated by this signal. - pub fn user_defined2() -> Self { + pub const fn user_defined2() -> Self { Self(libc::SIGUSR2) } @@ -185,11 +198,23 @@ impl SignalKind { /// /// On Unix systems this signal is sent when the terminal window is resized. /// By default, this signal is ignored. - pub fn window_change() -> Self { + pub const fn window_change() -> Self { Self(libc::SIGWINCH) } } +impl From for SignalKind { + fn from(signum: std::os::raw::c_int) -> Self { + Self::from_raw(signum as libc::c_int) + } +} + +impl From for std::os::raw::c_int { + fn from(kind: SignalKind) -> Self { + kind.as_raw_value() + } +} + pub(crate) struct SignalInfo { event_info: EventInfo, init: Once, @@ -214,7 +239,7 @@ impl Default for SignalInfo { /// 2. Wake up the driver by writing a byte to a pipe /// /// Those two operations should both be async-signal safe. -fn action(globals: Pin<&'static Globals>, signal: libc::c_int) { +fn action(globals: &'static Globals, signal: libc::c_int) { globals.record_event(signal as EventId); // Send a wakeup, ignore any errors (anything reasonably possible is @@ -267,7 +292,11 @@ fn signal_enable(signal: SignalKind, handle: &Handle) -> io::Result<()> { } } -/// A stream of events for receiving a particular type of OS signal. +/// An listener for receiving a particular type of OS signal. +/// +/// The listener can be turned into a `Stream` using [`SignalStream`]. +/// +/// [`SignalStream`]: https://docs.rs/tokio-stream/latest/tokio_stream/wrappers/struct.SignalStream.html /// /// In general signal handling on Unix is a pretty tricky topic, and this /// structure is no exception! There are some important limitations to keep in @@ -282,7 +311,7 @@ fn signal_enable(signal: SignalKind, handle: &Handle) -> io::Result<()> { /// Once `poll` has been called, however, a further signal is guaranteed to /// be yielded as an item. /// -/// Put another way, any element pulled off the returned stream corresponds to +/// Put another way, any element pulled off the returned listener corresponds to /// *at least one* signal, but possibly more. /// /// * Signal handling in general is relatively inefficient. Although some @@ -320,11 +349,11 @@ fn signal_enable(signal: SignalKind, handle: &Handle) -> io::Result<()> { /// #[tokio::main] /// async fn main() -> Result<(), Box> { /// // An infinite stream of hangup signals. -/// let mut stream = signal(SignalKind::hangup())?; +/// let mut sig = signal(SignalKind::hangup())?; /// /// // Print whenever a HUP signal is received /// loop { -/// stream.recv().await; +/// sig.recv().await; /// println!("got signal HUP"); /// } /// } @@ -335,7 +364,7 @@ pub struct Signal { inner: RxFuture, } -/// Creates a new stream which will receive notifications when the current +/// Creates a new listener which will receive notifications when the current /// process receives the specified signal `kind`. /// /// This function will create a new stream which binds to the default reactor. @@ -357,8 +386,15 @@ pub struct Signal { /// * If the previous initialization of this specific signal failed. /// * If the signal is one of /// [`signal_hook::FORBIDDEN`](fn@signal_hook_registry::register#panics) +/// +/// # Panics +/// +/// This function panics if there is no current reactor set, or if the `rt` +/// feature flag is not enabled. +#[track_caller] pub fn signal(kind: SignalKind) -> io::Result { - let rx = signal_with_handle(kind, &Handle::current())?; + let handle = scheduler::Handle::current(); + let rx = signal_with_handle(kind, handle.driver().signal())?; Ok(Signal { inner: RxFuture::new(rx), @@ -380,6 +416,12 @@ impl Signal { /// /// `None` is returned if no more events can be received by this stream. /// + /// # Cancel safety + /// + /// This method is cancel safe. If you use it as the event in a + /// [`tokio::select!`](crate::select) statement and some other branch + /// completes first, then it is guaranteed that no signal is lost. + /// /// # Examples /// /// Wait for SIGHUP @@ -474,4 +516,15 @@ mod tests { ) .unwrap_err(); } + + #[test] + fn from_c_int() { + assert_eq!(SignalKind::from(2), SignalKind::interrupt()); + } + + #[test] + fn into_c_int() { + let value: std::os::raw::c_int = SignalKind::interrupt().into(); + assert_eq!(value, libc::SIGINT as _); + } } diff --git a/third_party/rust/tokio/src/signal/unix/driver.rs b/third_party/rust/tokio/src/signal/unix/driver.rs deleted file mode 100644 index 5fe7c354c5ff..000000000000 --- a/third_party/rust/tokio/src/signal/unix/driver.rs +++ /dev/null @@ -1,207 +0,0 @@ -#![cfg_attr(not(feature = "rt"), allow(dead_code))] - -//! Signal driver - -use crate::io::driver::{Driver as IoDriver, Interest}; -use crate::io::PollEvented; -use crate::park::Park; -use crate::signal::registry::globals; - -use mio::net::UnixStream; -use std::io::{self, Read}; -use std::ptr; -use std::sync::{Arc, Weak}; -use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; -use std::time::Duration; - -/// Responsible for registering wakeups when an OS signal is received, and -/// subsequently dispatching notifications to any signal listeners as appropriate. -/// -/// Note: this driver relies on having an enabled IO driver in order to listen to -/// pipe write wakeups. -#[derive(Debug)] -pub(crate) struct Driver { - /// Thread parker. The `Driver` park implementation delegates to this. - park: IoDriver, - - /// A pipe for receiving wake events from the signal handler - receiver: PollEvented, - - /// Shared state - inner: Arc, -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct Handle { - inner: Weak, -} - -#[derive(Debug)] -pub(super) struct Inner(()); - -// ===== impl Driver ===== - -impl Driver { - /// Creates a new signal `Driver` instance that delegates wakeups to `park`. - pub(crate) fn new(park: IoDriver) -> io::Result { - use std::mem::ManuallyDrop; - use std::os::unix::io::{AsRawFd, FromRawFd}; - - // NB: We give each driver a "fresh" receiver file descriptor to avoid - // the issues described in alexcrichton/tokio-process#42. - // - // In the past we would reuse the actual receiver file descriptor and - // swallow any errors around double registration of the same descriptor. - // I'm not sure if the second (failed) registration simply doesn't end - // up receiving wake up notifications, or there could be some race - // condition when consuming readiness events, but having distinct - // descriptors for distinct PollEvented instances appears to mitigate - // this. - // - // Unfortunately we cannot just use a single global PollEvented instance - // either, since we can't compare Handles or assume they will always - // point to the exact same reactor. - // - // Mio 0.7 removed `try_clone()` as an API due to unexpected behavior - // with registering dups with the same reactor. In this case, duping is - // safe as each dup is registered with separate reactors **and** we - // only expect at least one dup to receive the notification. - - // Manually drop as we don't actually own this instance of UnixStream. - let receiver_fd = globals().receiver.as_raw_fd(); - - // safety: there is nothing unsafe about this, but the `from_raw_fd` fn is marked as unsafe. - let original = - ManuallyDrop::new(unsafe { std::os::unix::net::UnixStream::from_raw_fd(receiver_fd) }); - let receiver = UnixStream::from_std(original.try_clone()?); - let receiver = PollEvented::new_with_interest_and_handle( - receiver, - Interest::READABLE | Interest::WRITABLE, - park.handle(), - )?; - - Ok(Self { - park, - receiver, - inner: Arc::new(Inner(())), - }) - } - - /// Returns a handle to this event loop which can be sent across threads - /// and can be used as a proxy to the event loop itself. - pub(crate) fn handle(&self) -> Handle { - Handle { - inner: Arc::downgrade(&self.inner), - } - } - - fn process(&self) { - // Check if the pipe is ready to read and therefore has "woken" us up - // - // To do so, we will `poll_read_ready` with a noop waker, since we don't - // need to actually be notified when read ready... - let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &NOOP_WAKER_VTABLE)) }; - let mut cx = Context::from_waker(&waker); - - let ev = match self.receiver.registration().poll_read_ready(&mut cx) { - Poll::Ready(Ok(ev)) => ev, - Poll::Ready(Err(e)) => panic!("reactor gone: {}", e), - Poll::Pending => return, // No wake has arrived, bail - }; - - // Drain the pipe completely so we can receive a new readiness event - // if another signal has come in. - let mut buf = [0; 128]; - loop { - match (&*self.receiver).read(&mut buf) { - Ok(0) => panic!("EOF on self-pipe"), - Ok(_) => continue, // Keep reading - Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, - Err(e) => panic!("Bad read on self-pipe: {}", e), - } - } - - self.receiver.registration().clear_readiness(ev); - - // Broadcast any signals which were received - globals().broadcast(); - } -} - -const NOOP_WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new(noop_clone, noop, noop, noop); - -unsafe fn noop_clone(_data: *const ()) -> RawWaker { - RawWaker::new(ptr::null(), &NOOP_WAKER_VTABLE) -} - -unsafe fn noop(_data: *const ()) {} - -// ===== impl Park for Driver ===== - -impl Park for Driver { - type Unpark = ::Unpark; - type Error = io::Error; - - fn unpark(&self) -> Self::Unpark { - self.park.unpark() - } - - fn park(&mut self) -> Result<(), Self::Error> { - self.park.park()?; - self.process(); - Ok(()) - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.park.park_timeout(duration)?; - self.process(); - Ok(()) - } - - fn shutdown(&mut self) { - self.park.shutdown() - } -} - -// ===== impl Handle ===== - -impl Handle { - pub(super) fn check_inner(&self) -> io::Result<()> { - if self.inner.strong_count() > 0 { - Ok(()) - } else { - Err(io::Error::new(io::ErrorKind::Other, "signal driver gone")) - } - } -} - -cfg_rt! { - impl Handle { - /// Returns a handle to the current driver - /// - /// # Panics - /// - /// This function panics if there is no current signal driver set. - pub(super) fn current() -> Self { - crate::runtime::context::signal_handle().expect( - "there is no signal driver running, must be called from the context of Tokio runtime", - ) - } - } -} - -cfg_not_rt! { - impl Handle { - /// Returns a handle to the current driver - /// - /// # Panics - /// - /// This function panics if there is no current signal driver set. - pub(super) fn current() -> Self { - panic!( - "there is no signal driver running, must be called from the context of Tokio runtime or with\ - `rt` enabled.", - ) - } - } -} diff --git a/third_party/rust/tokio/src/signal/windows.rs b/third_party/rust/tokio/src/signal/windows.rs index 11ec6cb08c13..2f70f98b15a7 100644 --- a/third_party/rust/tokio/src/signal/windows.rs +++ b/third_party/rust/tokio/src/signal/windows.rs @@ -1,9 +1,9 @@ //! Windows-specific types for signal handling. //! -//! This module is only defined on Windows and allows receiving "ctrl-c" -//! and "ctrl-break" notifications. These events are listened for via the -//! `SetConsoleCtrlHandler` function which receives events of the type -//! `CTRL_C_EVENT` and `CTRL_BREAK_EVENT`. +//! This module is only defined on Windows and allows receiving "ctrl-c", +//! "ctrl-break", "ctrl-logoff", "ctrl-shutdown", and "ctrl-close" +//! notifications. These events are listened for via the `SetConsoleCtrlHandler` +//! function which receives the corresponding windows_sys event type. #![cfg(any(windows, docsrs))] #![cfg_attr(docsrs, doc(cfg(all(windows, feature = "signal"))))] @@ -22,7 +22,7 @@ pub(crate) use self::imp::{OsExtraData, OsStorage}; #[path = "windows/stub.rs"] mod imp; -/// Creates a new stream which receives "ctrl-c" notifications sent to the +/// Creates a new listener which receives "ctrl-c" notifications sent to the /// process. /// /// # Examples @@ -32,12 +32,12 @@ mod imp; /// /// #[tokio::main] /// async fn main() -> Result<(), Box> { -/// // An infinite stream of CTRL-C events. -/// let mut stream = ctrl_c()?; +/// // A listener of CTRL-C events. +/// let mut signal = ctrl_c()?; /// /// // Print whenever a CTRL-C event is received. /// for countdown in (0..3).rev() { -/// stream.recv().await; +/// signal.recv().await; /// println!("got CTRL-C. {} more to exit", countdown); /// } /// @@ -50,14 +50,18 @@ pub fn ctrl_c() -> io::Result { }) } -/// Represents a stream which receives "ctrl-c" notifications sent to the process +/// Represents a listener which receives "ctrl-c" notifications sent to the process /// via `SetConsoleCtrlHandler`. /// -/// A notification to this process notifies *all* streams listening for +/// This event can be turned into a `Stream` using [`CtrlCStream`]. +/// +/// [`CtrlCStream`]: https://docs.rs/tokio-stream/latest/tokio_stream/wrappers/struct.CtrlCStream.html +/// +/// A notification to this process notifies *all* receivers for /// this event. Moreover, the notifications **are coalesced** if they aren't processed /// quickly enough. This means that if two notifications are received back-to-back, -/// then the stream may only receive one item about the two notifications. -#[must_use = "streams do nothing unless polled"] +/// then the listener may only receive one item about the two notifications. +#[must_use = "listeners do nothing unless polled"] #[derive(Debug)] pub struct CtrlC { inner: RxFuture, @@ -66,7 +70,7 @@ pub struct CtrlC { impl CtrlC { /// Receives the next signal notification event. /// - /// `None` is returned if no more events can be received by this stream. + /// `None` is returned if no more events can be received by the listener. /// /// # Examples /// @@ -75,12 +79,11 @@ impl CtrlC { /// /// #[tokio::main] /// async fn main() -> Result<(), Box> { - /// // An infinite stream of CTRL-C events. - /// let mut stream = ctrl_c()?; + /// let mut signal = ctrl_c()?; /// /// // Print whenever a CTRL-C event is received. /// for countdown in (0..3).rev() { - /// stream.recv().await; + /// signal.recv().await; /// println!("got CTRL-C. {} more to exit", countdown); /// } /// @@ -94,7 +97,7 @@ impl CtrlC { /// Polls to receive the next signal notification event, outside of an /// `async` context. /// - /// `None` is returned if no more events can be received by this stream. + /// `None` is returned if no more events can be received. /// /// # Examples /// @@ -124,14 +127,18 @@ impl CtrlC { } } -/// Represents a stream which receives "ctrl-break" notifications sent to the process +/// Represents a listener which receives "ctrl-break" notifications sent to the process /// via `SetConsoleCtrlHandler`. /// -/// A notification to this process notifies *all* streams listening for +/// This listener can be turned into a `Stream` using [`CtrlBreakStream`]. +/// +/// [`CtrlBreakStream`]: https://docs.rs/tokio-stream/latest/tokio_stream/wrappers/struct.CtrlBreakStream.html +/// +/// A notification to this process notifies *all* receivers for /// this event. Moreover, the notifications **are coalesced** if they aren't processed /// quickly enough. This means that if two notifications are received back-to-back, -/// then the stream may only receive one item about the two notifications. -#[must_use = "streams do nothing unless polled"] +/// then the listener may only receive one item about the two notifications. +#[must_use = "listeners do nothing unless polled"] #[derive(Debug)] pub struct CtrlBreak { inner: RxFuture, @@ -140,7 +147,7 @@ pub struct CtrlBreak { impl CtrlBreak { /// Receives the next signal notification event. /// - /// `None` is returned if no more events can be received by this stream. + /// `None` is returned if no more events can be received by this listener. /// /// # Examples /// @@ -149,12 +156,12 @@ impl CtrlBreak { /// /// #[tokio::main] /// async fn main() -> Result<(), Box> { - /// // An infinite stream of CTRL-BREAK events. - /// let mut stream = ctrl_break()?; + /// // A listener of CTRL-BREAK events. + /// let mut signal = ctrl_break()?; /// /// // Print whenever a CTRL-BREAK event is received. /// loop { - /// stream.recv().await; + /// signal.recv().await; /// println!("got signal CTRL-BREAK"); /// } /// } @@ -166,7 +173,7 @@ impl CtrlBreak { /// Polls to receive the next signal notification event, outside of an /// `async` context. /// - /// `None` is returned if no more events can be received by this stream. + /// `None` is returned if no more events can be received by this listener. /// /// # Examples /// @@ -196,7 +203,7 @@ impl CtrlBreak { } } -/// Creates a new stream which receives "ctrl-break" notifications sent to the +/// Creates a new listener which receives "ctrl-break" notifications sent to the /// process. /// /// # Examples @@ -206,12 +213,12 @@ impl CtrlBreak { /// /// #[tokio::main] /// async fn main() -> Result<(), Box> { -/// // An infinite stream of CTRL-BREAK events. -/// let mut stream = ctrl_break()?; +/// // A listener of CTRL-BREAK events. +/// let mut signal = ctrl_break()?; /// /// // Print whenever a CTRL-BREAK event is received. /// loop { -/// stream.recv().await; +/// signal.recv().await; /// println!("got signal CTRL-BREAK"); /// } /// } @@ -221,3 +228,297 @@ pub fn ctrl_break() -> io::Result { inner: self::imp::ctrl_break()?, }) } + +/// Creates a new listener which receives "ctrl-close" notifications sent to the +/// process. +/// +/// # Examples +/// +/// ```rust,no_run +/// use tokio::signal::windows::ctrl_close; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box> { +/// // A listener of CTRL-CLOSE events. +/// let mut signal = ctrl_close()?; +/// +/// // Print whenever a CTRL-CLOSE event is received. +/// for countdown in (0..3).rev() { +/// signal.recv().await; +/// println!("got CTRL-CLOSE. {} more to exit", countdown); +/// } +/// +/// Ok(()) +/// } +/// ``` +pub fn ctrl_close() -> io::Result { + Ok(CtrlClose { + inner: self::imp::ctrl_close()?, + }) +} + +/// Represents a listener which receives "ctrl-close" notitifications sent to the process +/// via 'SetConsoleCtrlHandler'. +/// +/// A notification to this process notifies *all* listeners listening for +/// this event. Moreover, the notifications **are coalesced** if they aren't processed +/// quickly enough. This means that if two notifications are received back-to-back, +/// then the listener may only receive one item about the two notifications. +#[must_use = "listeners do nothing unless polled"] +#[derive(Debug)] +pub struct CtrlClose { + inner: RxFuture, +} + +impl CtrlClose { + /// Receives the next signal notification event. + /// + /// `None` is returned if no more events can be received by this listener. + /// + /// # Examples + /// + /// ```rust,no_run + /// use tokio::signal::windows::ctrl_close; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// // A listener of CTRL-CLOSE events. + /// let mut signal = ctrl_close()?; + /// + /// // Print whenever a CTRL-CLOSE event is received. + /// signal.recv().await; + /// println!("got CTRL-CLOSE. Cleaning up before exiting"); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv(&mut self) -> Option<()> { + self.inner.recv().await + } + + /// Polls to receive the next signal notification event, outside of an + /// `async` context. + /// + /// `None` is returned if no more events can be received by this listener. + /// + /// # Examples + /// + /// Polling from a manually implemented future + /// + /// ```rust,no_run + /// use std::pin::Pin; + /// use std::future::Future; + /// use std::task::{Context, Poll}; + /// use tokio::signal::windows::CtrlClose; + /// + /// struct MyFuture { + /// ctrl_close: CtrlClose, + /// } + /// + /// impl Future for MyFuture { + /// type Output = Option<()>; + /// + /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + /// println!("polling MyFuture"); + /// self.ctrl_close.poll_recv(cx) + /// } + /// } + /// ``` + pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_recv(cx) + } +} + +/// Creates a new listener which receives "ctrl-shutdown" notifications sent to the +/// process. +/// +/// # Examples +/// +/// ```rust,no_run +/// use tokio::signal::windows::ctrl_shutdown; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box> { +/// // A listener of CTRL-SHUTDOWN events. +/// let mut signal = ctrl_shutdown()?; +/// +/// signal.recv().await; +/// println!("got CTRL-SHUTDOWN. Cleaning up before exiting"); +/// +/// Ok(()) +/// } +/// ``` +pub fn ctrl_shutdown() -> io::Result { + Ok(CtrlShutdown { + inner: self::imp::ctrl_shutdown()?, + }) +} + +/// Represents a listener which receives "ctrl-shutdown" notitifications sent to the process +/// via 'SetConsoleCtrlHandler'. +/// +/// A notification to this process notifies *all* listeners listening for +/// this event. Moreover, the notifications **are coalesced** if they aren't processed +/// quickly enough. This means that if two notifications are received back-to-back, +/// then the listener may only receive one item about the two notifications. +#[must_use = "listeners do nothing unless polled"] +#[derive(Debug)] +pub struct CtrlShutdown { + inner: RxFuture, +} + +impl CtrlShutdown { + /// Receives the next signal notification event. + /// + /// `None` is returned if no more events can be received by this listener. + /// + /// # Examples + /// + /// ```rust,no_run + /// use tokio::signal::windows::ctrl_shutdown; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// // A listener of CTRL-SHUTDOWN events. + /// let mut signal = ctrl_shutdown()?; + /// + /// // Print whenever a CTRL-SHUTDOWN event is received. + /// signal.recv().await; + /// println!("got CTRL-SHUTDOWN. Cleaning up before exiting"); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv(&mut self) -> Option<()> { + self.inner.recv().await + } + + /// Polls to receive the next signal notification event, outside of an + /// `async` context. + /// + /// `None` is returned if no more events can be received by this listener. + /// + /// # Examples + /// + /// Polling from a manually implemented future + /// + /// ```rust,no_run + /// use std::pin::Pin; + /// use std::future::Future; + /// use std::task::{Context, Poll}; + /// use tokio::signal::windows::CtrlShutdown; + /// + /// struct MyFuture { + /// ctrl_shutdown: CtrlShutdown, + /// } + /// + /// impl Future for MyFuture { + /// type Output = Option<()>; + /// + /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + /// println!("polling MyFuture"); + /// self.ctrl_shutdown.poll_recv(cx) + /// } + /// } + /// ``` + pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_recv(cx) + } +} + +/// Creates a new listener which receives "ctrl-logoff" notifications sent to the +/// process. +/// +/// # Examples +/// +/// ```rust,no_run +/// use tokio::signal::windows::ctrl_logoff; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box> { +/// // A listener of CTRL-LOGOFF events. +/// let mut signal = ctrl_logoff()?; +/// +/// signal.recv().await; +/// println!("got CTRL-LOGOFF. Cleaning up before exiting"); +/// +/// Ok(()) +/// } +/// ``` +pub fn ctrl_logoff() -> io::Result { + Ok(CtrlLogoff { + inner: self::imp::ctrl_logoff()?, + }) +} + +/// Represents a listener which receives "ctrl-logoff" notitifications sent to the process +/// via 'SetConsoleCtrlHandler'. +/// +/// A notification to this process notifies *all* listeners listening for +/// this event. Moreover, the notifications **are coalesced** if they aren't processed +/// quickly enough. This means that if two notifications are received back-to-back, +/// then the listener may only receive one item about the two notifications. +#[must_use = "listeners do nothing unless polled"] +#[derive(Debug)] +pub struct CtrlLogoff { + inner: RxFuture, +} + +impl CtrlLogoff { + /// Receives the next signal notification event. + /// + /// `None` is returned if no more events can be received by this listener. + /// + /// # Examples + /// + /// ```rust,no_run + /// use tokio::signal::windows::ctrl_logoff; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// // An listener of CTRL-LOGOFF events. + /// let mut signal = ctrl_logoff()?; + /// + /// // Print whenever a CTRL-LOGOFF event is received. + /// signal.recv().await; + /// println!("got CTRL-LOGOFF. Cleaning up before exiting"); + /// + /// Ok(()) + /// } + /// ``` + pub async fn recv(&mut self) -> Option<()> { + self.inner.recv().await + } + + /// Polls to receive the next signal notification event, outside of an + /// `async` context. + /// + /// `None` is returned if no more events can be received by this listener. + /// + /// # Examples + /// + /// Polling from a manually implemented future + /// + /// ```rust,no_run + /// use std::pin::Pin; + /// use std::future::Future; + /// use std::task::{Context, Poll}; + /// use tokio::signal::windows::CtrlLogoff; + /// + /// struct MyFuture { + /// ctrl_logoff: CtrlLogoff, + /// } + /// + /// impl Future for MyFuture { + /// type Output = Option<()>; + /// + /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + /// println!("polling MyFuture"); + /// self.ctrl_logoff.poll_recv(cx) + /// } + /// } + /// ``` + pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_recv(cx) + } +} diff --git a/third_party/rust/tokio/src/signal/windows/stub.rs b/third_party/rust/tokio/src/signal/windows/stub.rs index 88630543da92..61df30979b69 100644 --- a/third_party/rust/tokio/src/signal/windows/stub.rs +++ b/third_party/rust/tokio/src/signal/windows/stub.rs @@ -4,10 +4,22 @@ use crate::signal::RxFuture; use std::io; +pub(super) fn ctrl_break() -> io::Result { + panic!() +} + +pub(super) fn ctrl_close() -> io::Result { + panic!() +} + pub(super) fn ctrl_c() -> io::Result { panic!() } -pub(super) fn ctrl_break() -> io::Result { +pub(super) fn ctrl_logoff() -> io::Result { + panic!() +} + +pub(super) fn ctrl_shutdown() -> io::Result { panic!() } diff --git a/third_party/rust/tokio/src/signal/windows/sys.rs b/third_party/rust/tokio/src/signal/windows/sys.rs index 8d29c357b63c..26e6bdf8182d 100644 --- a/third_party/rust/tokio/src/signal/windows/sys.rs +++ b/third_party/rust/tokio/src/signal/windows/sys.rs @@ -1,23 +1,33 @@ -use std::convert::TryFrom; use std::io; use std::sync::Once; use crate::signal::registry::{globals, EventId, EventInfo, Init, Storage}; use crate::signal::RxFuture; -use winapi::shared::minwindef::{BOOL, DWORD, FALSE, TRUE}; -use winapi::um::consoleapi::SetConsoleCtrlHandler; -use winapi::um::wincon::{CTRL_BREAK_EVENT, CTRL_C_EVENT}; - -pub(super) fn ctrl_c() -> io::Result { - new(CTRL_C_EVENT) -} +use windows_sys::Win32::Foundation::BOOL; +use windows_sys::Win32::System::Console as console; pub(super) fn ctrl_break() -> io::Result { - new(CTRL_BREAK_EVENT) + new(console::CTRL_BREAK_EVENT) } -fn new(signum: DWORD) -> io::Result { +pub(super) fn ctrl_close() -> io::Result { + new(console::CTRL_CLOSE_EVENT) +} + +pub(super) fn ctrl_c() -> io::Result { + new(console::CTRL_C_EVENT) +} + +pub(super) fn ctrl_logoff() -> io::Result { + new(console::CTRL_LOGOFF_EVENT) +} + +pub(super) fn ctrl_shutdown() -> io::Result { + new(console::CTRL_SHUTDOWN_EVENT) +} + +fn new(signum: u32) -> io::Result { global_init()?; let rx = globals().register_listener(signum as EventId); Ok(RxFuture::new(rx)) @@ -25,24 +35,33 @@ fn new(signum: DWORD) -> io::Result { #[derive(Debug)] pub(crate) struct OsStorage { - ctrl_c: EventInfo, ctrl_break: EventInfo, + ctrl_close: EventInfo, + ctrl_c: EventInfo, + ctrl_logoff: EventInfo, + ctrl_shutdown: EventInfo, } impl Init for OsStorage { fn init() -> Self { Self { - ctrl_c: EventInfo::default(), - ctrl_break: EventInfo::default(), + ctrl_break: Default::default(), + ctrl_close: Default::default(), + ctrl_c: Default::default(), + ctrl_logoff: Default::default(), + ctrl_shutdown: Default::default(), } } } impl Storage for OsStorage { fn event_info(&self, id: EventId) -> Option<&EventInfo> { - match DWORD::try_from(id) { - Ok(CTRL_C_EVENT) => Some(&self.ctrl_c), - Ok(CTRL_BREAK_EVENT) => Some(&self.ctrl_break), + match u32::try_from(id) { + Ok(console::CTRL_BREAK_EVENT) => Some(&self.ctrl_break), + Ok(console::CTRL_CLOSE_EVENT) => Some(&self.ctrl_close), + Ok(console::CTRL_C_EVENT) => Some(&self.ctrl_c), + Ok(console::CTRL_LOGOFF_EVENT) => Some(&self.ctrl_logoff), + Ok(console::CTRL_SHUTDOWN_EVENT) => Some(&self.ctrl_shutdown), _ => None, } } @@ -51,8 +70,11 @@ impl Storage for OsStorage { where F: FnMut(&'a EventInfo), { - f(&self.ctrl_c); f(&self.ctrl_break); + f(&self.ctrl_close); + f(&self.ctrl_c); + f(&self.ctrl_logoff); + f(&self.ctrl_shutdown); } } @@ -71,7 +93,7 @@ fn global_init() -> io::Result<()> { let mut init = None; INIT.call_once(|| unsafe { - let rc = SetConsoleCtrlHandler(Some(handler), TRUE); + let rc = console::SetConsoleCtrlHandler(Some(handler), 1); let ret = if rc == 0 { Err(io::Error::last_os_error()) } else { @@ -84,7 +106,7 @@ fn global_init() -> io::Result<()> { init.unwrap_or_else(|| Ok(())) } -unsafe extern "system" fn handler(ty: DWORD) -> BOOL { +unsafe extern "system" fn handler(ty: u32) -> BOOL { let globals = globals(); globals.record_event(ty as EventId); @@ -93,11 +115,11 @@ unsafe extern "system" fn handler(ty: DWORD) -> BOOL { // have the same restrictions as in Unix signal handlers, meaning we can // go ahead and perform the broadcast here. if globals.broadcast() { - TRUE + 1 } else { // No one is listening for this notification any more // let the OS fire the next (possibly the default) handler. - FALSE + 0 } } @@ -121,7 +143,7 @@ mod tests { // like sending signals on Unix, so we'll stub out the actual OS // integration and test that our handling works. unsafe { - super::handler(CTRL_C_EVENT); + super::handler(console::CTRL_C_EVENT); } assert_ready_ok!(ctrl_c.poll()); @@ -138,13 +160,67 @@ mod tests { // like sending signals on Unix, so we'll stub out the actual OS // integration and test that our handling works. unsafe { - super::handler(CTRL_BREAK_EVENT); + super::handler(console::CTRL_BREAK_EVENT); } ctrl_break.recv().await.unwrap(); }); } + #[test] + fn ctrl_close() { + let rt = rt(); + + rt.block_on(async { + let mut ctrl_close = assert_ok!(crate::signal::windows::ctrl_close()); + + // Windows doesn't have a good programmatic way of sending events + // like sending signals on Unix, so we'll stub out the actual OS + // integration and test that our handling works. + unsafe { + super::handler(console::CTRL_CLOSE_EVENT); + } + + ctrl_close.recv().await.unwrap(); + }); + } + + #[test] + fn ctrl_shutdown() { + let rt = rt(); + + rt.block_on(async { + let mut ctrl_shutdown = assert_ok!(crate::signal::windows::ctrl_shutdown()); + + // Windows doesn't have a good programmatic way of sending events + // like sending signals on Unix, so we'll stub out the actual OS + // integration and test that our handling works. + unsafe { + super::handler(console::CTRL_SHUTDOWN_EVENT); + } + + ctrl_shutdown.recv().await.unwrap(); + }); + } + + #[test] + fn ctrl_logoff() { + let rt = rt(); + + rt.block_on(async { + let mut ctrl_logoff = assert_ok!(crate::signal::windows::ctrl_logoff()); + + // Windows doesn't have a good programmatic way of sending events + // like sending signals on Unix, so we'll stub out the actual OS + // integration and test that our handling works. + unsafe { + super::handler(console::CTRL_LOGOFF_EVENT); + } + + ctrl_logoff.recv().await.unwrap(); + }); + } + fn rt() -> Runtime { crate::runtime::Builder::new_current_thread() .build() diff --git a/third_party/rust/tokio/src/sync/barrier.rs b/third_party/rust/tokio/src/sync/barrier.rs index dfc76a40ebf0..29b6d4e48481 100644 --- a/third_party/rust/tokio/src/sync/barrier.rs +++ b/third_party/rust/tokio/src/sync/barrier.rs @@ -105,7 +105,7 @@ impl Barrier { n, wait, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: resource_span, + resource_span, } } @@ -132,10 +132,12 @@ impl Barrier { return self.wait_internal().await; } async fn wait_internal(&self) -> BarrierWaitResult { + crate::trace::async_trace_leaf().await; + // NOTE: we are taking a _synchronous_ lock here. // It is okay to do so because the critical section is fast and never yields, so it cannot // deadlock even if another future is concurrently holding the lock. - // It is _desireable_ to do so as synchronous Mutexes are, at least in theory, faster than + // It is _desirable_ to do so as synchronous Mutexes are, at least in theory, faster than // the asynchronous counter-parts, so we should use them where possible [citation needed]. // NOTE: the extra scope here is so that the compiler doesn't think `state` is held across // a yield point, and thus marks the returned future as !Send. diff --git a/third_party/rust/tokio/src/sync/batch_semaphore.rs b/third_party/rust/tokio/src/sync/batch_semaphore.rs index 4f5effff3195..a762f799d56e 100644 --- a/third_party/rust/tokio/src/sync/batch_semaphore.rs +++ b/third_party/rust/tokio/src/sync/batch_semaphore.rs @@ -49,7 +49,7 @@ struct Waitlist { /// Error returned from the [`Semaphore::try_acquire`] function. /// /// [`Semaphore::try_acquire`]: crate::sync::Semaphore::try_acquire -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum TryAcquireError { /// The semaphore has been [closed] and cannot issue new permits. /// @@ -112,6 +112,14 @@ struct Waiter { _p: PhantomPinned, } +generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull) -> NonNull> { + &self.pointers + } + } +} + impl Semaphore { /// The maximum number of permits which a semaphore can hold. /// @@ -256,7 +264,7 @@ impl Semaphore { match self.permits.compare_exchange(curr, next, AcqRel, Acquire) { Ok(_) => { - // TODO: Instrument once issue has been solved} + // TODO: Instrument once issue has been solved return Ok(()); } Err(actual) => curr = actual, @@ -436,6 +444,7 @@ impl Semaphore { } assert_eq!(acquired, 0); + let mut old_waker = None; // Otherwise, register the waker & enqueue the node. node.waker.with_mut(|waker| { @@ -447,7 +456,7 @@ impl Semaphore { .map(|waker| !waker.will_wake(cx.waker())) .unwrap_or(true) { - *waker = Some(cx.waker().clone()); + old_waker = std::mem::replace(waker, Some(cx.waker().clone())); } }); @@ -460,6 +469,8 @@ impl Semaphore { waiters.queue.push_front(node); } + drop(waiters); + drop(old_waker); Pending } @@ -532,11 +543,11 @@ impl Future for Acquire<'_> { #[cfg(all(tokio_unstable, feature = "tracing"))] let coop = ready!(trace_poll_op!( "poll_acquire", - crate::coop::poll_proceed(cx), + crate::runtime::coop::poll_proceed(cx), )); #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let result = match semaphore.poll_acquire(cx, needed, node, *queued) { Pending => { @@ -582,7 +593,7 @@ impl<'a> Acquire<'a> { tracing::trace!( target: "runtime::resource::async_op::state_update", - permits_obtained = 0 as usize, + permits_obtained = 0usize, permits.op = "override", ); @@ -704,12 +715,6 @@ impl std::error::Error for TryAcquireError {} /// /// `Waiter` is forced to be !Unpin. unsafe impl linked_list::Link for Waiter { - // XXX: ideally, we would be able to use `Pin` here, to enforce the - // invariant that list entries may not move while in the list. However, we - // can't do this currently, as using `Pin<&'a mut Waiter>` as the `Handle` - // type would require `Semaphore` to be generic over a lifetime. We can't - // use `Pin<*mut Waiter>`, as raw pointers are `Unpin` regardless of whether - // or not they dereference to an `!Unpin` target. type Handle = NonNull; type Target = Waiter; @@ -721,7 +726,7 @@ unsafe impl linked_list::Link for Waiter { ptr } - unsafe fn pointers(mut target: NonNull) -> NonNull> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull) -> NonNull> { + Waiter::addr_of_pointers(target) } } diff --git a/third_party/rust/tokio/src/sync/broadcast.rs b/third_party/rust/tokio/src/sync/broadcast.rs index 0d9cd3bc176f..4b36452cec37 100644 --- a/third_party/rust/tokio/src/sync/broadcast.rs +++ b/third_party/rust/tokio/src/sync/broadcast.rs @@ -4,7 +4,7 @@ //! A [`Sender`] is used to broadcast values to **all** connected [`Receiver`] //! values. [`Sender`] handles are clone-able, allowing concurrent send and //! receive actions. [`Sender`] and [`Receiver`] are both `Send` and `Sync` as -//! long as `T` is also `Send` or `Sync` respectively. +//! long as `T` is `Send`. //! //! When a value is sent, **all** [`Receiver`] handles are notified and will //! receive the value. The value is stored once inside the channel and cloned on @@ -18,6 +18,9 @@ //! returned [`Receiver`] will receive values sent **after** the call to //! `subscribe`. //! +//! This channel is also suitable for the single-producer multi-consumer +//! use-case, where a single sender broadcasts values to many receivers. +//! //! ## Lagging //! //! As sent messages must be retained until **all** [`Receiver`] handles receive @@ -51,6 +54,10 @@ //! all values retained by the channel, the next call to [`recv`] will return //! with [`RecvError::Closed`]. //! +//! When a [`Receiver`] handle is dropped, any messages not read by the receiver +//! will be marked as read. If this receiver was the only one not to have read +//! that message, the message will be dropped at this point. +//! //! [`Sender`]: crate::sync::broadcast::Sender //! [`Sender::subscribe`]: crate::sync::broadcast::Sender::subscribe //! [`Receiver`]: crate::sync::broadcast::Receiver @@ -111,8 +118,9 @@ use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::AtomicUsize; -use crate::loom::sync::{Arc, Mutex, RwLock, RwLockReadGuard}; +use crate::loom::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard}; use crate::util::linked_list::{self, LinkedList}; +use crate::util::WakeList; use std::fmt; use std::future::Future; @@ -230,7 +238,7 @@ pub mod error { /// /// [`recv`]: crate::sync::broadcast::Receiver::recv /// [`Receiver`]: crate::sync::broadcast::Receiver - #[derive(Debug, PartialEq)] + #[derive(Debug, PartialEq, Eq, Clone)] pub enum RecvError { /// There are no more active senders implying no further messages will ever /// be sent. @@ -258,7 +266,7 @@ pub mod error { /// /// [`try_recv`]: crate::sync::broadcast::Receiver::try_recv /// [`Receiver`]: crate::sync::broadcast::Receiver - #[derive(Debug, PartialEq)] + #[derive(Debug, PartialEq, Eq, Clone)] pub enum TryRecvError { /// The channel is currently empty. There are still active /// [`Sender`] handles, so data may yet become available. @@ -336,9 +344,6 @@ struct Slot { /// Uniquely identifies the `send` stored in the slot. pos: u64, - /// True signals the channel is closed. - closed: bool, - /// The value being broadcast. /// /// The value is set by `send` when the write lock is held. When a reader @@ -361,6 +366,14 @@ struct Waiter { _p: PhantomPinned, } +generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull) -> NonNull> { + &self.pointers + } + } +} + struct RecvGuard<'a, T> { slot: RwLockReadGuard<'a, Slot>, } @@ -425,6 +438,12 @@ const MAX_RECEIVERS: usize = usize::MAX >> 2; /// tx.send(20).unwrap(); /// } /// ``` +/// +/// # Panics +/// +/// This will panic if `capacity` is equal to `0` or larger +/// than `usize::MAX / 2`. +#[track_caller] pub fn channel(mut capacity: usize) -> (Sender, Receiver) { assert!(capacity > 0, "capacity is empty"); assert!(capacity <= usize::MAX >> 1, "requested capacity too large"); @@ -438,7 +457,6 @@ pub fn channel(mut capacity: usize) -> (Sender, Receiver) { buffer.push(RwLock::new(Slot { rem: AtomicUsize::new(0), pos: (i as u64).wrapping_sub(capacity as u64), - closed: false, val: UnsafeCell::new(None), })); } @@ -523,8 +541,41 @@ impl Sender { /// } /// ``` pub fn send(&self, value: T) -> Result> { - self.send2(Some(value)) - .map_err(|SendError(maybe_v)| SendError(maybe_v.unwrap())) + let mut tail = self.shared.tail.lock(); + + if tail.rx_cnt == 0 { + return Err(SendError(value)); + } + + // Position to write into + let pos = tail.pos; + let rem = tail.rx_cnt; + let idx = (pos & self.shared.mask as u64) as usize; + + // Update the tail position + tail.pos = tail.pos.wrapping_add(1); + + // Get the slot + let mut slot = self.shared.buffer[idx].write().unwrap(); + + // Track the position + slot.pos = pos; + + // Set remaining receivers + slot.rem.with_mut(|v| *v = rem); + + // Write the value + slot.val = UnsafeCell::new(Some(value)); + + // Release the slot lock before notifying the receivers. + drop(slot); + + // Notify and release the mutex. This must happen after the slot lock is + // released, otherwise the writer lock bit could be cleared while another + // thread is in the critical section. + self.shared.notify_rx(tail); + + Ok(rem) } /// Creates a new [`Receiver`] handle that will receive values sent **after** @@ -555,6 +606,97 @@ impl Sender { new_receiver(shared) } + /// Returns the number of queued values. + /// + /// A value is queued until it has either been seen by all receivers that were alive at the time + /// it was sent, or has been evicted from the queue by subsequent sends that exceeded the + /// queue's capacity. + /// + /// # Note + /// + /// In contrast to [`Receiver::len`], this method only reports queued values and not values that + /// have been evicted from the queue before being seen by all receivers. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// let mut rx2 = tx.subscribe(); + /// + /// tx.send(10).unwrap(); + /// tx.send(20).unwrap(); + /// tx.send(30).unwrap(); + /// + /// assert_eq!(tx.len(), 3); + /// + /// rx1.recv().await.unwrap(); + /// + /// // The len is still 3 since rx2 hasn't seen the first value yet. + /// assert_eq!(tx.len(), 3); + /// + /// rx2.recv().await.unwrap(); + /// + /// assert_eq!(tx.len(), 2); + /// } + /// ``` + pub fn len(&self) -> usize { + let tail = self.shared.tail.lock(); + + let base_idx = (tail.pos & self.shared.mask as u64) as usize; + let mut low = 0; + let mut high = self.shared.buffer.len(); + while low < high { + let mid = low + (high - low) / 2; + let idx = base_idx.wrapping_add(mid) & self.shared.mask; + if self.shared.buffer[idx].read().unwrap().rem.load(SeqCst) == 0 { + low = mid + 1; + } else { + high = mid; + } + } + + self.shared.buffer.len() - low + } + + /// Returns true if there are no queued values. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// let mut rx2 = tx.subscribe(); + /// + /// assert!(tx.is_empty()); + /// + /// tx.send(10).unwrap(); + /// + /// assert!(!tx.is_empty()); + /// + /// rx1.recv().await.unwrap(); + /// + /// // The queue is still not empty since rx2 hasn't seen the value. + /// assert!(!tx.is_empty()); + /// + /// rx2.recv().await.unwrap(); + /// + /// assert!(tx.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + let tail = self.shared.tail.lock(); + + let idx = (tail.pos.wrapping_sub(1) & self.shared.mask as u64) as usize; + self.shared.buffer[idx].read().unwrap().rem.load(SeqCst) == 0 + } + /// Returns the number of active receivers /// /// An active receiver is a [`Receiver`] handle returned from [`channel`] or @@ -596,52 +738,38 @@ impl Sender { tail.rx_cnt } - fn send2(&self, value: Option) -> Result>> { + /// Returns `true` if senders belong to the same channel. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, _rx) = broadcast::channel::<()>(16); + /// let tx2 = tx.clone(); + /// + /// assert!(tx.same_channel(&tx2)); + /// + /// let (tx3, _rx3) = broadcast::channel::<()>(16); + /// + /// assert!(!tx3.same_channel(&tx2)); + /// } + /// ``` + pub fn same_channel(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.shared, &other.shared) + } + + fn close_channel(&self) { let mut tail = self.shared.tail.lock(); + tail.closed = true; - if tail.rx_cnt == 0 { - return Err(SendError(value)); - } - - // Position to write into - let pos = tail.pos; - let rem = tail.rx_cnt; - let idx = (pos & self.shared.mask as u64) as usize; - - // Update the tail position - tail.pos = tail.pos.wrapping_add(1); - - // Get the slot - let mut slot = self.shared.buffer[idx].write().unwrap(); - - // Track the position - slot.pos = pos; - - // Set remaining receivers - slot.rem.with_mut(|v| *v = rem); - - // Set the closed bit if the value is `None`; otherwise write the value - if value.is_none() { - tail.closed = true; - slot.closed = true; - } else { - slot.val.with_mut(|ptr| unsafe { *ptr = value }); - } - - // Release the slot lock before notifying the receivers. - drop(slot); - - tail.notify_rx(); - - // Release the mutex. This must happen after the slot lock is released, - // otherwise the writer lock bit could be cleared while another thread - // is in the critical section. - drop(tail); - - Ok(rem) + self.shared.notify_rx(tail); } } +/// Create a new `Receiver` which reads starting from the tail. fn new_receiver(shared: Arc>) -> Receiver { let mut tail = shared.tail.lock(); @@ -658,18 +786,47 @@ fn new_receiver(shared: Arc>) -> Receiver { Receiver { shared, next } } -impl Tail { - fn notify_rx(&mut self) { - while let Some(mut waiter) = self.waiters.pop_back() { - // Safety: `waiters` lock is still held. - let waiter = unsafe { waiter.as_mut() }; +impl Shared { + fn notify_rx<'a, 'b: 'a>(&'b self, mut tail: MutexGuard<'a, Tail>) { + let mut wakers = WakeList::new(); + 'outer: loop { + while wakers.can_push() { + match tail.waiters.pop_back() { + Some(mut waiter) => { + // Safety: `tail` lock is still held. + let waiter = unsafe { waiter.as_mut() }; - assert!(waiter.queued); - waiter.queued = false; + assert!(waiter.queued); + waiter.queued = false; - let waker = waiter.waker.take().unwrap(); - waker.wake(); + if let Some(waker) = waiter.waker.take() { + wakers.push(waker); + } + } + None => { + break 'outer; + } + } + } + + // Release the lock before waking. + drop(tail); + + // Before we acquire the lock again all sorts of things can happen: + // some waiters may remove themselves from the list and new waiters + // may be added. This is fine since at worst we will unnecessarily + // wake up waiters which will then queue themselves again. + + wakers.wake_all(); + + // Acquire the lock again. + tail = self.tail.lock(); } + + // Release the lock before waking. + drop(tail); + + wakers.wake_all(); } } @@ -685,12 +842,102 @@ impl Clone for Sender { impl Drop for Sender { fn drop(&mut self) { if 1 == self.shared.num_tx.fetch_sub(1, SeqCst) { - let _ = self.send2(None); + self.close_channel(); } } } impl Receiver { + /// Returns the number of messages that were sent into the channel and that + /// this [`Receiver`] has yet to receive. + /// + /// If the returned value from `len` is larger than the next largest power of 2 + /// of the capacity of the channel any call to [`recv`] will return an + /// `Err(RecvError::Lagged)` and any call to [`try_recv`] will return an + /// `Err(TryRecvError::Lagged)`, e.g. if the capacity of the channel is 10, + /// [`recv`] will start to return `Err(RecvError::Lagged)` once `len` returns + /// values larger than 16. + /// + /// [`Receiver`]: crate::sync::broadcast::Receiver + /// [`recv`]: crate::sync::broadcast::Receiver::recv + /// [`try_recv`]: crate::sync::broadcast::Receiver::try_recv + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// + /// tx.send(10).unwrap(); + /// tx.send(20).unwrap(); + /// + /// assert_eq!(rx1.len(), 2); + /// assert_eq!(rx1.recv().await.unwrap(), 10); + /// assert_eq!(rx1.len(), 1); + /// assert_eq!(rx1.recv().await.unwrap(), 20); + /// assert_eq!(rx1.len(), 0); + /// } + /// ``` + pub fn len(&self) -> usize { + let next_send_pos = self.shared.tail.lock().pos; + (next_send_pos - self.next) as usize + } + + /// Returns true if there aren't any messages in the channel that the [`Receiver`] + /// has yet to receive. + /// + /// [`Receiver]: create::sync::broadcast::Receiver + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx1) = broadcast::channel(16); + /// + /// assert!(rx1.is_empty()); + /// + /// tx.send(10).unwrap(); + /// tx.send(20).unwrap(); + /// + /// assert!(!rx1.is_empty()); + /// assert_eq!(rx1.recv().await.unwrap(), 10); + /// assert_eq!(rx1.recv().await.unwrap(), 20); + /// assert!(rx1.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns `true` if receivers belong to the same channel. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, rx) = broadcast::channel::<()>(16); + /// let rx2 = tx.subscribe(); + /// + /// assert!(rx.same_channel(&rx2)); + /// + /// let (_tx3, rx3) = broadcast::channel::<()>(16); + /// + /// assert!(!rx3.same_channel(&rx2)); + /// } + /// ``` + pub fn same_channel(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.shared, &other.shared) + } + /// Locks the next value if there is one. fn recv_ref( &mut self, @@ -702,14 +949,6 @@ impl Receiver { let mut slot = self.shared.buffer[idx].read().unwrap(); if slot.pos != self.next { - let next_pos = slot.pos.wrapping_add(self.shared.buffer.len() as u64); - - // The receiver has read all current values in the channel and there - // is no waiter to register - if waiter.is_none() && next_pos == self.next { - return Err(TryRecvError::Empty); - } - // Release the `slot` lock before attempting to acquire the `tail` // lock. This is required because `send2` acquires the tail lock // first followed by the slot lock. Acquiring the locks in reverse @@ -719,6 +958,8 @@ impl Receiver { // the slot lock. drop(slot); + let mut old_waker = None; + let mut tail = self.shared.tail.lock(); // Acquire slot lock again @@ -731,6 +972,13 @@ impl Receiver { let next_pos = slot.pos.wrapping_add(self.shared.buffer.len() as u64); if next_pos == self.next { + // At this point the channel is empty for *this* receiver. If + // it's been closed, then that's what we return, otherwise we + // set a waker and return empty. + if tail.closed { + return Err(TryRecvError::Closed); + } + // Store the waker if let Some((waiter, waker)) = waiter { // Safety: called while locked. @@ -744,7 +992,10 @@ impl Receiver { match (*ptr).waker { Some(ref w) if w.will_wake(waker) => {} _ => { - (*ptr).waker = Some(waker.clone()); + old_waker = std::mem::replace( + &mut (*ptr).waker, + Some(waker.clone()), + ); } } @@ -756,6 +1007,11 @@ impl Receiver { } } + // Drop the old waker after releasing the locks. + drop(slot); + drop(tail); + drop(old_waker); + return Err(TryRecvError::Empty); } @@ -764,22 +1020,7 @@ impl Receiver { // catch up by skipping dropped messages and setting the // internal cursor to the **oldest** message stored by the // channel. - // - // However, finding the oldest position is a bit more - // complicated than `tail-position - buffer-size`. When - // the channel is closed, the tail position is incremented to - // signal a new `None` message, but `None` is not stored in the - // channel itself (see issue #2425 for why). - // - // To account for this, if the channel is closed, the tail - // position is decremented by `buffer-size + 1`. - let mut adjust = 0; - if tail.closed { - adjust = 1 - } - let next = tail - .pos - .wrapping_sub(self.shared.buffer.len() as u64 + adjust); + let next = tail.pos.wrapping_sub(self.shared.buffer.len() as u64); let missed = next.wrapping_sub(self.next); @@ -800,15 +1041,38 @@ impl Receiver { self.next = self.next.wrapping_add(1); - if slot.closed { - return Err(TryRecvError::Closed); - } - Ok(RecvGuard { slot }) } } impl Receiver { + /// Re-subscribes to the channel starting from the current tail element. + /// + /// This [`Receiver`] handle will receive a clone of all values sent + /// **after** it has resubscribed. This will not include elements that are + /// in the queue of the current receiver. Consider the following example. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx) = broadcast::channel(2); + /// + /// tx.send(1).unwrap(); + /// let mut rx2 = rx.resubscribe(); + /// tx.send(2).unwrap(); + /// + /// assert_eq!(rx2.recv().await.unwrap(), 2); + /// assert_eq!(rx.recv().await.unwrap(), 1); + /// } + /// ``` + pub fn resubscribe(&self) -> Self { + let shared = self.shared.clone(); + new_receiver(shared) + } /// Receives the next value for this receiver. /// /// Each [`Receiver`] handle will receive a clone of all values sent @@ -930,6 +1194,33 @@ impl Receiver { let guard = self.recv_ref(None)?; guard.clone_value().ok_or(TryRecvError::Closed) } + + /// Blocking receive to call outside of asynchronous contexts. + /// + /// # Panics + /// + /// This function panics if called within an asynchronous execution + /// context. + /// + /// # Examples + /// ``` + /// use std::thread; + /// use tokio::sync::broadcast; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx) = broadcast::channel(16); + /// + /// let sync_code = thread::spawn(move || { + /// assert_eq!(rx.blocking_recv(), Ok(10)); + /// }); + /// + /// let _ = tx.send(10); + /// sync_code.join().unwrap(); + /// } + pub fn blocking_recv(&mut self) -> Result { + crate::future::block_on(self.recv()) + } } impl Drop for Receiver { @@ -988,6 +1279,8 @@ where type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); + let (receiver, waiter) = self.project(); let guard = match receiver.recv_ref(Some((waiter, cx.waker()))) { @@ -1039,8 +1332,8 @@ unsafe impl linked_list::Link for Waiter { ptr } - unsafe fn pointers(mut target: NonNull) -> NonNull> { - NonNull::from(&mut target.as_mut().pointers) + unsafe fn pointers(target: NonNull) -> NonNull> { + Waiter::addr_of_pointers(target) } } diff --git a/third_party/rust/tokio/src/sync/mod.rs b/third_party/rust/tokio/src/sync/mod.rs index 457e6ab29467..70fd9b9e32db 100644 --- a/third_party/rust/tokio/src/sync/mod.rs +++ b/third_party/rust/tokio/src/sync/mod.rs @@ -94,6 +94,10 @@ //! producers to a single consumer. This channel is often used to send work to a //! task or to receive the result of many computations. //! +//! This is also the channel you should use if you want to send many messages +//! from a single producer to a single consumer. There is no dedicated spsc +//! channel. +//! //! **Example:** using an mpsc to incrementally stream the results of a series //! of computations. //! @@ -244,6 +248,10 @@ //! This channel tends to be used less often than `oneshot` and `mpsc` but still //! has its use cases. //! +//! This is also the channel you should use if you want to broadcast values from +//! a single producer to many consumers. There is no dedicated spmc broadcast +//! channel. +//! //! Basic usage //! //! ``` @@ -441,7 +449,7 @@ cfg_sync! { pub mod mpsc; mod mutex; - pub use mutex::{Mutex, MutexGuard, TryLockError, OwnedMutexGuard, MappedMutexGuard}; + pub use mutex::{Mutex, MutexGuard, TryLockError, OwnedMutexGuard, MappedMutexGuard, OwnedMappedMutexGuard}; pub(crate) mod notify; pub use notify::Notify; diff --git a/third_party/rust/tokio/src/sync/mpsc/block.rs b/third_party/rust/tokio/src/sync/mpsc/block.rs index 58f4a9f6cc31..39c3e1be2d9f 100644 --- a/third_party/rust/tokio/src/sync/mpsc/block.rs +++ b/third_party/rust/tokio/src/sync/mpsc/block.rs @@ -1,6 +1,7 @@ use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize}; +use std::alloc::Layout; use std::mem::MaybeUninit; use std::ops; use std::ptr::{self, NonNull}; @@ -10,6 +11,17 @@ use std::sync::atomic::Ordering::{self, AcqRel, Acquire, Release}; /// /// Each block in the list can hold up to `BLOCK_CAP` messages. pub(crate) struct Block { + /// The header fields. + header: BlockHeader, + + /// Array containing values pushed into the block. Values are stored in a + /// continuous array in order to improve cache line behavior when reading. + /// The values must be manually dropped. + values: Values, +} + +/// Extra fields for a `Block`. +struct BlockHeader { /// The start index of this block. /// /// Slots in this block have indices in `start_index .. start_index + BLOCK_CAP`. @@ -24,11 +36,6 @@ pub(crate) struct Block { /// The observed `tail_position` value *after* the block has been passed by /// `block_tail`. observed_tail_position: UnsafeCell, - - /// Array containing values pushed into the block. Values are stored in a - /// continuous array in order to improve cache line behavior when reading. - /// The values must be manually dropped. - values: Values, } pub(crate) enum Read { @@ -36,6 +43,7 @@ pub(crate) enum Read { Closed, } +#[repr(transparent)] struct Values([UnsafeCell>; BLOCK_CAP]); use super::BLOCK_CAP; @@ -71,28 +79,56 @@ pub(crate) fn offset(slot_index: usize) -> usize { SLOT_MASK & slot_index } +generate_addr_of_methods! { + impl Block { + unsafe fn addr_of_header(self: NonNull) -> NonNull> { + &self.header + } + + unsafe fn addr_of_values(self: NonNull) -> NonNull> { + &self.values + } + } +} + impl Block { - pub(crate) fn new(start_index: usize) -> Block { - Block { - // The absolute index in the channel of the first slot in the block. - start_index, + pub(crate) fn new(start_index: usize) -> Box> { + unsafe { + // Allocate the block on the heap. + // SAFETY: The size of the Block is non-zero, since it is at least the size of the header. + let block = std::alloc::alloc(Layout::new::>()) as *mut Block; + let block = match NonNull::new(block) { + Some(block) => block, + None => std::alloc::handle_alloc_error(Layout::new::>()), + }; - // Pointer to the next block in the linked list. - next: AtomicPtr::new(ptr::null_mut()), + // Write the header to the block. + Block::addr_of_header(block).as_ptr().write(BlockHeader { + // The absolute index in the channel of the first slot in the block. + start_index, - ready_slots: AtomicUsize::new(0), + // Pointer to the next block in the linked list. + next: AtomicPtr::new(ptr::null_mut()), - observed_tail_position: UnsafeCell::new(0), + ready_slots: AtomicUsize::new(0), - // Value storage - values: unsafe { Values::uninitialized() }, + observed_tail_position: UnsafeCell::new(0), + }); + + // Initialize the values array. + Values::initialize(Block::addr_of_values(block)); + + // Convert the pointer to a `Box`. + // Safety: The raw pointer was allocated using the global allocator, and with + // the layout for a `Block`, so it's valid to convert it to box. + Box::from_raw(block.as_ptr()) } } /// Returns `true` if the block matches the given index. pub(crate) fn is_at_index(&self, index: usize) -> bool { debug_assert!(offset(index) == 0); - self.start_index == index + self.header.start_index == index } /// Returns the number of blocks between `self` and the block at the @@ -101,7 +137,7 @@ impl Block { /// `start_index` must represent a block *after* `self`. pub(crate) fn distance(&self, other_index: usize) -> usize { debug_assert!(offset(other_index) == 0); - other_index.wrapping_sub(self.start_index) / BLOCK_CAP + other_index.wrapping_sub(self.header.start_index) / BLOCK_CAP } /// Reads the value at the given offset. @@ -116,7 +152,7 @@ impl Block { pub(crate) unsafe fn read(&self, slot_index: usize) -> Option> { let offset = offset(slot_index); - let ready_bits = self.ready_slots.load(Acquire); + let ready_bits = self.header.ready_slots.load(Acquire); if !is_ready(ready_bits, offset) { if is_tx_closed(ready_bits) { @@ -156,7 +192,7 @@ impl Block { /// Signal to the receiver that the sender half of the list is closed. pub(crate) unsafe fn tx_close(&self) { - self.ready_slots.fetch_or(TX_CLOSED, Release); + self.header.ready_slots.fetch_or(TX_CLOSED, Release); } /// Resets the block to a blank state. This enables reusing blocks in the @@ -169,9 +205,9 @@ impl Block { /// * All slots are empty. /// * The caller holds a unique pointer to the block. pub(crate) unsafe fn reclaim(&mut self) { - self.start_index = 0; - self.next = AtomicPtr::new(ptr::null_mut()); - self.ready_slots = AtomicUsize::new(0); + self.header.start_index = 0; + self.header.next = AtomicPtr::new(ptr::null_mut()); + self.header.ready_slots = AtomicUsize::new(0); } /// Releases the block to the rx half for freeing. @@ -187,19 +223,20 @@ impl Block { pub(crate) unsafe fn tx_release(&self, tail_position: usize) { // Track the observed tail_position. Any sender targeting a greater // tail_position is guaranteed to not access this block. - self.observed_tail_position + self.header + .observed_tail_position .with_mut(|ptr| *ptr = tail_position); // Set the released bit, signalling to the receiver that it is safe to // free the block's memory as soon as all slots **prior** to // `observed_tail_position` have been filled. - self.ready_slots.fetch_or(RELEASED, Release); + self.header.ready_slots.fetch_or(RELEASED, Release); } /// Mark a slot as ready fn set_ready(&self, slot: usize) { let mask = 1 << slot; - self.ready_slots.fetch_or(mask, Release); + self.header.ready_slots.fetch_or(mask, Release); } /// Returns `true` when all slots have their `ready` bits set. @@ -214,25 +251,31 @@ impl Block { /// single atomic cell. However, this could have negative impact on cache /// behavior as there would be many more mutations to a single slot. pub(crate) fn is_final(&self) -> bool { - self.ready_slots.load(Acquire) & READY_MASK == READY_MASK + self.header.ready_slots.load(Acquire) & READY_MASK == READY_MASK } /// Returns the `observed_tail_position` value, if set pub(crate) fn observed_tail_position(&self) -> Option { - if 0 == RELEASED & self.ready_slots.load(Acquire) { + if 0 == RELEASED & self.header.ready_slots.load(Acquire) { None } else { - Some(self.observed_tail_position.with(|ptr| unsafe { *ptr })) + Some( + self.header + .observed_tail_position + .with(|ptr| unsafe { *ptr }), + ) } } /// Loads the next block pub(crate) fn load_next(&self, ordering: Ordering) -> Option>> { - let ret = NonNull::new(self.next.load(ordering)); + let ret = NonNull::new(self.header.next.load(ordering)); debug_assert!(unsafe { - ret.map(|block| block.as_ref().start_index == self.start_index.wrapping_add(BLOCK_CAP)) - .unwrap_or(true) + ret.map(|block| { + block.as_ref().header.start_index == self.header.start_index.wrapping_add(BLOCK_CAP) + }) + .unwrap_or(true) }); ret @@ -260,9 +303,10 @@ impl Block { success: Ordering, failure: Ordering, ) -> Result<(), NonNull>> { - block.as_mut().start_index = self.start_index.wrapping_add(BLOCK_CAP); + block.as_mut().header.start_index = self.header.start_index.wrapping_add(BLOCK_CAP); let next_ptr = self + .header .next .compare_exchange(ptr::null_mut(), block.as_ptr(), success, failure) .unwrap_or_else(|x| x); @@ -291,7 +335,7 @@ impl Block { // Create the new block. It is assumed that the block will become the // next one after `&self`. If this turns out to not be the case, // `start_index` is updated accordingly. - let new_block = Box::new(Block::new(self.start_index + BLOCK_CAP)); + let new_block = Block::new(self.header.start_index + BLOCK_CAP); let mut new_block = unsafe { NonNull::new_unchecked(Box::into_raw(new_block)) }; @@ -308,7 +352,8 @@ impl Block { // `Release` ensures that the newly allocated block is available to // other threads acquiring the next pointer. let next = NonNull::new( - self.next + self.header + .next .compare_exchange(ptr::null_mut(), new_block.as_ptr(), AcqRel, Acquire) .unwrap_or_else(|x| x), ); @@ -360,19 +405,20 @@ fn is_tx_closed(bits: usize) -> bool { } impl Values { - unsafe fn uninitialized() -> Values { - let mut vals = MaybeUninit::uninit(); - + /// Initialize a `Values` struct from a pointer. + /// + /// # Safety + /// + /// The raw pointer must be valid for writing a `Values`. + unsafe fn initialize(_value: NonNull>) { // When fuzzing, `UnsafeCell` needs to be initialized. if_loom! { - let p = vals.as_mut_ptr() as *mut UnsafeCell>; + let p = _value.as_ptr() as *mut UnsafeCell>; for i in 0..BLOCK_CAP { p.add(i) .write(UnsafeCell::new(MaybeUninit::uninit())); } } - - Values(vals.assume_init()) } } @@ -383,3 +429,20 @@ impl ops::Index for Values { self.0.index(index) } } + +#[cfg(all(test, not(loom)))] +#[test] +fn assert_no_stack_overflow() { + // https://github.com/tokio-rs/tokio/issues/5293 + + struct Foo { + _a: [u8; 2_000_000], + } + + assert_eq!( + Layout::new::>>(), + Layout::new::>() + ); + + let _block = Block::::new(0); +} diff --git a/third_party/rust/tokio/src/sync/mpsc/bounded.rs b/third_party/rust/tokio/src/sync/mpsc/bounded.rs index ddded8ebb31e..e870ae5f42eb 100644 --- a/third_party/rust/tokio/src/sync/mpsc/bounded.rs +++ b/third_party/rust/tokio/src/sync/mpsc/bounded.rs @@ -1,3 +1,4 @@ +use crate::loom::sync::Arc; use crate::sync::batch_semaphore::{self as semaphore, TryAcquireError}; use crate::sync::mpsc::chan; use crate::sync::mpsc::error::{SendError, TryRecvError, TrySendError}; @@ -17,11 +18,45 @@ use std::task::{Context, Poll}; /// To convert the `Sender` into a `Sink` or use it in a poll function, you can /// use the [`PollSender`] utility. /// -/// [`PollSender`]: https://docs.rs/tokio-util/0.6/tokio_util/sync/struct.PollSender.html +/// [`PollSender`]: https://docs.rs/tokio-util/latest/tokio_util/sync/struct.PollSender.html pub struct Sender { chan: chan::Tx, } +/// A sender that does not prevent the channel from being closed. +/// +/// If all [`Sender`] instances of a channel were dropped and only `WeakSender` +/// instances remain, the channel is closed. +/// +/// In order to send messages, the `WeakSender` needs to be upgraded using +/// [`WeakSender::upgrade`], which returns `Option`. It returns `None` +/// if all `Sender`s have been dropped, and otherwise it returns a `Sender`. +/// +/// [`Sender`]: Sender +/// [`WeakSender::upgrade`]: WeakSender::upgrade +/// +/// #Examples +/// +/// ``` +/// use tokio::sync::mpsc::channel; +/// +/// #[tokio::main] +/// async fn main() { +/// let (tx, _rx) = channel::(15); +/// let tx_weak = tx.downgrade(); +/// +/// // Upgrading will succeed because `tx` still exists. +/// assert!(tx_weak.upgrade().is_some()); +/// +/// // If we drop `tx`, then it will fail. +/// drop(tx); +/// assert!(tx_weak.clone().upgrade().is_none()); +/// } +/// ``` +pub struct WeakSender { + chan: Arc>, +} + /// Permits to send one value into the channel. /// /// `Permit` values are returned by [`Sender::reserve()`] and [`Sender::try_reserve()`] @@ -105,9 +140,13 @@ pub struct Receiver { /// } /// } /// ``` +#[track_caller] pub fn channel(buffer: usize) -> (Sender, Receiver) { assert!(buffer > 0, "mpsc bounded channel requires buffer > 0"); - let semaphore = (semaphore::Semaphore::new(buffer), buffer); + let semaphore = Semaphore { + semaphore: semaphore::Semaphore::new(buffer), + bound: buffer, + }; let (tx, rx) = chan::channel(semaphore); let tx = Sender::new(tx); @@ -118,7 +157,11 @@ pub fn channel(buffer: usize) -> (Sender, Receiver) { /// Channel semaphore is a tuple of the semaphore implementation and a `usize` /// representing the channel bound. -type Semaphore = (semaphore::Semaphore, usize); +#[derive(Debug)] +pub(crate) struct Semaphore { + pub(crate) semaphore: semaphore::Semaphore, + pub(crate) bound: usize, +} impl Receiver { pub(crate) fn new(chan: chan::Rx) -> Receiver { @@ -281,7 +324,9 @@ impl Receiver { /// sync_code.join().unwrap() /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] + #[cfg_attr(docsrs, doc(alias = "recv_blocking"))] pub fn blocking_recv(&mut self) -> Option { crate::future::block_on(self.recv()) } @@ -535,7 +580,7 @@ impl Sender { /// } /// ``` pub fn try_send(&self, message: T) -> Result<(), TrySendError> { - match self.chan.semaphore().0.try_acquire(1) { + match self.chan.semaphore().semaphore.try_acquire(1) { Ok(_) => {} Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(message)), Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(message)), @@ -650,7 +695,9 @@ impl Sender { /// sync_code.join().unwrap() /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] + #[cfg_attr(docsrs, doc(alias = "send_blocking"))] pub fn blocking_send(&self, value: T) -> Result<(), SendError> { crate::future::block_on(self.send(value)) } @@ -814,7 +861,9 @@ impl Sender { } async fn reserve_inner(&self) -> Result<(), SendError<()>> { - match self.chan.semaphore().0.acquire(1).await { + crate::trace::async_trace_leaf().await; + + match self.chan.semaphore().semaphore.acquire(1).await { Ok(_) => Ok(()), Err(_) => Err(SendError(())), } @@ -864,7 +913,7 @@ impl Sender { /// } /// ``` pub fn try_reserve(&self) -> Result, TrySendError<()>> { - match self.chan.semaphore().0.try_acquire(1) { + match self.chan.semaphore().semaphore.try_acquire(1) { Ok(_) => {} Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(())), Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(())), @@ -929,7 +978,7 @@ impl Sender { /// } /// ``` pub fn try_reserve_owned(self) -> Result, TrySendError> { - match self.chan.semaphore().0.try_acquire(1) { + match self.chan.semaphore().semaphore.try_acquire(1) { Ok(_) => {} Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(self)), Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(self)), @@ -960,6 +1009,8 @@ impl Sender { /// /// The capacity goes down when sending a value by calling [`send`] or by reserving capacity /// with [`reserve`]. The capacity goes up when values are received by the [`Receiver`]. + /// This is distinct from [`max_capacity`], which always returns buffer capacity initially + /// specified when calling [`channel`] /// /// # Examples /// @@ -985,8 +1036,56 @@ impl Sender { /// /// [`send`]: Sender::send /// [`reserve`]: Sender::reserve + /// [`channel`]: channel + /// [`max_capacity`]: Sender::max_capacity pub fn capacity(&self) -> usize { - self.chan.semaphore().0.available_permits() + self.chan.semaphore().semaphore.available_permits() + } + + /// Converts the `Sender` to a [`WeakSender`] that does not count + /// towards RAII semantics, i.e. if all `Sender` instances of the + /// channel were dropped and only `WeakSender` instances remain, + /// the channel is closed. + pub fn downgrade(&self) -> WeakSender { + WeakSender { + chan: self.chan.downgrade(), + } + } + + /// Returns the maximum buffer capacity of the channel. + /// + /// The maximum capacity is the buffer capacity initially specified when calling + /// [`channel`]. This is distinct from [`capacity`], which returns the *current* + /// available buffer capacity: as messages are sent and received, the + /// value returned by [`capacity`] will go up or down, whereas the value + /// returned by `max_capacity` will remain constant. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::mpsc; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, _rx) = mpsc::channel::<()>(5); + /// + /// // both max capacity and capacity are the same at first + /// assert_eq!(tx.max_capacity(), 5); + /// assert_eq!(tx.capacity(), 5); + /// + /// // Making a reservation doesn't change the max capacity. + /// let permit = tx.reserve().await.unwrap(); + /// assert_eq!(tx.max_capacity(), 5); + /// // but drops the capacity by one + /// assert_eq!(tx.capacity(), 4); + /// } + /// ``` + /// + /// [`channel`]: channel + /// [`max_capacity`]: Sender::max_capacity + /// [`capacity`]: Sender::capacity + pub fn max_capacity(&self) -> usize { + self.chan.semaphore().bound } } @@ -1006,6 +1105,29 @@ impl fmt::Debug for Sender { } } +impl Clone for WeakSender { + fn clone(&self) -> Self { + WeakSender { + chan: self.chan.clone(), + } + } +} + +impl WeakSender { + /// Tries to convert a WeakSender into a [`Sender`]. This will return `Some` + /// if there are other `Sender` instances alive and the channel wasn't + /// previously dropped, otherwise `None` is returned. + pub fn upgrade(&self) -> Option> { + chan::Tx::upgrade(self.chan.clone()).map(Sender::new) + } +} + +impl fmt::Debug for WeakSender { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("WeakSender").finish() + } +} + // ===== impl Permit ===== impl Permit<'_, T> { diff --git a/third_party/rust/tokio/src/sync/mpsc/chan.rs b/third_party/rust/tokio/src/sync/mpsc/chan.rs index c3007de89c71..6f87715dd34f 100644 --- a/third_party/rust/tokio/src/sync/mpsc/chan.rs +++ b/third_party/rust/tokio/src/sync/mpsc/chan.rs @@ -2,15 +2,14 @@ use crate::loom::cell::UnsafeCell; use crate::loom::future::AtomicWaker; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::Arc; -use crate::park::thread::CachedParkThread; -use crate::park::Park; +use crate::runtime::park::CachedParkThread; use crate::sync::mpsc::error::TryRecvError; -use crate::sync::mpsc::list; +use crate::sync::mpsc::{bounded, list, unbounded}; use crate::sync::notify::Notify; use std::fmt; use std::process; -use std::sync::atomic::Ordering::{AcqRel, Relaxed}; +use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use std::task::Poll::{Pending, Ready}; use std::task::{Context, Poll}; @@ -46,7 +45,7 @@ pub(crate) trait Semaphore { fn is_closed(&self) -> bool; } -struct Chan { +pub(super) struct Chan { /// Notifies all tasks listening for the receiver being dropped. notify_rx_closed: Notify, @@ -129,6 +128,30 @@ impl Tx { Tx { inner: chan } } + pub(super) fn downgrade(&self) -> Arc> { + self.inner.clone() + } + + // Returns the upgraded channel or None if the upgrade failed. + pub(super) fn upgrade(chan: Arc>) -> Option { + let mut tx_count = chan.tx_count.load(Acquire); + + loop { + if tx_count == 0 { + // channel is closed + return None; + } + + match chan + .tx_count + .compare_exchange_weak(tx_count, tx_count + 1, AcqRel, Acquire) + { + Ok(_) => return Some(Tx { inner: chan }), + Err(prev_count) => tx_count = prev_count, + } + } + } + pub(super) fn semaphore(&self) -> &S { &self.inner.semaphore } @@ -219,8 +242,10 @@ impl Rx { pub(crate) fn recv(&mut self, cx: &mut Context<'_>) -> Poll> { use super::block::Read::*; + ready!(crate::trace::trace_leaf(cx)); + // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); self.inner.rx_fields.with_mut(|rx_fields_ptr| { let rx_fields = unsafe { &mut *rx_fields_ptr }; @@ -301,13 +326,13 @@ impl Rx { // Park the thread until the problematic send has completed. let mut park = CachedParkThread::new(); - let waker = park.unpark().into_waker(); + let waker = park.waker().unwrap(); loop { self.inner.rx_waker.register_by_ref(&waker); // It is possible that the problematic send has now completed, // so we have to check for messages again. try_recv!(); - park.park().expect("park failed"); + park.park(); } }) } @@ -345,7 +370,7 @@ impl Drop for Chan { fn drop(&mut self) { use super::block::Read::Value; - // Safety: the only owner of the rx fields is Chan, and eing + // Safety: the only owner of the rx fields is Chan, and being // inside its own Drop means we're the last ones to touch it. self.rx_fields.with_mut(|rx_fields_ptr| { let rx_fields = unsafe { &mut *rx_fields_ptr }; @@ -358,32 +383,29 @@ impl Drop for Chan { // ===== impl Semaphore for (::Semaphore, capacity) ===== -impl Semaphore for (crate::sync::batch_semaphore::Semaphore, usize) { +impl Semaphore for bounded::Semaphore { fn add_permit(&self) { - self.0.release(1) + self.semaphore.release(1) } fn is_idle(&self) -> bool { - self.0.available_permits() == self.1 + self.semaphore.available_permits() == self.bound } fn close(&self) { - self.0.close(); + self.semaphore.close(); } fn is_closed(&self) -> bool { - self.0.is_closed() + self.semaphore.is_closed() } } // ===== impl Semaphore for AtomicUsize ===== -use std::sync::atomic::Ordering::{Acquire, Release}; -use std::usize; - -impl Semaphore for AtomicUsize { +impl Semaphore for unbounded::Semaphore { fn add_permit(&self) { - let prev = self.fetch_sub(2, Release); + let prev = self.0.fetch_sub(2, Release); if prev >> 1 == 0 { // Something went wrong @@ -392,14 +414,14 @@ impl Semaphore for AtomicUsize { } fn is_idle(&self) -> bool { - self.load(Acquire) >> 1 == 0 + self.0.load(Acquire) >> 1 == 0 } fn close(&self) { - self.fetch_or(1, Release); + self.0.fetch_or(1, Release); } fn is_closed(&self) -> bool { - self.load(Acquire) & 1 == 1 + self.0.load(Acquire) & 1 == 1 } } diff --git a/third_party/rust/tokio/src/sync/mpsc/error.rs b/third_party/rust/tokio/src/sync/mpsc/error.rs index 3fe6bac5e179..25b4455bee70 100644 --- a/third_party/rust/tokio/src/sync/mpsc/error.rs +++ b/third_party/rust/tokio/src/sync/mpsc/error.rs @@ -4,22 +4,28 @@ use std::error::Error; use std::fmt; /// Error returned by the `Sender`. -#[derive(Debug)] +#[derive(PartialEq, Eq, Clone, Copy)] pub struct SendError(pub T); +impl fmt::Debug for SendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SendError").finish_non_exhaustive() + } +} + impl fmt::Display for SendError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { write!(fmt, "channel closed") } } -impl std::error::Error for SendError {} +impl std::error::Error for SendError {} // ===== TrySendError ===== /// This enumeration is the list of the possible error outcomes for the /// [try_send](super::Sender::try_send) method. -#[derive(Debug, Eq, PartialEq)] +#[derive(PartialEq, Eq, Clone, Copy)] pub enum TrySendError { /// The data could not be sent on the channel because the channel is /// currently full and sending would require blocking. @@ -30,7 +36,14 @@ pub enum TrySendError { Closed(T), } -impl Error for TrySendError {} +impl fmt::Debug for TrySendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + TrySendError::Full(..) => "Full(..)".fmt(f), + TrySendError::Closed(..) => "Closed(..)".fmt(f), + } + } +} impl fmt::Display for TrySendError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -45,6 +58,8 @@ impl fmt::Display for TrySendError { } } +impl Error for TrySendError {} + impl From> for TrySendError { fn from(src: SendError) -> TrySendError { TrySendError::Closed(src.0) @@ -78,7 +93,7 @@ impl Error for TryRecvError {} // ===== RecvError ===== /// Error returned by `Receiver`. -#[derive(Debug)] +#[derive(Debug, Clone)] #[doc(hidden)] #[deprecated(note = "This type is unused because recv returns an Option.")] pub struct RecvError(()); @@ -96,7 +111,7 @@ impl Error for RecvError {} cfg_time! { // ===== SendTimeoutError ===== - #[derive(Debug, Eq, PartialEq)] + #[derive(PartialEq, Eq, Clone, Copy)] /// Error returned by [`Sender::send_timeout`](super::Sender::send_timeout)]. pub enum SendTimeoutError { /// The data could not be sent on the channel because the channel is @@ -108,7 +123,14 @@ cfg_time! { Closed(T), } - impl Error for SendTimeoutError {} + impl fmt::Debug for SendTimeoutError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + SendTimeoutError::Timeout(..) => "Timeout(..)".fmt(f), + SendTimeoutError::Closed(..) => "Closed(..)".fmt(f), + } + } + } impl fmt::Display for SendTimeoutError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -122,4 +144,6 @@ cfg_time! { ) } } + + impl Error for SendTimeoutError {} } diff --git a/third_party/rust/tokio/src/sync/mpsc/list.rs b/third_party/rust/tokio/src/sync/mpsc/list.rs index e4eeb4541189..10b29575bdb4 100644 --- a/third_party/rust/tokio/src/sync/mpsc/list.rs +++ b/third_party/rust/tokio/src/sync/mpsc/list.rs @@ -44,7 +44,7 @@ pub(crate) enum TryPopResult { pub(crate) fn channel() -> (Tx, Rx) { // Create the initial block shared between the tx and rx halves. - let initial_block = Box::new(Block::new(0)); + let initial_block = Block::new(0); let initial_block_ptr = Box::into_raw(initial_block); let tx = Tx { diff --git a/third_party/rust/tokio/src/sync/mpsc/mod.rs b/third_party/rust/tokio/src/sync/mpsc/mod.rs index b1513a9da51c..b2af084b2ae2 100644 --- a/third_party/rust/tokio/src/sync/mpsc/mod.rs +++ b/third_party/rust/tokio/src/sync/mpsc/mod.rs @@ -21,6 +21,9 @@ //! when additional capacity is available. In other words, the channel provides //! backpressure. //! +//! This channel is also suitable for the single-producer single-consumer +//! use-case. (Unless you only need to send one message, in which case you +//! should use the [oneshot] channel.) //! //! # Disconnection //! @@ -30,7 +33,8 @@ //! //! If the [`Receiver`] handle is dropped, then messages can no longer //! be read out of the channel. In this case, all further attempts to send will -//! result in an error. +//! result in an error. Additionally, all unread messages will be drained from the +//! channel and dropped. //! //! # Clean Shutdown //! @@ -62,7 +66,7 @@ //! in mind, but they can also be generalized to other kinds of channels. In //! general, any channel method that isn't marked async can be called anywhere, //! including outside of the runtime. For example, sending a message on a -//! oneshot channel from outside the runtime is perfectly fine. +//! [oneshot] channel from outside the runtime is perfectly fine. //! //! # Multiple runtimes //! @@ -82,6 +86,7 @@ //! [blocking-recv]: crate::sync::mpsc::Receiver::blocking_recv() //! [`UnboundedSender`]: crate::sync::mpsc::UnboundedSender //! [`UnboundedReceiver`]: crate::sync::mpsc::UnboundedReceiver +//! [oneshot]: crate::sync::oneshot //! [`Handle::block_on`]: crate::runtime::Handle::block_on() //! [std-unbounded]: std::sync::mpsc::channel //! [crossbeam-unbounded]: https://docs.rs/crossbeam/*/crossbeam/channel/fn.unbounded.html @@ -90,14 +95,16 @@ pub(super) mod block; mod bounded; -pub use self::bounded::{channel, OwnedPermit, Permit, Receiver, Sender}; +pub use self::bounded::{channel, OwnedPermit, Permit, Receiver, Sender, WeakSender}; mod chan; pub(super) mod list; mod unbounded; -pub use self::unbounded::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +pub use self::unbounded::{ + unbounded_channel, UnboundedReceiver, UnboundedSender, WeakUnboundedSender, +}; pub mod error; diff --git a/third_party/rust/tokio/src/sync/mpsc/unbounded.rs b/third_party/rust/tokio/src/sync/mpsc/unbounded.rs index b133f9f35e35..cd83fc125139 100644 --- a/third_party/rust/tokio/src/sync/mpsc/unbounded.rs +++ b/third_party/rust/tokio/src/sync/mpsc/unbounded.rs @@ -1,4 +1,4 @@ -use crate::loom::sync::atomic::AtomicUsize; +use crate::loom::sync::{atomic::AtomicUsize, Arc}; use crate::sync::mpsc::chan; use crate::sync::mpsc::error::{SendError, TryRecvError}; @@ -13,6 +13,40 @@ pub struct UnboundedSender { chan: chan::Tx, } +/// An unbounded sender that does not prevent the channel from being closed. +/// +/// If all [`UnboundedSender`] instances of a channel were dropped and only +/// `WeakUnboundedSender` instances remain, the channel is closed. +/// +/// In order to send messages, the `WeakUnboundedSender` needs to be upgraded using +/// [`WeakUnboundedSender::upgrade`], which returns `Option`. It returns `None` +/// if all `UnboundedSender`s have been dropped, and otherwise it returns an `UnboundedSender`. +/// +/// [`UnboundedSender`]: UnboundedSender +/// [`WeakUnboundedSender::upgrade`]: WeakUnboundedSender::upgrade +/// +/// #Examples +/// +/// ``` +/// use tokio::sync::mpsc::unbounded_channel; +/// +/// #[tokio::main] +/// async fn main() { +/// let (tx, _rx) = unbounded_channel::(); +/// let tx_weak = tx.downgrade(); +/// +/// // Upgrading will succeed because `tx` still exists. +/// assert!(tx_weak.upgrade().is_some()); +/// +/// // If we drop `tx`, then it will fail. +/// drop(tx); +/// assert!(tx_weak.clone().upgrade().is_none()); +/// } +/// ``` +pub struct WeakUnboundedSender { + chan: Arc>, +} + impl Clone for UnboundedSender { fn clone(&self) -> Self { UnboundedSender { @@ -61,7 +95,7 @@ impl fmt::Debug for UnboundedReceiver { /// the channel. Using an `unbounded` channel has the ability of causing the /// process to run out of memory. In this case, the process will be aborted. pub fn unbounded_channel() -> (UnboundedSender, UnboundedReceiver) { - let (tx, rx) = chan::channel(AtomicUsize::new(0)); + let (tx, rx) = chan::channel(Semaphore(AtomicUsize::new(0))); let tx = UnboundedSender::new(tx); let rx = UnboundedReceiver::new(rx); @@ -70,7 +104,8 @@ pub fn unbounded_channel() -> (UnboundedSender, UnboundedReceiver) { } /// No capacity -type Semaphore = AtomicUsize; +#[derive(Debug)] +pub(crate) struct Semaphore(pub(crate) AtomicUsize); impl UnboundedReceiver { pub(crate) fn new(chan: chan::Rx) -> UnboundedReceiver { @@ -79,8 +114,14 @@ impl UnboundedReceiver { /// Receives the next value for this receiver. /// - /// `None` is returned when all `Sender` halves have dropped, indicating - /// that no further values can be sent on the channel. + /// This method returns `None` if the channel has been closed and there are + /// no remaining messages in the channel's buffer. This indicates that no + /// further values can ever be received from this `Receiver`. The channel is + /// closed when all senders have been dropped, or when [`close`] is called. + /// + /// If there are no messages in the channel's buffer, but the channel has + /// not yet been closed, this method will sleep until a message is sent or + /// the channel is closed. /// /// # Cancel safety /// @@ -89,6 +130,8 @@ impl UnboundedReceiver { /// completes first, it is guaranteed that no messages were received on this /// channel. /// + /// [`close`]: Self::close + /// /// # Examples /// /// ``` @@ -198,7 +241,9 @@ impl UnboundedReceiver { /// sync_code.join().unwrap(); /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] + #[cfg_attr(docsrs, doc(alias = "recv_blocking"))] pub fn blocking_recv(&mut self) -> Option { crate::future::block_on(self.recv()) } @@ -207,6 +252,9 @@ impl UnboundedReceiver { /// /// This prevents any further messages from being sent on the channel while /// still enabling the receiver to drain messages that are buffered. + /// + /// To guarantee that no messages are dropped, after calling `close()`, + /// `recv()` must be called until `None` is returned. pub fn close(&mut self) { self.chan.close(); } @@ -267,7 +315,7 @@ impl UnboundedSender { use std::process; use std::sync::atomic::Ordering::{AcqRel, Acquire}; - let mut curr = self.chan.semaphore().load(Acquire); + let mut curr = self.chan.semaphore().0.load(Acquire); loop { if curr & 1 == 1 { @@ -283,6 +331,7 @@ impl UnboundedSender { match self .chan .semaphore() + .0 .compare_exchange(curr, curr + 2, AcqRel, Acquire) { Ok(_) => return true, @@ -370,4 +419,37 @@ impl UnboundedSender { pub fn same_channel(&self, other: &Self) -> bool { self.chan.same_channel(&other.chan) } + + /// Converts the `UnboundedSender` to a [`WeakUnboundedSender`] that does not count + /// towards RAII semantics, i.e. if all `UnboundedSender` instances of the + /// channel were dropped and only `WeakUnboundedSender` instances remain, + /// the channel is closed. + pub fn downgrade(&self) -> WeakUnboundedSender { + WeakUnboundedSender { + chan: self.chan.downgrade(), + } + } +} + +impl Clone for WeakUnboundedSender { + fn clone(&self) -> Self { + WeakUnboundedSender { + chan: self.chan.clone(), + } + } +} + +impl WeakUnboundedSender { + /// Tries to convert a WeakUnboundedSender into an [`UnboundedSender`]. + /// This will return `Some` if there are other `Sender` instances alive and + /// the channel wasn't previously dropped, otherwise `None` is returned. + pub fn upgrade(&self) -> Option> { + chan::Tx::upgrade(self.chan.clone()).map(UnboundedSender::new) + } +} + +impl fmt::Debug for WeakUnboundedSender { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("WeakUnboundedSender").finish() + } } diff --git a/third_party/rust/tokio/src/sync/mutex.rs b/third_party/rust/tokio/src/sync/mutex.rs index b8d5ba74e758..549c77b321ee 100644 --- a/third_party/rust/tokio/src/sync/mutex.rs +++ b/third_party/rust/tokio/src/sync/mutex.rs @@ -6,9 +6,10 @@ use crate::util::trace; use std::cell::UnsafeCell; use std::error::Error; +use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::sync::Arc; -use std::{fmt, marker, mem}; +use std::{fmt, mem, ptr}; /// An asynchronous `Mutex`-like type. /// @@ -141,7 +142,11 @@ pub struct Mutex { /// /// The lock is automatically released whenever the guard is dropped, at which /// point `lock` will succeed yet again. +#[clippy::has_significant_drop] +#[must_use = "if unused the Mutex will immediately unlock"] pub struct MutexGuard<'a, T: ?Sized> { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] resource_span: tracing::Span, lock: &'a Mutex, @@ -162,7 +167,10 @@ pub struct MutexGuard<'a, T: ?Sized> { /// point `lock` will succeed yet again. /// /// [`Arc`]: std::sync::Arc +#[clippy::has_significant_drop] pub struct OwnedMutexGuard { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] resource_span: tracing::Span, lock: Arc>, @@ -173,12 +181,71 @@ pub struct OwnedMutexGuard { /// This can be used to hold a subfield of the protected data. /// /// [`MutexGuard::map`]: method@MutexGuard::map +#[clippy::has_significant_drop] #[must_use = "if unused the Mutex will immediately unlock"] pub struct MappedMutexGuard<'a, T: ?Sized> { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, s: &'a semaphore::Semaphore, data: *mut T, // Needed to tell the borrow checker that we are holding a `&mut T` - marker: marker::PhantomData<&'a mut T>, + marker: PhantomData<&'a mut T>, +} + +/// A owned handle to a held `Mutex` that has had a function applied to it via +/// [`OwnedMutexGuard::map`]. +/// +/// This can be used to hold a subfield of the protected data. +/// +/// [`OwnedMutexGuard::map`]: method@OwnedMutexGuard::map +#[clippy::has_significant_drop] +#[must_use = "if unused the Mutex will immediately unlock"] +pub struct OwnedMappedMutexGuard { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + data: *mut U, + lock: Arc>, +} + +/// A helper type used when taking apart a `MutexGuard` without running its +/// Drop implementation. +#[allow(dead_code)] // Unused fields are still used in Drop. +struct MutexGuardInner<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + lock: &'a Mutex, +} + +/// A helper type used when taking apart a `OwnedMutexGuard` without running +/// its Drop implementation. +struct OwnedMutexGuardInner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + lock: Arc>, +} + +/// A helper type used when taking apart a `MappedMutexGuard` without running +/// its Drop implementation. +#[allow(dead_code)] // Unused fields are still used in Drop. +struct MappedMutexGuardInner<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + s: &'a semaphore::Semaphore, + data: *mut T, +} + +/// A helper type used when taking apart a `OwnedMappedMutexGuard` without running +/// its Drop implementation. +#[allow(dead_code)] // Unused fields are still used in Drop. +struct OwnedMappedMutexGuardInner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + data: *mut U, + lock: Arc>, } // As long as T: Send, it's fine to send and share Mutex between threads. @@ -191,6 +258,19 @@ unsafe impl Sync for OwnedMutexGuard where T: ?Sized + Send + Sync {} unsafe impl<'a, T> Sync for MappedMutexGuard<'a, T> where T: ?Sized + Sync + 'a {} unsafe impl<'a, T> Send for MappedMutexGuard<'a, T> where T: ?Sized + Send + 'a {} +unsafe impl Sync for OwnedMappedMutexGuard +where + T: ?Sized + Send + Sync, + U: ?Sized + Send + Sync, +{ +} +unsafe impl Send for OwnedMappedMutexGuard +where + T: ?Sized + Send, + U: ?Sized + Send, +{ +} + /// Error returned from the [`Mutex::try_lock`], [`RwLock::try_read`] and /// [`RwLock::try_write`] functions. /// @@ -199,8 +279,8 @@ unsafe impl<'a, T> Send for MappedMutexGuard<'a, T> where T: ?Sized + Send + 'a /// `RwLock::try_read` operation will only fail if the lock is currently held /// by an exclusive writer. /// -/// `RwLock::try_write` operation will if lock is held by any reader or by an -/// exclusive writer. +/// `RwLock::try_write` operation will only fail if the lock is currently held +/// by any reader or by an exclusive writer. /// /// [`Mutex::try_lock`]: Mutex::try_lock /// [`RwLock::try_read`]: fn@super::RwLock::try_read @@ -336,15 +416,27 @@ impl Mutex { /// } /// ``` pub async fn lock(&self) -> MutexGuard<'_, T> { + let acquire_fut = async { + self.acquire().await; + + MutexGuard { + lock: self, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + } + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] - trace::async_op( - || self.acquire(), + let acquire_fut = trace::async_op( + move || acquire_fut, self.resource_span.clone(), "Mutex::lock", "poll", false, - ) - .await; + ); + + #[allow(clippy::let_and_return)] // this lint triggers when disabling tracing + let guard = acquire_fut.await; #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { @@ -354,14 +446,7 @@ impl Mutex { ); }); - #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] - self.acquire().await; - - MutexGuard { - lock: self, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: self.resource_span.clone(), - } + guard } /// Blockingly locks this `Mutex`. When the lock has been acquired, function returns a @@ -411,11 +496,69 @@ impl Mutex { /// } /// /// ``` + #[track_caller] #[cfg(feature = "sync")] + #[cfg_attr(docsrs, doc(alias = "lock_blocking"))] pub fn blocking_lock(&self) -> MutexGuard<'_, T> { crate::future::block_on(self.lock()) } + /// Blockingly locks this `Mutex`. When the lock has been acquired, function returns an + /// [`OwnedMutexGuard`]. + /// + /// This method is identical to [`Mutex::blocking_lock`], except that the returned + /// guard references the `Mutex` with an [`Arc`] rather than by borrowing + /// it. Therefore, the `Mutex` must be wrapped in an `Arc` to call this + /// method, and the guard will live for the `'static` lifetime, as it keeps + /// the `Mutex` alive by holding an `Arc`. + /// + /// # Panics + /// + /// This function panics if called within an asynchronous execution context. + /// + /// - If you find yourself in an asynchronous execution context and needing + /// to call some (synchronous) function which performs one of these + /// `blocking_` operations, then consider wrapping that call inside + /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking] + /// (or [`block_in_place()`][crate::task::block_in_place]). + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// use tokio::sync::Mutex; + /// + /// #[tokio::main] + /// async fn main() { + /// let mutex = Arc::new(Mutex::new(1)); + /// let lock = mutex.lock().await; + /// + /// let mutex1 = Arc::clone(&mutex); + /// let blocking_task = tokio::task::spawn_blocking(move || { + /// // This shall block until the `lock` is released. + /// let mut n = mutex1.blocking_lock_owned(); + /// *n = 2; + /// }); + /// + /// assert_eq!(*lock, 1); + /// // Release the lock. + /// drop(lock); + /// + /// // Await the completion of the blocking task. + /// blocking_task.await.unwrap(); + /// + /// // Assert uncontended. + /// let n = mutex.try_lock().unwrap(); + /// assert_eq!(*n, 2); + /// } + /// + /// ``` + #[track_caller] + #[cfg(feature = "sync")] + pub fn blocking_lock_owned(self: Arc) -> OwnedMutexGuard { + crate::future::block_on(self.lock_owned()) + } + /// Locks this mutex, causing the current task to yield until the lock has /// been acquired. When the lock has been acquired, this returns an /// [`OwnedMutexGuard`]. @@ -450,37 +593,44 @@ impl Mutex { /// [`Arc`]: std::sync::Arc pub async fn lock_owned(self: Arc) -> OwnedMutexGuard { #[cfg(all(tokio_unstable, feature = "tracing"))] - trace::async_op( - || self.acquire(), - self.resource_span.clone(), + let resource_span = self.resource_span.clone(); + + let acquire_fut = async { + self.acquire().await; + + OwnedMutexGuard { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + lock: self, + } + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let acquire_fut = trace::async_op( + move || acquire_fut, + resource_span, "Mutex::lock_owned", "poll", false, - ) - .await; + ); + + #[allow(clippy::let_and_return)] // this lint triggers when disabling tracing + let guard = acquire_fut.await; #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", locked = true, ); }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - - #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] - self.acquire().await; - - OwnedMutexGuard { - lock: self, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - } + guard } async fn acquire(&self) { + crate::trace::async_trace_leaf().await; + self.s.acquire(1).await.unwrap_or_else(|_| { // The semaphore was closed. but, we never explicitly close it, and // we own it exclusively, which means that this can never happen. @@ -508,6 +658,12 @@ impl Mutex { pub fn try_lock(&self) -> Result, TryLockError> { match self.s.try_acquire(1) { Ok(_) => { + let guard = MutexGuard { + lock: self, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { tracing::trace!( @@ -516,11 +672,7 @@ impl Mutex { ); }); - Ok(MutexGuard { - lock: self, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: self.resource_span.clone(), - }) + Ok(guard) } Err(_) => Err(TryLockError(())), } @@ -577,22 +729,21 @@ impl Mutex { pub fn try_lock_owned(self: Arc) -> Result, TryLockError> { match self.s.try_acquire(1) { Ok(_) => { + let guard = OwnedMutexGuard { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + lock: self, + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", locked = true, ); }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - - Ok(OwnedMutexGuard { - lock: self, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - }) + Ok(guard) } Err(_) => Err(TryLockError(())), } @@ -652,6 +803,17 @@ where // === impl MutexGuard === impl<'a, T: ?Sized> MutexGuard<'a, T> { + fn skip_drop(self) -> MutexGuardInner<'a, T> { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the `resource_span` and then forgets the + // original. In the end, we have not duplicated or forgotten any values. + MutexGuardInner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: unsafe { std::ptr::read(&me.resource_span) }, + lock: me.lock, + } + } + /// Makes a new [`MappedMutexGuard`] for a component of the locked data. /// /// This operation cannot fail as the [`MutexGuard`] passed in already locked the mutex. @@ -688,12 +850,13 @@ impl<'a, T: ?Sized> MutexGuard<'a, T> { F: FnOnce(&mut T) -> &mut U, { let data = f(&mut *this) as *mut U; - let s = &this.lock.s; - mem::forget(this); + let inner = this.skip_drop(); MappedMutexGuard { - s, + s: &inner.lock.s, data, - marker: marker::PhantomData, + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, } } @@ -738,12 +901,13 @@ impl<'a, T: ?Sized> MutexGuard<'a, T> { Some(data) => data as *mut U, None => return Err(this), }; - let s = &this.lock.s; - mem::forget(this); + let inner = this.skip_drop(); Ok(MappedMutexGuard { - s, + s: &inner.lock.s, data, - marker: marker::PhantomData, + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, }) } @@ -765,7 +929,7 @@ impl<'a, T: ?Sized> MutexGuard<'a, T> { /// # async fn main() { /// # let mutex = Mutex::new(0u32); /// # let guard = mutex.lock().await; - /// # unlock_and_relock(guard).await; + /// # let _guard = unlock_and_relock(guard).await; /// # } /// ``` #[inline] @@ -776,6 +940,8 @@ impl<'a, T: ?Sized> MutexGuard<'a, T> { impl Drop for MutexGuard<'_, T> { fn drop(&mut self) { + self.lock.s.release(1); + #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { tracing::trace!( @@ -783,7 +949,6 @@ impl Drop for MutexGuard<'_, T> { locked = false, ); }); - self.lock.s.release(1); } } @@ -815,6 +980,116 @@ impl fmt::Display for MutexGuard<'_, T> { // === impl OwnedMutexGuard === impl OwnedMutexGuard { + fn skip_drop(self) -> OwnedMutexGuardInner { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + unsafe { + OwnedMutexGuardInner { + lock: ptr::read(&me.lock), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: ptr::read(&me.resource_span), + } + } + } + + /// Makes a new [`OwnedMappedMutexGuard`] for a component of the locked data. + /// + /// This operation cannot fail as the [`OwnedMutexGuard`] passed in already locked the mutex. + /// + /// This is an associated function that needs to be used as `OwnedMutexGuard::map(...)`. A method + /// would interfere with methods of the same name on the contents of the locked data. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::{Mutex, OwnedMutexGuard}; + /// use std::sync::Arc; + /// + /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] + /// struct Foo(u32); + /// + /// # #[tokio::main] + /// # async fn main() { + /// let foo = Arc::new(Mutex::new(Foo(1))); + /// + /// { + /// let mut mapped = OwnedMutexGuard::map(foo.clone().lock_owned().await, |f| &mut f.0); + /// *mapped = 2; + /// } + /// + /// assert_eq!(Foo(2), *foo.lock().await); + /// # } + /// ``` + /// + /// [`OwnedMutexGuard`]: struct@OwnedMutexGuard + /// [`OwnedMappedMutexGuard`]: struct@OwnedMappedMutexGuard + #[inline] + pub fn map(mut this: Self, f: F) -> OwnedMappedMutexGuard + where + F: FnOnce(&mut T) -> &mut U, + { + let data = f(&mut *this) as *mut U; + let inner = this.skip_drop(); + OwnedMappedMutexGuard { + data, + lock: inner.lock, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, + } + } + + /// Attempts to make a new [`OwnedMappedMutexGuard`] for a component of the locked data. The + /// original guard is returned if the closure returns `None`. + /// + /// This operation cannot fail as the [`OwnedMutexGuard`] passed in already locked the mutex. + /// + /// This is an associated function that needs to be used as `OwnedMutexGuard::try_map(...)`. A + /// method would interfere with methods of the same name on the contents of the locked data. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::{Mutex, OwnedMutexGuard}; + /// use std::sync::Arc; + /// + /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] + /// struct Foo(u32); + /// + /// # #[tokio::main] + /// # async fn main() { + /// let foo = Arc::new(Mutex::new(Foo(1))); + /// + /// { + /// let mut mapped = OwnedMutexGuard::try_map(foo.clone().lock_owned().await, |f| Some(&mut f.0)) + /// .expect("should not fail"); + /// *mapped = 2; + /// } + /// + /// assert_eq!(Foo(2), *foo.lock().await); + /// # } + /// ``` + /// + /// [`OwnedMutexGuard`]: struct@OwnedMutexGuard + /// [`OwnedMappedMutexGuard`]: struct@OwnedMappedMutexGuard + #[inline] + pub fn try_map(mut this: Self, f: F) -> Result, Self> + where + F: FnOnce(&mut T) -> Option<&mut U>, + { + let data = match f(&mut *this) { + Some(data) => data as *mut U, + None => return Err(this), + }; + let inner = this.skip_drop(); + Ok(OwnedMappedMutexGuard { + data, + lock: inner.lock, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, + }) + } + /// Returns a reference to the original `Arc`. /// /// ``` @@ -845,6 +1120,8 @@ impl OwnedMutexGuard { impl Drop for OwnedMutexGuard { fn drop(&mut self) { + self.lock.s.release(1); + #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { tracing::trace!( @@ -852,7 +1129,6 @@ impl Drop for OwnedMutexGuard { locked = false, ); }); - self.lock.s.release(1) } } @@ -884,6 +1160,16 @@ impl fmt::Display for OwnedMutexGuard { // === impl MappedMutexGuard === impl<'a, T: ?Sized> MappedMutexGuard<'a, T> { + fn skip_drop(self) -> MappedMutexGuardInner<'a, T> { + let me = mem::ManuallyDrop::new(self); + MappedMutexGuardInner { + s: me.s, + data: me.data, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: unsafe { std::ptr::read(&me.resource_span) }, + } + } + /// Makes a new [`MappedMutexGuard`] for a component of the locked data. /// /// This operation cannot fail as the [`MappedMutexGuard`] passed in already locked the mutex. @@ -898,12 +1184,13 @@ impl<'a, T: ?Sized> MappedMutexGuard<'a, T> { F: FnOnce(&mut T) -> &mut U, { let data = f(&mut *this) as *mut U; - let s = this.s; - mem::forget(this); + let inner = this.skip_drop(); MappedMutexGuard { - s, + s: inner.s, data, - marker: marker::PhantomData, + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, } } @@ -925,19 +1212,28 @@ impl<'a, T: ?Sized> MappedMutexGuard<'a, T> { Some(data) => data as *mut U, None => return Err(this), }; - let s = this.s; - mem::forget(this); + let inner = this.skip_drop(); Ok(MappedMutexGuard { - s, + s: inner.s, data, - marker: marker::PhantomData, + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, }) } } impl<'a, T: ?Sized> Drop for MappedMutexGuard<'a, T> { fn drop(&mut self) { - self.s.release(1) + self.s.release(1); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = false, + ); + }); } } @@ -965,3 +1261,111 @@ impl<'a, T: ?Sized + fmt::Display> fmt::Display for MappedMutexGuard<'a, T> { fmt::Display::fmt(&**self, f) } } + +// === impl OwnedMappedMutexGuard === + +impl OwnedMappedMutexGuard { + fn skip_drop(self) -> OwnedMappedMutexGuardInner { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + unsafe { + OwnedMappedMutexGuardInner { + data: me.data, + lock: ptr::read(&me.lock), + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: ptr::read(&me.resource_span), + } + } + } + + /// Makes a new [`OwnedMappedMutexGuard`] for a component of the locked data. + /// + /// This operation cannot fail as the [`OwnedMappedMutexGuard`] passed in already locked the mutex. + /// + /// This is an associated function that needs to be used as `OwnedMappedMutexGuard::map(...)`. A method + /// would interfere with methods of the same name on the contents of the locked data. + /// + /// [`OwnedMappedMutexGuard`]: struct@OwnedMappedMutexGuard + #[inline] + pub fn map(mut this: Self, f: F) -> OwnedMappedMutexGuard + where + F: FnOnce(&mut U) -> &mut S, + { + let data = f(&mut *this) as *mut S; + let inner = this.skip_drop(); + OwnedMappedMutexGuard { + data, + lock: inner.lock, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, + } + } + + /// Attempts to make a new [`OwnedMappedMutexGuard`] for a component of the locked data. The + /// original guard is returned if the closure returns `None`. + /// + /// This operation cannot fail as the [`OwnedMutexGuard`] passed in already locked the mutex. + /// + /// This is an associated function that needs to be used as `OwnedMutexGuard::try_map(...)`. A + /// method would interfere with methods of the same name on the contents of the locked data. + /// + /// [`OwnedMutexGuard`]: struct@OwnedMutexGuard + /// [`OwnedMappedMutexGuard`]: struct@OwnedMappedMutexGuard + #[inline] + pub fn try_map(mut this: Self, f: F) -> Result, Self> + where + F: FnOnce(&mut U) -> Option<&mut S>, + { + let data = match f(&mut *this) { + Some(data) => data as *mut S, + None => return Err(this), + }; + let inner = this.skip_drop(); + Ok(OwnedMappedMutexGuard { + data, + lock: inner.lock, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: inner.resource_span, + }) + } +} + +impl Drop for OwnedMappedMutexGuard { + fn drop(&mut self) { + self.lock.s.release(1); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + self.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + locked = false, + ); + }); + } +} + +impl Deref for OwnedMappedMutexGuard { + type Target = U; + fn deref(&self) -> &Self::Target { + unsafe { &*self.data } + } +} + +impl DerefMut for OwnedMappedMutexGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.data } + } +} + +impl fmt::Debug for OwnedMappedMutexGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl fmt::Display for OwnedMappedMutexGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} diff --git a/third_party/rust/tokio/src/sync/notify.rs b/third_party/rust/tokio/src/sync/notify.rs index 83d0de4fbe65..0f104b71aa2c 100644 --- a/third_party/rust/tokio/src/sync/notify.rs +++ b/third_party/rust/tokio/src/sync/notify.rs @@ -5,20 +5,22 @@ // triggers this warning but it is safe to ignore in this case. #![cfg_attr(not(feature = "sync"), allow(unreachable_pub, dead_code))] +use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::Mutex; -use crate::util::linked_list::{self, LinkedList}; +use crate::util::linked_list::{self, GuardedLinkedList, LinkedList}; use crate::util::WakeList; -use std::cell::UnsafeCell; use std::future::Future; use std::marker::PhantomPinned; +use std::panic::{RefUnwindSafe, UnwindSafe}; use std::pin::Pin; use std::ptr::NonNull; -use std::sync::atomic::Ordering::SeqCst; +use std::sync::atomic::Ordering::{self, Acquire, Relaxed, Release, SeqCst}; use std::task::{Context, Poll, Waker}; type WaitList = LinkedList::Target>; +type GuardedWaitList = GuardedLinkedList::Target>; /// Notifies a single task to wake up. /// @@ -26,22 +28,23 @@ type WaitList = LinkedList::Target>; /// `Notify` itself does not carry any data. Instead, it is to be used to signal /// another task to perform an operation. /// -/// `Notify` can be thought of as a [`Semaphore`] starting with 0 permits. -/// [`notified().await`] waits for a permit to become available, and [`notify_one()`] -/// sets a permit **if there currently are no available permits**. +/// A `Notify` can be thought of as a [`Semaphore`] starting with 0 permits. The +/// [`notified().await`] method waits for a permit to become available, and +/// [`notify_one()`] sets a permit **if there currently are no available +/// permits**. /// /// The synchronization details of `Notify` are similar to /// [`thread::park`][park] and [`Thread::unpark`][unpark] from std. A [`Notify`] /// value contains a single permit. [`notified().await`] waits for the permit to -/// be made available, consumes the permit, and resumes. [`notify_one()`] sets the -/// permit, waking a pending task if there is one. +/// be made available, consumes the permit, and resumes. [`notify_one()`] sets +/// the permit, waking a pending task if there is one. /// -/// If `notify_one()` is called **before** `notified().await`, then the next call to -/// `notified().await` will complete immediately, consuming the permit. Any -/// subsequent calls to `notified().await` will wait for a new permit. +/// If `notify_one()` is called **before** `notified().await`, then the next +/// call to `notified().await` will complete immediately, consuming the permit. +/// Any subsequent calls to `notified().await` will wait for a new permit. /// -/// If `notify_one()` is called **multiple** times before `notified().await`, only a -/// **single** permit is stored. The next call to `notified().await` will +/// If `notify_one()` is called **multiple** times before `notified().await`, +/// only a **single** permit is stored. The next call to `notified().await` will /// complete immediately, but the one after will wait for a new permit. /// /// # Examples @@ -70,7 +73,11 @@ type WaitList = LinkedList::Target>; /// } /// ``` /// -/// Unbound mpsc channel. +/// Unbound multi-producer single-consumer (mpsc) channel. +/// +/// No wakeups can be lost when using this channel because the call to +/// `notify_one()` will store a permit in the `Notify`, which the following call +/// to `notified()` will consume. /// /// ``` /// use tokio::sync::Notify; @@ -92,6 +99,8 @@ type WaitList = LinkedList::Target>; /// self.notify.notify_one(); /// } /// +/// // This is a single-consumer channel, so several concurrent calls to +/// // `recv` are not allowed. /// pub async fn recv(&self) -> T { /// loop { /// // Drain values @@ -106,46 +115,250 @@ type WaitList = LinkedList::Target>; /// } /// ``` /// +/// Unbound multi-producer multi-consumer (mpmc) channel. +/// +/// The call to [`enable`] is important because otherwise if you have two +/// calls to `recv` and two calls to `send` in parallel, the following could +/// happen: +/// +/// 1. Both calls to `try_recv` return `None`. +/// 2. Both new elements are added to the vector. +/// 3. The `notify_one` method is called twice, adding only a single +/// permit to the `Notify`. +/// 4. Both calls to `recv` reach the `Notified` future. One of them +/// consumes the permit, and the other sleeps forever. +/// +/// By adding the `Notified` futures to the list by calling `enable` before +/// `try_recv`, the `notify_one` calls in step three would remove the +/// futures from the list and mark them notified instead of adding a permit +/// to the `Notify`. This ensures that both futures are woken. +/// +/// Notice that this failure can only happen if there are two concurrent calls +/// to `recv`. This is why the mpsc example above does not require a call to +/// `enable`. +/// +/// ``` +/// use tokio::sync::Notify; +/// +/// use std::collections::VecDeque; +/// use std::sync::Mutex; +/// +/// struct Channel { +/// messages: Mutex>, +/// notify_on_sent: Notify, +/// } +/// +/// impl Channel { +/// pub fn send(&self, msg: T) { +/// let mut locked_queue = self.messages.lock().unwrap(); +/// locked_queue.push_back(msg); +/// drop(locked_queue); +/// +/// // Send a notification to one of the calls currently +/// // waiting in a call to `recv`. +/// self.notify_on_sent.notify_one(); +/// } +/// +/// pub fn try_recv(&self) -> Option { +/// let mut locked_queue = self.messages.lock().unwrap(); +/// locked_queue.pop_front() +/// } +/// +/// pub async fn recv(&self) -> T { +/// let future = self.notify_on_sent.notified(); +/// tokio::pin!(future); +/// +/// loop { +/// // Make sure that no wakeup is lost if we get +/// // `None` from `try_recv`. +/// future.as_mut().enable(); +/// +/// if let Some(msg) = self.try_recv() { +/// return msg; +/// } +/// +/// // Wait for a call to `notify_one`. +/// // +/// // This uses `.as_mut()` to avoid consuming the future, +/// // which lets us call `Pin::set` below. +/// future.as_mut().await; +/// +/// // Reset the future in case another call to +/// // `try_recv` got the message before us. +/// future.set(self.notify_on_sent.notified()); +/// } +/// } +/// } +/// ``` +/// /// [park]: std::thread::park /// [unpark]: std::thread::Thread::unpark /// [`notified().await`]: Notify::notified() /// [`notify_one()`]: Notify::notify_one() +/// [`enable`]: Notified::enable() /// [`Semaphore`]: crate::sync::Semaphore #[derive(Debug)] pub struct Notify { - // This uses 2 bits to store one of `EMPTY`, + // `state` uses 2 bits to store one of `EMPTY`, // `WAITING` or `NOTIFIED`. The rest of the bits // are used to store the number of times `notify_waiters` // was called. + // + // Throughout the code there are two assumptions: + // - state can be transitioned *from* `WAITING` only if + // `waiters` lock is held + // - number of times `notify_waiters` was called can + // be modified only if `waiters` lock is held state: AtomicUsize, waiters: Mutex, } -#[derive(Debug, Clone, Copy)] -enum NotificationType { - // Notification triggered by calling `notify_waiters` - AllWaiters, - // Notification triggered by calling `notify_one` - OneWaiter, -} - #[derive(Debug)] -#[repr(C)] // required by `linked_list::Link` impl struct Waiter { /// Intrusive linked-list pointers. pointers: linked_list::Pointers, - /// Waiting task's waker. - waker: Option, + /// Waiting task's waker. Depending on the value of `notification`, + /// this field is either protected by the `waiters` lock in + /// `Notify`, or it is exclusively owned by the enclosing `Waiter`. + waker: UnsafeCell>, - /// `true` if the notification has been assigned to this waiter. - notified: Option, + /// Notification for this waiter. + /// * if it's `None`, then `waker` is protected by the `waiters` lock. + /// * if it's `Some`, then `waker` is exclusively owned by the + /// enclosing `Waiter` and can be accessed without locking. + notification: AtomicNotification, /// Should not be `Unpin`. _p: PhantomPinned, } -/// Future returned from [`Notify::notified()`] +impl Waiter { + fn new() -> Waiter { + Waiter { + pointers: linked_list::Pointers::new(), + waker: UnsafeCell::new(None), + notification: AtomicNotification::none(), + _p: PhantomPinned, + } + } +} + +generate_addr_of_methods! { + impl<> Waiter { + unsafe fn addr_of_pointers(self: NonNull) -> NonNull> { + &self.pointers + } + } +} + +// No notification. +const NOTIFICATION_NONE: usize = 0; + +// Notification type used by `notify_one`. +const NOTIFICATION_ONE: usize = 1; + +// Notification type used by `notify_waiters`. +const NOTIFICATION_ALL: usize = 2; + +/// Notification for a `Waiter`. +/// This struct is equivalent to `Option`, but uses +/// `AtomicUsize` inside for atomic operations. +#[derive(Debug)] +struct AtomicNotification(AtomicUsize); + +impl AtomicNotification { + fn none() -> Self { + AtomicNotification(AtomicUsize::new(NOTIFICATION_NONE)) + } + + /// Store-release a notification. + /// This method should be called exactly once. + fn store_release(&self, notification: Notification) { + self.0.store(notification as usize, Release); + } + + fn load(&self, ordering: Ordering) -> Option { + match self.0.load(ordering) { + NOTIFICATION_NONE => None, + NOTIFICATION_ONE => Some(Notification::One), + NOTIFICATION_ALL => Some(Notification::All), + _ => unreachable!(), + } + } + + /// Clears the notification. + /// This method is used by a `Notified` future to consume the + /// notification. It uses relaxed ordering and should be only + /// used once the atomic notification is no longer shared. + fn clear(&self) { + self.0.store(NOTIFICATION_NONE, Relaxed); + } +} + +#[derive(Debug, PartialEq, Eq)] +#[repr(usize)] +enum Notification { + One = NOTIFICATION_ONE, + All = NOTIFICATION_ALL, +} + +/// List used in `Notify::notify_waiters`. It wraps a guarded linked list +/// and gates the access to it on `notify.waiters` mutex. It also empties +/// the list on drop. +struct NotifyWaitersList<'a> { + list: GuardedWaitList, + is_empty: bool, + notify: &'a Notify, +} + +impl<'a> NotifyWaitersList<'a> { + fn new( + unguarded_list: WaitList, + guard: Pin<&'a Waiter>, + notify: &'a Notify, + ) -> NotifyWaitersList<'a> { + let guard_ptr = NonNull::from(guard.get_ref()); + let list = unguarded_list.into_guarded(guard_ptr); + NotifyWaitersList { + list, + is_empty: false, + notify, + } + } + + /// Removes the last element from the guarded list. Modifying this list + /// requires an exclusive access to the main list in `Notify`. + fn pop_back_locked(&mut self, _waiters: &mut WaitList) -> Option> { + let result = self.list.pop_back(); + if result.is_none() { + // Save information about emptiness to avoid waiting for lock + // in the destructor. + self.is_empty = true; + } + result + } +} + +impl Drop for NotifyWaitersList<'_> { + fn drop(&mut self) { + // If the list is not empty, we unlink all waiters from it. + // We do not wake the waiters to avoid double panics. + if !self.is_empty { + let _lock_guard = self.notify.waiters.lock(); + while let Some(waiter) = self.list.pop_back() { + // Safety: we never make mutable references to waiters. + let waiter = unsafe { waiter.as_ref() }; + waiter.notification.store_release(Notification::All); + } + } + } +} + +/// Future returned from [`Notify::notified()`]. +/// +/// This future is fused, so once it has completed, any future calls to poll +/// will immediately return `Poll::Ready`. #[derive(Debug)] pub struct Notified<'a> { /// The `Notify` being received on. @@ -154,8 +367,11 @@ pub struct Notified<'a> { /// The current state of the receiving process. state: State, + /// Number of calls to `notify_waiters` at the time of creation. + notify_waiters_calls: usize, + /// Entry in the waiter `LinkedList`. - waiter: UnsafeCell, + waiter: Waiter, } unsafe impl<'a> Send for Notified<'a> {} @@ -163,7 +379,7 @@ unsafe impl<'a> Sync for Notified<'a> {} #[derive(Debug)] enum State { - Init(usize), + Init, Waiting, Done, } @@ -249,7 +465,16 @@ impl Notify { /// immediately, consuming that permit. Otherwise, `notified().await` waits /// for a permit to be made available by the next call to `notify_one()`. /// + /// The `Notified` future is not guaranteed to receive wakeups from calls to + /// `notify_one()` if it has not yet been polled. See the documentation for + /// [`Notified::enable()`] for more details. + /// + /// The `Notified` future is guaranteed to receive wakeups from + /// `notify_waiters()` as soon as it has been created, even if it has not + /// yet been polled. + /// /// [`notify_one()`]: Notify::notify_one + /// [`Notified::enable()`]: Notified::enable /// /// # Cancel safety /// @@ -279,17 +504,13 @@ impl Notify { /// ``` pub fn notified(&self) -> Notified<'_> { // we load the number of times notify_waiters - // was called and store that in our initial state + // was called and store that in the future. let state = self.state.load(SeqCst); Notified { notify: self, - state: State::Init(state >> NOTIFY_WAITERS_SHIFT), - waiter: UnsafeCell::new(Waiter { - pointers: linked_list::Pointers::new(), - waker: None, - notified: None, - _p: PhantomPinned, - }), + state: State::Init, + notify_waiters_calls: get_num_notify_waiters_calls(state), + waiter: Waiter::new(), } } @@ -396,39 +617,56 @@ impl Notify { /// } /// ``` pub fn notify_waiters(&self) { - let mut wakers = WakeList::new(); - - // There are waiters, the lock must be acquired to notify. let mut waiters = self.waiters.lock(); - // The state must be reloaded while the lock is held. The state may only + // The state must be loaded while the lock is held. The state may only // transition out of WAITING while the lock is held. let curr = self.state.load(SeqCst); - if let EMPTY | NOTIFIED = get_state(curr) { + if matches!(get_state(curr), EMPTY | NOTIFIED) { // There are no waiting tasks. All we need to do is increment the // number of times this method was called. atomic_inc_num_notify_waiters_calls(&self.state); return; } - // At this point, it is guaranteed that the state will not - // concurrently change, as holding the lock is required to - // transition **out** of `WAITING`. + // Increment the number of times this method was called + // and transition to empty. + let new_state = set_state(inc_num_notify_waiters_calls(curr), EMPTY); + self.state.store(new_state, SeqCst); + + // It is critical for `GuardedLinkedList` safety that the guard node is + // pinned in memory and is not dropped until the guarded list is dropped. + let guard = Waiter::new(); + pin!(guard); + + // We move all waiters to a secondary list. It uses a `GuardedLinkedList` + // underneath to allow every waiter to safely remove itself from it. + // + // * This list will be still guarded by the `waiters` lock. + // `NotifyWaitersList` wrapper makes sure we hold the lock to modify it. + // * This wrapper will empty the list on drop. It is critical for safety + // that we will not leave any list entry with a pointer to the local + // guard node after this function returns / panics. + let mut list = NotifyWaitersList::new(std::mem::take(&mut *waiters), guard.as_ref(), self); + + let mut wakers = WakeList::new(); 'outer: loop { while wakers.can_push() { - match waiters.pop_back() { - Some(mut waiter) => { - // Safety: `waiters` lock is still held. - let waiter = unsafe { waiter.as_mut() }; + match list.pop_back_locked(&mut waiters) { + Some(waiter) => { + // Safety: we never make mutable references to waiters. + let waiter = unsafe { waiter.as_ref() }; - assert!(waiter.notified.is_none()); - - waiter.notified = Some(NotificationType::AllWaiters); - - if let Some(waker) = waiter.waker.take() { + // Safety: we hold the lock, so we can access the waker. + if let Some(waker) = + unsafe { waiter.waker.with_mut(|waker| (*waker).take()) } + { wakers.push(waker); } + + // This waiter is unlinked and will not be shared ever again, release it. + waiter.notification.store_release(Notification::All); } None => { break 'outer; @@ -436,20 +674,17 @@ impl Notify { } } + // Release the lock before notifying. drop(waiters); + // One of the wakers may panic, but the remaining waiters will still + // be unlinked from the list in `NotifyWaitersList` destructor. wakers.wake_all(); // Acquire the lock again. waiters = self.waiters.lock(); } - // All waiters will be notified, the state must be transitioned to - // `EMPTY`. As transitioning **from** `WAITING` requires the lock to be - // held, a `store` is sufficient. - let new = set_state(inc_num_notify_waiters_calls(curr), EMPTY); - self.state.store(new, SeqCst); - // Release the lock before notifying drop(waiters); @@ -463,6 +698,9 @@ impl Default for Notify { } } +impl UnwindSafe for Notify {} +impl RefUnwindSafe for Notify {} + fn notify_locked(waiters: &mut WaitList, state: &AtomicUsize, curr: usize) -> Option { loop { match get_state(curr) { @@ -485,15 +723,16 @@ fn notify_locked(waiters: &mut WaitList, state: &AtomicUsize, curr: usize) -> Op // transition **out** of `WAITING`. // // Get a pending waiter - let mut waiter = waiters.pop_back().unwrap(); + let waiter = waiters.pop_back().unwrap(); - // Safety: `waiters` lock is still held. - let waiter = unsafe { waiter.as_mut() }; + // Safety: we never make mutable references to waiters. + let waiter = unsafe { waiter.as_ref() }; - assert!(waiter.notified.is_none()); + // Safety: we hold the lock, so we can access the waker. + let waker = unsafe { waiter.waker.with_mut(|waker| (*waker).take()) }; - waiter.notified = Some(NotificationType::OneWaiter); - let waker = waiter.waker.take(); + // This waiter is unlinked and will not be shared ever again, release it. + waiter.notification.store_release(Notification::One); if waiters.is_empty() { // As this the **final** waiter in the list, the state @@ -513,32 +752,142 @@ fn notify_locked(waiters: &mut WaitList, state: &AtomicUsize, curr: usize) -> Op // ===== impl Notified ===== impl Notified<'_> { + /// Adds this future to the list of futures that are ready to receive + /// wakeups from calls to [`notify_one`]. + /// + /// Polling the future also adds it to the list, so this method should only + /// be used if you want to add the future to the list before the first call + /// to `poll`. (In fact, this method is equivalent to calling `poll` except + /// that no `Waker` is registered.) + /// + /// This has no effect on notifications sent using [`notify_waiters`], which + /// are received as long as they happen after the creation of the `Notified` + /// regardless of whether `enable` or `poll` has been called. + /// + /// This method returns true if the `Notified` is ready. This happens in the + /// following situations: + /// + /// 1. The `notify_waiters` method was called between the creation of the + /// `Notified` and the call to this method. + /// 2. This is the first call to `enable` or `poll` on this future, and the + /// `Notify` was holding a permit from a previous call to `notify_one`. + /// The call consumes the permit in that case. + /// 3. The future has previously been enabled or polled, and it has since + /// then been marked ready by either consuming a permit from the + /// `Notify`, or by a call to `notify_one` or `notify_waiters` that + /// removed it from the list of futures ready to receive wakeups. + /// + /// If this method returns true, any future calls to poll on the same future + /// will immediately return `Poll::Ready`. + /// + /// # Examples + /// + /// Unbound multi-producer multi-consumer (mpmc) channel. + /// + /// The call to `enable` is important because otherwise if you have two + /// calls to `recv` and two calls to `send` in parallel, the following could + /// happen: + /// + /// 1. Both calls to `try_recv` return `None`. + /// 2. Both new elements are added to the vector. + /// 3. The `notify_one` method is called twice, adding only a single + /// permit to the `Notify`. + /// 4. Both calls to `recv` reach the `Notified` future. One of them + /// consumes the permit, and the other sleeps forever. + /// + /// By adding the `Notified` futures to the list by calling `enable` before + /// `try_recv`, the `notify_one` calls in step three would remove the + /// futures from the list and mark them notified instead of adding a permit + /// to the `Notify`. This ensures that both futures are woken. + /// + /// ``` + /// use tokio::sync::Notify; + /// + /// use std::collections::VecDeque; + /// use std::sync::Mutex; + /// + /// struct Channel { + /// messages: Mutex>, + /// notify_on_sent: Notify, + /// } + /// + /// impl Channel { + /// pub fn send(&self, msg: T) { + /// let mut locked_queue = self.messages.lock().unwrap(); + /// locked_queue.push_back(msg); + /// drop(locked_queue); + /// + /// // Send a notification to one of the calls currently + /// // waiting in a call to `recv`. + /// self.notify_on_sent.notify_one(); + /// } + /// + /// pub fn try_recv(&self) -> Option { + /// let mut locked_queue = self.messages.lock().unwrap(); + /// locked_queue.pop_front() + /// } + /// + /// pub async fn recv(&self) -> T { + /// let future = self.notify_on_sent.notified(); + /// tokio::pin!(future); + /// + /// loop { + /// // Make sure that no wakeup is lost if we get + /// // `None` from `try_recv`. + /// future.as_mut().enable(); + /// + /// if let Some(msg) = self.try_recv() { + /// return msg; + /// } + /// + /// // Wait for a call to `notify_one`. + /// // + /// // This uses `.as_mut()` to avoid consuming the future, + /// // which lets us call `Pin::set` below. + /// future.as_mut().await; + /// + /// // Reset the future in case another call to + /// // `try_recv` got the message before us. + /// future.set(self.notify_on_sent.notified()); + /// } + /// } + /// } + /// ``` + /// + /// [`notify_one`]: Notify::notify_one() + /// [`notify_waiters`]: Notify::notify_waiters() + pub fn enable(self: Pin<&mut Self>) -> bool { + self.poll_notified(None).is_ready() + } + /// A custom `project` implementation is used in place of `pin-project-lite` /// as a custom drop implementation is needed. - fn project(self: Pin<&mut Self>) -> (&Notify, &mut State, &UnsafeCell) { + fn project(self: Pin<&mut Self>) -> (&Notify, &mut State, &usize, &Waiter) { unsafe { - // Safety: both `notify` and `state` are `Unpin`. + // Safety: `notify`, `state` and `notify_waiters_calls` are `Unpin`. is_unpin::<&Notify>(); - is_unpin::(); + is_unpin::(); + is_unpin::(); let me = self.get_unchecked_mut(); - (me.notify, &mut me.state, &me.waiter) + ( + me.notify, + &mut me.state, + &me.notify_waiters_calls, + &me.waiter, + ) } } -} -impl Future for Notified<'_> { - type Output = (); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + fn poll_notified(self: Pin<&mut Self>, waker: Option<&Waker>) -> Poll<()> { use State::*; - let (notify, state, waiter) = self.project(); + let (notify, state, notify_waiters_calls, waiter) = self.project(); - loop { + 'outer_loop: loop { match *state { - Init(initial_notify_waiters_calls) => { + Init => { let curr = notify.state.load(SeqCst); // Optimistically try acquiring a pending notification @@ -552,12 +901,12 @@ impl Future for Notified<'_> { if res.is_ok() { // Acquired the notification *state = Done; - return Poll::Ready(()); + continue 'outer_loop; } // Clone the waker before locking, a waker clone can be // triggering arbitrary code. - let waker = cx.waker().clone(); + let waker = waker.cloned(); // Acquire the lock and attempt to transition to the waiting // state. @@ -568,9 +917,9 @@ impl Future for Notified<'_> { // if notify_waiters has been called after the future // was created, then we are done - if get_num_notify_waiters_calls(curr) != initial_notify_waiters_calls { + if get_num_notify_waiters_calls(curr) != *notify_waiters_calls { *state = Done; - return Poll::Ready(()); + continue 'outer_loop; } // Transition the state to WAITING. @@ -606,7 +955,7 @@ impl Future for Notified<'_> { Ok(_) => { // Acquired the notification *state = Done; - return Poll::Ready(()); + continue 'outer_loop; } Err(actual) => { assert_eq!(get_state(actual), EMPTY); @@ -618,44 +967,109 @@ impl Future for Notified<'_> { } } - // Safety: called while locked. - unsafe { - (*waiter.get()).waker = Some(waker); + let mut old_waker = None; + if waker.is_some() { + // Safety: called while locked. + // + // The use of `old_waiter` here is not necessary, as the field is always + // None when we reach this line. + unsafe { + old_waker = + waiter.waker.with_mut(|v| std::mem::replace(&mut *v, waker)); + } } // Insert the waiter into the linked list - // - // safety: pointers from `UnsafeCell` are never null. - waiters.push_front(unsafe { NonNull::new_unchecked(waiter.get()) }); + waiters.push_front(NonNull::from(waiter)); *state = Waiting; + drop(waiters); + drop(old_waker); + return Poll::Pending; } Waiting => { - // Currently in the "Waiting" state, implying the caller has - // a waiter stored in the waiter list (guarded by - // `notify.waiters`). In order to access the waker fields, - // we must hold the lock. + #[cfg(tokio_taskdump)] + if let Some(waker) = waker { + let mut ctx = Context::from_waker(waker); + ready!(crate::trace::trace_leaf(&mut ctx)); + } - let waiters = notify.waiters.lock(); + if waiter.notification.load(Acquire).is_some() { + // Safety: waiter is already unlinked and will not be shared again, + // so we have an exclusive access to `waker`. + drop(unsafe { waiter.waker.with_mut(|waker| (*waker).take()) }); - // Safety: called while locked - let w = unsafe { &mut *waiter.get() }; + waiter.notification.clear(); + *state = Done; + return Poll::Ready(()); + } - if w.notified.is_some() { - // Our waker has been notified. Reset the fields and - // remove it from the list. - w.waker = None; - w.notified = None; + // Our waiter was not notified, implying it is still stored in a waiter + // list (guarded by `notify.waiters`). In order to access the waker + // fields, we must acquire the lock. + + let mut old_waker = None; + let mut waiters = notify.waiters.lock(); + + // We hold the lock and notifications are set only with the lock held, + // so this can be relaxed, because the happens-before relationship is + // established through the mutex. + if waiter.notification.load(Relaxed).is_some() { + // Safety: waiter is already unlinked and will not be shared again, + // so we have an exclusive access to `waker`. + old_waker = unsafe { waiter.waker.with_mut(|waker| (*waker).take()) }; + + waiter.notification.clear(); + + // Drop the old waker after releasing the lock. + drop(waiters); + drop(old_waker); + + *state = Done; + return Poll::Ready(()); + } + + // Load the state with the lock held. + let curr = notify.state.load(SeqCst); + + if get_num_notify_waiters_calls(curr) != *notify_waiters_calls { + // Before we add a waiter to the list we check if these numbers are + // different while holding the lock. If these numbers are different now, + // it means that there is a call to `notify_waiters` in progress and this + // waiter must be contained by a guarded list used in `notify_waiters`. + // We can treat the waiter as notified and remove it from the list, as + // it would have been notified in the `notify_waiters` call anyways. + + // Safety: we hold the lock, so we can modify the waker. + old_waker = unsafe { waiter.waker.with_mut(|waker| (*waker).take()) }; + + // Safety: we hold the lock, so we have an exclusive access to the list. + // The list is used in `notify_waiters`, so it must be guarded. + unsafe { waiters.remove(NonNull::from(waiter)) }; *state = Done; } else { - // Update the waker, if necessary. - if !w.waker.as_ref().unwrap().will_wake(cx.waker()) { - w.waker = Some(cx.waker().clone()); + // Safety: we hold the lock, so we can modify the waker. + unsafe { + waiter.waker.with_mut(|v| { + if let Some(waker) = waker { + let should_update = match &*v { + Some(current_waker) => !current_waker.will_wake(waker), + None => true, + }; + if should_update { + old_waker = std::mem::replace(&mut *v, Some(waker.clone())); + } + } + }); } + // Drop the old waker after releasing the lock. + drop(waiters); + drop(old_waker); + return Poll::Pending; } @@ -665,8 +1079,16 @@ impl Future for Notified<'_> { // is helpful to visualize the scope of the critical // section. drop(waiters); + + // Drop the old waker after releasing the lock. + drop(old_waker); } Done => { + #[cfg(tokio_taskdump)] + if let Some(waker) = waker { + let mut ctx = Context::from_waker(waker); + ready!(crate::trace::trace_leaf(&mut ctx)); + } return Poll::Ready(()); } } @@ -674,40 +1096,48 @@ impl Future for Notified<'_> { } } +impl Future for Notified<'_> { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + self.poll_notified(Some(cx.waker())) + } +} + impl Drop for Notified<'_> { fn drop(&mut self) { use State::*; // Safety: The type only transitions to a "Waiting" state when pinned. - let (notify, state, waiter) = unsafe { Pin::new_unchecked(self).project() }; + let (notify, state, _, waiter) = unsafe { Pin::new_unchecked(self).project() }; // This is where we ensure safety. The `Notified` value is being // dropped, which means we must ensure that the waiter entry is no // longer stored in the linked list. - if let Waiting = *state { + if matches!(*state, Waiting) { let mut waiters = notify.waiters.lock(); let mut notify_state = notify.state.load(SeqCst); + // We hold the lock, so this field is not concurrently accessed by + // `notify_*` functions and we can use the relaxed ordering. + let notification = waiter.notification.load(Relaxed); + // remove the entry from the list (if not already removed) // - // safety: the waiter is only added to `waiters` by virtue of it - // being the only `LinkedList` available to the type. - unsafe { waiters.remove(NonNull::new_unchecked(waiter.get())) }; + // Safety: we hold the lock, so we have an exclusive access to every list the + // waiter may be contained in. If the node is not contained in the `waiters` + // list, then it is contained by a guarded list used by `notify_waiters`. + unsafe { waiters.remove(NonNull::from(waiter)) }; - if waiters.is_empty() { - if let WAITING = get_state(notify_state) { - notify_state = set_state(notify_state, EMPTY); - notify.state.store(notify_state, SeqCst); - } + if waiters.is_empty() && get_state(notify_state) == WAITING { + notify_state = set_state(notify_state, EMPTY); + notify.state.store(notify_state, SeqCst); } // See if the node was notified but not received. In this case, if // the notification was triggered via `notify_one`, it must be sent // to the next waiter. - // - // Safety: with the entry removed from the linked list, there can be - // no concurrent access to the entry - if let Some(NotificationType::OneWaiter) = unsafe { (*waiter.get()).notified } { + if notification == Some(Notification::One) { if let Some(waker) = notify_locked(&mut waiters, ¬ify.state, notify_state) { drop(waiters); waker.wake(); @@ -733,7 +1163,7 @@ unsafe impl linked_list::Link for Waiter { } unsafe fn pointers(target: NonNull) -> NonNull> { - target.cast() + Waiter::addr_of_pointers(target) } } diff --git a/third_party/rust/tokio/src/sync/once_cell.rs b/third_party/rust/tokio/src/sync/once_cell.rs index d31a40e2c854..90ea5cd68625 100644 --- a/third_party/rust/tokio/src/sync/once_cell.rs +++ b/third_party/rust/tokio/src/sync/once_cell.rs @@ -106,7 +106,7 @@ impl Drop for OnceCell { if self.initialized_mut() { unsafe { self.value - .with_mut(|ptr| ptr::drop_in_place((&mut *ptr).as_mut_ptr())); + .with_mut(|ptr| ptr::drop_in_place((*ptr).as_mut_ptr())); }; } } @@ -301,6 +301,8 @@ impl OnceCell { F: FnOnce() -> Fut, Fut: Future, { + crate::trace::async_trace_leaf().await; + if self.initialized() { // SAFETY: The OnceCell has been fully initialized. unsafe { self.get_unchecked() } @@ -349,6 +351,8 @@ impl OnceCell { F: FnOnce() -> Fut, Fut: Future>, { + crate::trace::async_trace_leaf().await; + if self.initialized() { // SAFETY: The OnceCell has been fully initialized. unsafe { Ok(self.get_unchecked()) } @@ -416,7 +420,7 @@ unsafe impl Send for OnceCell {} /// Errors that can be returned from [`OnceCell::set`]. /// /// [`OnceCell::set`]: crate::sync::OnceCell::set -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum SetError { /// The cell was already initialized when [`OnceCell::set`] was called. /// diff --git a/third_party/rust/tokio/src/sync/oneshot.rs b/third_party/rust/tokio/src/sync/oneshot.rs index 2240074e733d..af3cc854f882 100644 --- a/third_party/rust/tokio/src/sync/oneshot.rs +++ b/third_party/rust/tokio/src/sync/oneshot.rs @@ -12,6 +12,10 @@ //! Since the `send` method is not async, it can be used anywhere. This includes //! sending between two runtimes, and using it from non-async code. //! +//! If the [`Receiver`] is closed before receiving a message which has already +//! been sent, the message will remain in the channel until the receiver is +//! dropped, at which point the message will be dropped immediately. +//! //! # Examples //! //! ``` @@ -227,7 +231,15 @@ pub struct Sender { /// [`channel`](fn@channel) function. /// /// This channel has no `recv` method because the receiver itself implements the -/// [`Future`] trait. To receive a value, `.await` the `Receiver` object directly. +/// [`Future`] trait. To receive a `Result`, `.await` the `Receiver` object directly. +/// +/// The `poll` method on the `Future` trait is allowed to spuriously return +/// `Poll::Pending` even if the message has been sent. If such a spurious +/// failure happens, then the caller will be woken when the spurious failure has +/// been resolved so that the caller can attempt to receive the message again. +/// Note that receiving such a wakeup does not guarantee that the next call will +/// succeed — it could fail with another spurious failure. (A spurious failure +/// does not mean that the message is lost. It is just delayed.) /// /// [`Future`]: trait@std::future::Future /// @@ -323,11 +335,13 @@ pub mod error { use std::fmt; /// Error returned by the `Future` implementation for `Receiver`. - #[derive(Debug, Eq, PartialEq)] + /// + /// This error is returned by the receiver when the sender is dropped without sending. + #[derive(Debug, Eq, PartialEq, Clone)] pub struct RecvError(pub(super) ()); /// Error returned by the `try_recv` function on `Receiver`. - #[derive(Debug, Eq, PartialEq)] + #[derive(Debug, Eq, PartialEq, Clone)] pub enum TryRecvError { /// The send half of the channel has not yet sent a value. Empty, @@ -399,21 +413,21 @@ impl Task { F: FnOnce(&Waker) -> R, { self.0.with(|ptr| { - let waker: *const Waker = (&*ptr).as_ptr(); + let waker: *const Waker = (*ptr).as_ptr(); f(&*waker) }) } unsafe fn drop_task(&self) { self.0.with_mut(|ptr| { - let ptr: *mut Waker = (&mut *ptr).as_mut_ptr(); + let ptr: *mut Waker = (*ptr).as_mut_ptr(); ptr.drop_in_place(); }); } unsafe fn set_task(&self, cx: &mut Context<'_>) { self.0.with_mut(|ptr| { - let ptr: *mut Waker = (&mut *ptr).as_mut_ptr(); + let ptr: *mut Waker = (*ptr).as_mut_ptr(); ptr.write(cx.waker().clone()); }); } @@ -526,7 +540,7 @@ pub fn channel() -> (Sender, Receiver) { let rx = Receiver { inner: Some(inner), #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: resource_span, + resource_span, #[cfg(all(tokio_unstable, feature = "tracing"))] async_op_span, #[cfg(all(tokio_unstable, feature = "tracing"))] @@ -776,8 +790,10 @@ impl Sender { /// } /// ``` pub fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll<()> { + ready!(crate::trace::trace_leaf(cx)); + // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let inner = self.inner.as_ref().unwrap(); @@ -785,7 +801,7 @@ impl Sender { if state.is_closed() { coop.made_progress(); - return Poll::Ready(()); + return Ready(()); } if state.is_tx_task_set() { @@ -923,12 +939,16 @@ impl Receiver { /// This function is useful to call from outside the context of an /// asynchronous task. /// + /// Note that unlike the `poll` method, the `try_recv` method cannot fail + /// spuriously. Any send or close event that happens before this call to + /// `try_recv` will be correctly returned to the caller. + /// /// # Return /// /// - `Ok(T)` if a value is pending in the channel. /// - `Err(TryRecvError::Empty)` if no value has been sent yet. /// - `Err(TryRecvError::Closed)` if the sender has dropped without sending - /// a value. + /// a value, or if the message has already been received. /// /// # Examples /// @@ -1040,7 +1060,9 @@ impl Receiver { /// sync_code.join().unwrap(); /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] + #[cfg_attr(docsrs, doc(alias = "recv_blocking"))] pub fn blocking_recv(self) -> Result { crate::future::block_on(self) } @@ -1110,8 +1132,9 @@ impl Inner { } fn poll_recv(&self, cx: &mut Context<'_>) -> Poll> { + ready!(crate::trace::trace_leaf(cx)); // Keep track of task budget - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); // Load the state let mut state = State::load(&self.state, Acquire); diff --git a/third_party/rust/tokio/src/sync/rwlock.rs b/third_party/rust/tokio/src/sync/rwlock.rs index b856cfc85677..dd4928546fca 100644 --- a/third_party/rust/tokio/src/sync/rwlock.rs +++ b/third_party/rust/tokio/src/sync/rwlock.rs @@ -5,7 +5,6 @@ use crate::util::trace; use std::cell::UnsafeCell; use std::marker; use std::marker::PhantomData; -use std::mem::ManuallyDrop; use std::sync::Arc; pub(crate) mod owned_read_guard; @@ -86,7 +85,6 @@ const MAX_READS: u32 = 10; /// [`RwLockWriteGuard`]: struct@RwLockWriteGuard /// [`Send`]: trait@std::marker::Send /// [_write-preferring_]: https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock#Priority_policies -#[derive(Debug)] pub struct RwLock { #[cfg(all(tokio_unstable, feature = "tracing"))] resource_span: tracing::Span, @@ -423,23 +421,33 @@ impl RwLock { /// } /// ``` pub async fn read(&self) -> RwLockReadGuard<'_, T> { + let acquire_fut = async { + self.s.acquire(1).await.unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); + + RwLockReadGuard { + s: &self.s, + data: self.c.get(), + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + } + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] - let inner = trace::async_op( - || self.s.acquire(1), + let acquire_fut = trace::async_op( + move || acquire_fut, self.resource_span.clone(), "RwLock::read", "poll", false, ); - #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let inner = self.s.acquire(1); - - inner.await.unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + #[allow(clippy::let_and_return)] // this lint triggers when disabling tracing + let guard = acquire_fut.await; #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { @@ -450,13 +458,7 @@ impl RwLock { ) }); - RwLockReadGuard { - s: &self.s, - data: self.c.get(), - marker: marker::PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: self.resource_span.clone(), - } + guard } /// Blockingly locks this `RwLock` with shared read access. @@ -506,6 +508,7 @@ impl RwLock { /// assert!(rwlock.try_write().is_ok()); /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] pub fn blocking_read(&self) -> RwLockReadGuard<'_, T> { crate::future::block_on(self.read()) @@ -564,25 +567,38 @@ impl RwLock { /// ``` pub async fn read_owned(self: Arc) -> OwnedRwLockReadGuard { #[cfg(all(tokio_unstable, feature = "tracing"))] - let inner = trace::async_op( - || self.s.acquire(1), - self.resource_span.clone(), + let resource_span = self.resource_span.clone(); + + let acquire_fut = async { + self.s.acquire(1).await.unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); + + OwnedRwLockReadGuard { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + data: self.c.get(), + lock: self, + _p: PhantomData, + } + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let acquire_fut = trace::async_op( + move || acquire_fut, + resource_span, "RwLock::read_owned", "poll", false, ); - #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let inner = self.s.acquire(1); - - inner.await.unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + #[allow(clippy::let_and_return)] // this lint triggers when disabling tracing + let guard = acquire_fut.await; #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", current_readers = 1, @@ -590,16 +606,7 @@ impl RwLock { ) }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - - OwnedRwLockReadGuard { - data: self.c.get(), - lock: ManuallyDrop::new(self), - _p: PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - } + guard } /// Attempts to acquire this `RwLock` with shared read access. @@ -641,6 +648,14 @@ impl RwLock { Err(TryAcquireError::Closed) => unreachable!(), } + let guard = RwLockReadGuard { + s: &self.s, + data: self.c.get(), + marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { tracing::trace!( @@ -650,13 +665,7 @@ impl RwLock { ) }); - Ok(RwLockReadGuard { - s: &self.s, - data: self.c.get(), - marker: marker::PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: self.resource_span.clone(), - }) + Ok(guard) } /// Attempts to acquire this `RwLock` with shared read access. @@ -704,8 +713,16 @@ impl RwLock { Err(TryAcquireError::Closed) => unreachable!(), } + let guard = OwnedRwLockReadGuard { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + data: self.c.get(), + lock: self, + _p: PhantomData, + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", current_readers = 1, @@ -713,16 +730,7 @@ impl RwLock { ) }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - - Ok(OwnedRwLockReadGuard { - data: self.c.get(), - lock: ManuallyDrop::new(self), - _p: PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - }) + Ok(guard) } /// Locks this `RwLock` with exclusive write access, causing the current @@ -754,23 +762,34 @@ impl RwLock { ///} /// ``` pub async fn write(&self) -> RwLockWriteGuard<'_, T> { + let acquire_fut = async { + self.s.acquire(self.mr).await.unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); + + RwLockWriteGuard { + permits_acquired: self.mr, + s: &self.s, + data: self.c.get(), + marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + } + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] - let inner = trace::async_op( - || self.s.acquire(self.mr), + let acquire_fut = trace::async_op( + move || acquire_fut, self.resource_span.clone(), "RwLock::write", "poll", false, ); - #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let inner = self.s.acquire(self.mr); - - inner.await.unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + #[allow(clippy::let_and_return)] // this lint triggers when disabling tracing + let guard = acquire_fut.await; #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { @@ -781,14 +800,7 @@ impl RwLock { ) }); - RwLockWriteGuard { - permits_acquired: self.mr, - s: &self.s, - data: self.c.get(), - marker: marker::PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: self.resource_span.clone(), - } + guard } /// Blockingly locks this `RwLock` with exclusive write access. @@ -840,6 +852,7 @@ impl RwLock { /// assert_eq!(*read_lock, 2); /// } /// ``` + #[track_caller] #[cfg(feature = "sync")] pub fn blocking_write(&self) -> RwLockWriteGuard<'_, T> { crate::future::block_on(self.write()) @@ -882,25 +895,39 @@ impl RwLock { /// ``` pub async fn write_owned(self: Arc) -> OwnedRwLockWriteGuard { #[cfg(all(tokio_unstable, feature = "tracing"))] - let inner = trace::async_op( - || self.s.acquire(self.mr), - self.resource_span.clone(), + let resource_span = self.resource_span.clone(); + + let acquire_fut = async { + self.s.acquire(self.mr).await.unwrap_or_else(|_| { + // The semaphore was closed. but, we never explicitly close it, and we have a + // handle to it through the Arc, which means that this can never happen. + unreachable!() + }); + + OwnedRwLockWriteGuard { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + permits_acquired: self.mr, + data: self.c.get(), + lock: self, + _p: PhantomData, + } + }; + + #[cfg(all(tokio_unstable, feature = "tracing"))] + let acquire_fut = trace::async_op( + move || acquire_fut, + resource_span, "RwLock::write_owned", "poll", false, ); - #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let inner = self.s.acquire(self.mr); - - inner.await.unwrap_or_else(|_| { - // The semaphore was closed. but, we never explicitly close it, and we have a - // handle to it through the Arc, which means that this can never happen. - unreachable!() - }); + #[allow(clippy::let_and_return)] // this lint triggers when disabling tracing + let guard = acquire_fut.await; #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", write_locked = true, @@ -908,17 +935,7 @@ impl RwLock { ) }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - - OwnedRwLockWriteGuard { - permits_acquired: self.mr, - data: self.c.get(), - lock: ManuallyDrop::new(self), - _p: PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - } + guard } /// Attempts to acquire this `RwLock` with exclusive write access. @@ -951,6 +968,15 @@ impl RwLock { Err(TryAcquireError::Closed) => unreachable!(), } + let guard = RwLockWriteGuard { + permits_acquired: self.mr, + s: &self.s, + data: self.c.get(), + marker: marker::PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { tracing::trace!( @@ -960,14 +986,7 @@ impl RwLock { ) }); - Ok(RwLockWriteGuard { - permits_acquired: self.mr, - s: &self.s, - data: self.c.get(), - marker: marker::PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span: self.resource_span.clone(), - }) + Ok(guard) } /// Attempts to acquire this `RwLock` with exclusive write access. @@ -1007,8 +1026,17 @@ impl RwLock { Err(TryAcquireError::Closed) => unreachable!(), } + let guard = OwnedRwLockWriteGuard { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: self.resource_span.clone(), + permits_acquired: self.mr, + data: self.c.get(), + lock: self, + _p: PhantomData, + }; + #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", write_locked = true, @@ -1016,17 +1044,7 @@ impl RwLock { ) }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - - Ok(OwnedRwLockWriteGuard { - permits_acquired: self.mr, - data: self.c.get(), - lock: ManuallyDrop::new(self), - _p: PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - }) + Ok(guard) } /// Returns a mutable reference to the underlying data. @@ -1076,3 +1094,17 @@ where Self::new(T::default()) } } + +impl std::fmt::Debug for RwLock +where + T: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("RwLock"); + match self.try_read() { + Ok(inner) => d.field("data", &&*inner), + Err(_) => d.field("data", &format_args!("")), + }; + d.finish() + } +} diff --git a/third_party/rust/tokio/src/sync/rwlock/owned_read_guard.rs b/third_party/rust/tokio/src/sync/rwlock/owned_read_guard.rs index 27b71bd988be..273e7b86f2fc 100644 --- a/third_party/rust/tokio/src/sync/rwlock/owned_read_guard.rs +++ b/third_party/rust/tokio/src/sync/rwlock/owned_read_guard.rs @@ -1,10 +1,7 @@ use crate::sync::rwlock::RwLock; -use std::fmt; use std::marker::PhantomData; -use std::mem; -use std::mem::ManuallyDrop; -use std::ops; use std::sync::Arc; +use std::{fmt, mem, ops, ptr}; /// Owned RAII structure used to release the shared read access of a lock when /// dropped. @@ -14,16 +11,40 @@ use std::sync::Arc; /// /// [`read_owned`]: method@crate::sync::RwLock::read_owned /// [`RwLock`]: struct@crate::sync::RwLock +#[clippy::has_significant_drop] pub struct OwnedRwLockReadGuard { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) resource_span: tracing::Span, - // ManuallyDrop allows us to destructure into this field without running the destructor. - pub(super) lock: ManuallyDrop>>, + pub(super) lock: Arc>, pub(super) data: *const U, pub(super) _p: PhantomData, } +#[allow(dead_code)] // Unused fields are still used in Drop. +struct Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + lock: Arc>, + data: *const U, +} + impl OwnedRwLockReadGuard { + fn skip_drop(self) -> Inner { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + unsafe { + Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: ptr::read(&me.resource_span), + lock: ptr::read(&me.lock), + data: me.data, + } + } + } + /// Makes a new `OwnedRwLockReadGuard` for a component of the locked data. /// This operation cannot fail as the `OwnedRwLockReadGuard` passed in /// already locked the data. @@ -52,23 +73,19 @@ impl OwnedRwLockReadGuard { /// # } /// ``` #[inline] - pub fn map(mut this: Self, f: F) -> OwnedRwLockReadGuard + pub fn map(this: Self, f: F) -> OwnedRwLockReadGuard where F: FnOnce(&U) -> &V, { let data = f(&*this) as *const V; - let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); OwnedRwLockReadGuard { - lock: ManuallyDrop::new(lock), + lock: this.lock, data, _p: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, } } @@ -103,7 +120,7 @@ impl OwnedRwLockReadGuard { /// # } /// ``` #[inline] - pub fn try_map(mut this: Self, f: F) -> Result, Self> + pub fn try_map(this: Self, f: F) -> Result, Self> where F: FnOnce(&U) -> Option<&V>, { @@ -111,18 +128,14 @@ impl OwnedRwLockReadGuard { Some(data) => data as *const V, None => return Err(this), }; - let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); Ok(OwnedRwLockReadGuard { - lock: ManuallyDrop::new(lock), + lock: this.lock, data, _p: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, }) } } @@ -156,7 +169,6 @@ where impl Drop for OwnedRwLockReadGuard { fn drop(&mut self) { self.lock.s.release(1); - unsafe { ManuallyDrop::drop(&mut self.lock) }; #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { diff --git a/third_party/rust/tokio/src/sync/rwlock/owned_write_guard.rs b/third_party/rust/tokio/src/sync/rwlock/owned_write_guard.rs index dbedab4cbb21..a8ce4a1603f3 100644 --- a/third_party/rust/tokio/src/sync/rwlock/owned_write_guard.rs +++ b/third_party/rust/tokio/src/sync/rwlock/owned_write_guard.rs @@ -1,11 +1,9 @@ use crate::sync::rwlock::owned_read_guard::OwnedRwLockReadGuard; use crate::sync::rwlock::owned_write_guard_mapped::OwnedRwLockMappedWriteGuard; use crate::sync::rwlock::RwLock; -use std::fmt; use std::marker::PhantomData; -use std::mem::{self, ManuallyDrop}; -use std::ops; use std::sync::Arc; +use std::{fmt, mem, ops, ptr}; /// Owned RAII structure used to release the exclusive write access of a lock when /// dropped. @@ -15,17 +13,43 @@ use std::sync::Arc; /// /// [`write_owned`]: method@crate::sync::RwLock::write_owned /// [`RwLock`]: struct@crate::sync::RwLock +#[clippy::has_significant_drop] pub struct OwnedRwLockWriteGuard { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, - // ManuallyDrop allows us to destructure into this field without running the destructor. - pub(super) lock: ManuallyDrop>>, + pub(super) lock: Arc>, pub(super) data: *mut T, pub(super) _p: PhantomData, } +#[allow(dead_code)] // Unused fields are still used in Drop. +struct Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + permits_acquired: u32, + lock: Arc>, + data: *const T, +} + impl OwnedRwLockWriteGuard { + fn skip_drop(self) -> Inner { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + unsafe { + Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: ptr::read(&me.resource_span), + permits_acquired: me.permits_acquired, + lock: ptr::read(&me.lock), + data: me.data, + } + } + } + /// Makes a new [`OwnedRwLockMappedWriteGuard`] for a component of the locked /// data. /// @@ -64,24 +88,90 @@ impl OwnedRwLockWriteGuard { F: FnOnce(&mut T) -> &mut U, { let data = f(&mut *this) as *mut U; - let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; - let permits_acquired = this.permits_acquired; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); OwnedRwLockMappedWriteGuard { - permits_acquired, - lock: ManuallyDrop::new(lock), + permits_acquired: this.permits_acquired, + lock: this.lock, data, _p: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, } } - /// Attempts to make a new [`OwnedRwLockMappedWriteGuard`] for a component + /// Makes a new [`OwnedRwLockReadGuard`] for a component of the locked data. + /// + /// This operation cannot fail as the `OwnedRwLockWriteGuard` passed in already + /// locked the data. + /// + /// This is an associated function that needs to be used as + /// `OwnedRwLockWriteGuard::downgrade_map(..)`. A method would interfere with methods of + /// the same name on the contents of the locked data. + /// + /// Inside of `f`, you retain exclusive access to the data, despite only being given a `&T`. Handing out a + /// `&mut T` would result in unsoundness, as you could use interior mutability. + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// use tokio::sync::{RwLock, OwnedRwLockWriteGuard}; + /// + /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] + /// struct Foo(u32); + /// + /// # #[tokio::main] + /// # async fn main() { + /// let lock = Arc::new(RwLock::new(Foo(1))); + /// + /// let guard = Arc::clone(&lock).write_owned().await; + /// let mapped = OwnedRwLockWriteGuard::downgrade_map(guard, |f| &f.0); + /// let foo = lock.read_owned().await; + /// assert_eq!(foo.0, *mapped); + /// # } + /// ``` + #[inline] + pub fn downgrade_map(this: Self, f: F) -> OwnedRwLockReadGuard + where + F: FnOnce(&T) -> &U, + { + let data = f(&*this) as *const U; + let this = this.skip_drop(); + let guard = OwnedRwLockReadGuard { + lock: this.lock, + data, + _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: this.resource_span, + }; + + // Release all but one of the permits held by the write guard + let to_release = (this.permits_acquired - 1) as usize; + guard.lock.s.release(to_release); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + guard + } + + /// Attempts to make a new [`OwnedRwLockMappedWriteGuard`] for a component /// of the locked data. The original guard is returned if the closure /// returns `None`. /// @@ -128,24 +218,99 @@ impl OwnedRwLockWriteGuard { Some(data) => data as *mut U, None => return Err(this), }; - let permits_acquired = this.permits_acquired; - let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); Ok(OwnedRwLockMappedWriteGuard { - permits_acquired, - lock: ManuallyDrop::new(lock), + permits_acquired: this.permits_acquired, + lock: this.lock, data, _p: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, }) } + /// Attempts to make a new [`OwnedRwLockReadGuard`] for a component of + /// the locked data. The original guard is returned if the closure returns + /// `None`. + /// + /// This operation cannot fail as the `OwnedRwLockWriteGuard` passed in already + /// locked the data. + /// + /// This is an associated function that needs to be + /// used as `OwnedRwLockWriteGuard::try_downgrade_map(...)`. A method would interfere with + /// methods of the same name on the contents of the locked data. + /// + /// Inside of `f`, you retain exclusive access to the data, despite only being given a `&T`. Handing out a + /// `&mut T` would result in unsoundness, as you could use interior mutability. + /// + /// If this function returns `Err(...)`, the lock is never unlocked nor downgraded. + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// use tokio::sync::{RwLock, OwnedRwLockWriteGuard}; + /// + /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] + /// struct Foo(u32); + /// + /// # #[tokio::main] + /// # async fn main() { + /// let lock = Arc::new(RwLock::new(Foo(1))); + /// + /// let guard = Arc::clone(&lock).write_owned().await; + /// let guard = OwnedRwLockWriteGuard::try_downgrade_map(guard, |f| Some(&f.0)).expect("should not fail"); + /// let foo = lock.read_owned().await; + /// assert_eq!(foo.0, *guard); + /// # } + /// ``` + #[inline] + pub fn try_downgrade_map( + this: Self, + f: F, + ) -> Result, Self> + where + F: FnOnce(&T) -> Option<&U>, + { + let data = match f(&*this) { + Some(data) => data as *const U, + None => return Err(this), + }; + let this = this.skip_drop(); + let guard = OwnedRwLockReadGuard { + lock: this.lock, + data, + _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: this.resource_span, + }; + + // Release all but one of the permits held by the write guard + let to_release = (this.permits_acquired - 1) as usize; + guard.lock.s.release(to_release); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + Ok(guard) + } + /// Converts this `OwnedRwLockWriteGuard` into an /// `OwnedRwLockMappedWriteGuard`. This method can be used to store a /// non-mapped guard in a struct field that expects a mapped guard. @@ -191,15 +356,22 @@ impl OwnedRwLockWriteGuard { /// assert_eq!(*lock.read().await, 2, "second writer obtained write lock"); /// # } /// ``` - pub fn downgrade(mut self) -> OwnedRwLockReadGuard { - let lock = unsafe { ManuallyDrop::take(&mut self.lock) }; - let data = self.data; - let to_release = (self.permits_acquired - 1) as usize; + pub fn downgrade(self) -> OwnedRwLockReadGuard { + let this = self.skip_drop(); + let guard = OwnedRwLockReadGuard { + lock: this.lock, + data: this.data, + _p: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: this.resource_span, + }; // Release all but one of the permits held by the write guard - lock.s.release(to_release); + let to_release = (this.permits_acquired - 1) as usize; + guard.lock.s.release(to_release); + #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", write_locked = false, @@ -208,7 +380,7 @@ impl OwnedRwLockWriteGuard { }); #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", current_readers = 1, @@ -216,18 +388,7 @@ impl OwnedRwLockWriteGuard { ) }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(self); - - OwnedRwLockReadGuard { - lock: ManuallyDrop::new(lock), - data, - _p: PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - } + guard } } @@ -266,6 +427,7 @@ where impl Drop for OwnedRwLockWriteGuard { fn drop(&mut self) { self.lock.s.release(self.permits_acquired as usize); + #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { tracing::trace!( @@ -274,6 +436,5 @@ impl Drop for OwnedRwLockWriteGuard { write_locked.op = "override", ) }); - unsafe { ManuallyDrop::drop(&mut self.lock) }; } } diff --git a/third_party/rust/tokio/src/sync/rwlock/owned_write_guard_mapped.rs b/third_party/rust/tokio/src/sync/rwlock/owned_write_guard_mapped.rs index 55a24d96ac38..9f4952100a57 100644 --- a/third_party/rust/tokio/src/sync/rwlock/owned_write_guard_mapped.rs +++ b/third_party/rust/tokio/src/sync/rwlock/owned_write_guard_mapped.rs @@ -1,9 +1,7 @@ use crate::sync::rwlock::RwLock; -use std::fmt; use std::marker::PhantomData; -use std::mem::{self, ManuallyDrop}; -use std::ops; use std::sync::Arc; +use std::{fmt, mem, ops, ptr}; /// Owned RAII structure used to release the exclusive write access of a lock when /// dropped. @@ -14,17 +12,43 @@ use std::sync::Arc; /// /// [mapping]: method@crate::sync::OwnedRwLockWriteGuard::map /// [`OwnedRwLockWriteGuard`]: struct@crate::sync::OwnedRwLockWriteGuard +#[clippy::has_significant_drop] pub struct OwnedRwLockMappedWriteGuard { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, - // ManuallyDrop allows us to destructure into this field without running the destructor. - pub(super) lock: ManuallyDrop>>, + pub(super) lock: Arc>, pub(super) data: *mut U, pub(super) _p: PhantomData, } +#[allow(dead_code)] // Unused fields are still used in Drop. +struct Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + permits_acquired: u32, + lock: Arc>, + data: *const U, +} + impl OwnedRwLockMappedWriteGuard { + fn skip_drop(self) -> Inner { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + unsafe { + Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: ptr::read(&me.resource_span), + permits_acquired: me.permits_acquired, + lock: ptr::read(&me.lock), + data: me.data, + } + } + } + /// Makes a new `OwnedRwLockMappedWriteGuard` for a component of the locked /// data. /// @@ -63,20 +87,15 @@ impl OwnedRwLockMappedWriteGuard { F: FnOnce(&mut U) -> &mut V, { let data = f(&mut *this) as *mut V; - let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; - let permits_acquired = this.permits_acquired; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); OwnedRwLockMappedWriteGuard { - permits_acquired, - lock: ManuallyDrop::new(lock), + permits_acquired: this.permits_acquired, + lock: this.lock, data, _p: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, } } @@ -125,20 +144,15 @@ impl OwnedRwLockMappedWriteGuard { Some(data) => data as *mut V, None => return Err(this), }; - let lock = unsafe { ManuallyDrop::take(&mut this.lock) }; - let permits_acquired = this.permits_acquired; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); Ok(OwnedRwLockMappedWriteGuard { - permits_acquired, - lock: ManuallyDrop::new(lock), + permits_acquired: this.permits_acquired, + lock: this.lock, data, _p: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, }) } } @@ -178,6 +192,7 @@ where impl Drop for OwnedRwLockMappedWriteGuard { fn drop(&mut self) { self.lock.s.release(self.permits_acquired as usize); + #[cfg(all(tokio_unstable, feature = "tracing"))] self.resource_span.in_scope(|| { tracing::trace!( @@ -186,6 +201,5 @@ impl Drop for OwnedRwLockMappedWriteGuard { write_locked.op = "override", ) }); - unsafe { ManuallyDrop::drop(&mut self.lock) }; } } diff --git a/third_party/rust/tokio/src/sync/rwlock/read_guard.rs b/third_party/rust/tokio/src/sync/rwlock/read_guard.rs index 369213199231..a04b59588d51 100644 --- a/third_party/rust/tokio/src/sync/rwlock/read_guard.rs +++ b/third_party/rust/tokio/src/sync/rwlock/read_guard.rs @@ -1,8 +1,6 @@ use crate::sync::batch_semaphore::Semaphore; -use std::fmt; -use std::marker; -use std::mem; -use std::ops; +use std::marker::PhantomData; +use std::{fmt, mem, ops}; /// RAII structure used to release the shared read access of a lock when /// dropped. @@ -12,15 +10,39 @@ use std::ops; /// /// [`read`]: method@crate::sync::RwLock::read /// [`RwLock`]: struct@crate::sync::RwLock +#[clippy::has_significant_drop] +#[must_use = "if unused the RwLock will immediately unlock"] pub struct RwLockReadGuard<'a, T: ?Sized> { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) resource_span: tracing::Span, pub(super) s: &'a Semaphore, pub(super) data: *const T, - pub(super) marker: marker::PhantomData<&'a T>, + pub(super) marker: PhantomData<&'a T>, +} + +#[allow(dead_code)] // Unused fields are still used in Drop. +struct Inner<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + s: &'a Semaphore, + data: *const T, } impl<'a, T: ?Sized> RwLockReadGuard<'a, T> { + fn skip_drop(self) -> Inner<'a, T> { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: unsafe { std::ptr::read(&me.resource_span) }, + s: me.s, + data: me.data, + } + } + /// Makes a new `RwLockReadGuard` for a component of the locked data. /// /// This operation cannot fail as the `RwLockReadGuard` passed in already @@ -60,18 +82,14 @@ impl<'a, T: ?Sized> RwLockReadGuard<'a, T> { F: FnOnce(&T) -> &U, { let data = f(&*this) as *const U; - let s = this.s; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); RwLockReadGuard { - s, + s: this.s, data, - marker: marker::PhantomData, + marker: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, } } @@ -119,18 +137,14 @@ impl<'a, T: ?Sized> RwLockReadGuard<'a, T> { Some(data) => data as *const U, None => return Err(this), }; - let s = this.s; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); Ok(RwLockReadGuard { - s, + s: this.s, data, - marker: marker::PhantomData, + marker: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, }) } } diff --git a/third_party/rust/tokio/src/sync/rwlock/write_guard.rs b/third_party/rust/tokio/src/sync/rwlock/write_guard.rs index 7cadd74c60d8..d405fc2b39fb 100644 --- a/third_party/rust/tokio/src/sync/rwlock/write_guard.rs +++ b/third_party/rust/tokio/src/sync/rwlock/write_guard.rs @@ -1,10 +1,8 @@ use crate::sync::batch_semaphore::Semaphore; use crate::sync::rwlock::read_guard::RwLockReadGuard; use crate::sync::rwlock::write_guard_mapped::RwLockMappedWriteGuard; -use std::fmt; -use std::marker; -use std::mem; -use std::ops; +use std::marker::PhantomData; +use std::{fmt, mem, ops}; /// RAII structure used to release the exclusive write access of a lock when /// dropped. @@ -14,16 +12,42 @@ use std::ops; /// /// [`write`]: method@crate::sync::RwLock::write /// [`RwLock`]: struct@crate::sync::RwLock +#[clippy::has_significant_drop] +#[must_use = "if unused the RwLock will immediately unlock"] pub struct RwLockWriteGuard<'a, T: ?Sized> { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, pub(super) s: &'a Semaphore, pub(super) data: *mut T, - pub(super) marker: marker::PhantomData<&'a mut T>, + pub(super) marker: PhantomData<&'a mut T>, +} + +#[allow(dead_code)] // Unused fields are still used in Drop. +struct Inner<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + permits_acquired: u32, + s: &'a Semaphore, + data: *mut T, } impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { + fn skip_drop(self) -> Inner<'a, T> { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: unsafe { std::ptr::read(&me.resource_span) }, + permits_acquired: me.permits_acquired, + s: me.s, + data: me.data, + } + } + /// Makes a new [`RwLockMappedWriteGuard`] for a component of the locked data. /// /// This operation cannot fail as the `RwLockWriteGuard` passed in already @@ -66,23 +90,96 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { F: FnOnce(&mut T) -> &mut U, { let data = f(&mut *this) as *mut U; - let s = this.s; - let permits_acquired = this.permits_acquired; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); + RwLockMappedWriteGuard { - permits_acquired, - s, + permits_acquired: this.permits_acquired, + s: this.s, data, - marker: marker::PhantomData, + marker: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, } } - /// Attempts to make a new [`RwLockMappedWriteGuard`] for a component of + /// Makes a new [`RwLockReadGuard`] for a component of the locked data. + /// + /// This operation cannot fail as the `RwLockWriteGuard` passed in already + /// locked the data. + /// + /// This is an associated function that needs to be used as + /// `RwLockWriteGuard::downgrade_map(..)`. A method would interfere with methods of + /// the same name on the contents of the locked data. + /// + /// This is equivalent to a combination of asynchronous [`RwLockWriteGuard::map`] and [`RwLockWriteGuard::downgrade`] + /// from the [`parking_lot` crate]. + /// + /// Inside of `f`, you retain exclusive access to the data, despite only being given a `&T`. Handing out a + /// `&mut T` would result in unsoundness, as you could use interior mutability. + /// + /// [`RwLockMappedWriteGuard`]: struct@crate::sync::RwLockMappedWriteGuard + /// [`RwLockWriteGuard::map`]: https://docs.rs/lock_api/latest/lock_api/struct.RwLockWriteGuard.html#method.map + /// [`RwLockWriteGuard::downgrade`]: https://docs.rs/lock_api/latest/lock_api/struct.RwLockWriteGuard.html#method.downgrade + /// [`parking_lot` crate]: https://crates.io/crates/parking_lot + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::{RwLock, RwLockWriteGuard}; + /// + /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] + /// struct Foo(u32); + /// + /// # #[tokio::main] + /// # async fn main() { + /// let lock = RwLock::new(Foo(1)); + /// + /// let mapped = RwLockWriteGuard::downgrade_map(lock.write().await, |f| &f.0); + /// let foo = lock.read().await; + /// assert_eq!(foo.0, *mapped); + /// # } + /// ``` + #[inline] + pub fn downgrade_map(this: Self, f: F) -> RwLockReadGuard<'a, U> + where + F: FnOnce(&T) -> &U, + { + let data = f(&*this) as *const U; + let this = this.skip_drop(); + let guard = RwLockReadGuard { + s: this.s, + data, + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: this.resource_span, + }; + + // Release all but one of the permits held by the write guard + let to_release = (this.permits_acquired - 1) as usize; + this.s.release(to_release); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + guard + } + + /// Attempts to make a new [`RwLockMappedWriteGuard`] for a component of /// the locked data. The original guard is returned if the closure returns /// `None`. /// @@ -133,22 +230,102 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { Some(data) => data as *mut U, None => return Err(this), }; - let s = this.s; - let permits_acquired = this.permits_acquired; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); + Ok(RwLockMappedWriteGuard { - permits_acquired, - s, + permits_acquired: this.permits_acquired, + s: this.s, data, - marker: marker::PhantomData, + marker: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, }) } + /// Attempts to make a new [`RwLockReadGuard`] for a component of + /// the locked data. The original guard is returned if the closure returns + /// `None`. + /// + /// This operation cannot fail as the `RwLockWriteGuard` passed in already + /// locked the data. + /// + /// This is an associated function that needs to be + /// used as `RwLockWriteGuard::try_downgrade_map(...)`. A method would interfere with + /// methods of the same name on the contents of the locked data. + /// + /// This is equivalent to a combination of asynchronous [`RwLockWriteGuard::try_map`] and [`RwLockWriteGuard::downgrade`] + /// from the [`parking_lot` crate]. + /// + /// Inside of `f`, you retain exclusive access to the data, despite only being given a `&T`. Handing out a + /// `&mut T` would result in unsoundness, as you could use interior mutability. + /// + /// If this function returns `Err(...)`, the lock is never unlocked nor downgraded. + /// + /// [`RwLockMappedWriteGuard`]: struct@crate::sync::RwLockMappedWriteGuard + /// [`RwLockWriteGuard::map`]: https://docs.rs/lock_api/latest/lock_api/struct.RwLockWriteGuard.html#method.map + /// [`RwLockWriteGuard::downgrade`]: https://docs.rs/lock_api/latest/lock_api/struct.RwLockWriteGuard.html#method.downgrade + /// [`parking_lot` crate]: https://crates.io/crates/parking_lot + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::{RwLock, RwLockWriteGuard}; + /// + /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] + /// struct Foo(u32); + /// + /// # #[tokio::main] + /// # async fn main() { + /// let lock = RwLock::new(Foo(1)); + /// + /// let guard = RwLockWriteGuard::try_downgrade_map(lock.write().await, |f| Some(&f.0)).expect("should not fail"); + /// let foo = lock.read().await; + /// assert_eq!(foo.0, *guard); + /// # } + /// ``` + #[inline] + pub fn try_downgrade_map(this: Self, f: F) -> Result, Self> + where + F: FnOnce(&T) -> Option<&U>, + { + let data = match f(&*this) { + Some(data) => data as *const U, + None => return Err(this), + }; + let this = this.skip_drop(); + let guard = RwLockReadGuard { + s: this.s, + data, + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: this.resource_span, + }; + + // Release all but one of the permits held by the write guard + let to_release = (this.permits_acquired - 1) as usize; + this.s.release(to_release); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + write_locked = false, + write_locked.op = "override", + ) + }); + + #[cfg(all(tokio_unstable, feature = "tracing"))] + guard.resource_span.in_scope(|| { + tracing::trace!( + target: "runtime::resource::state_update", + current_readers = 1, + current_readers.op = "add", + ) + }); + + Ok(guard) + } + /// Converts this `RwLockWriteGuard` into an `RwLockMappedWriteGuard`. This /// method can be used to store a non-mapped guard in a struct field that /// expects a mapped guard. @@ -197,12 +374,21 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { /// /// [`RwLock`]: struct@crate::sync::RwLock pub fn downgrade(self) -> RwLockReadGuard<'a, T> { - let RwLockWriteGuard { s, data, .. } = self; - let to_release = (self.permits_acquired - 1) as usize; + let this = self.skip_drop(); + let guard = RwLockReadGuard { + s: this.s, + data: this.data, + marker: PhantomData, + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: this.resource_span, + }; + // Release all but one of the permits held by the write guard - s.release(to_release); + let to_release = (this.permits_acquired - 1) as usize; + this.s.release(to_release); + #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", write_locked = false, @@ -211,7 +397,7 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { }); #[cfg(all(tokio_unstable, feature = "tracing"))] - self.resource_span.in_scope(|| { + guard.resource_span.in_scope(|| { tracing::trace!( target: "runtime::resource::state_update", current_readers = 1, @@ -219,18 +405,7 @@ impl<'a, T: ?Sized> RwLockWriteGuard<'a, T> { ) }); - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = self.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(self); - - RwLockReadGuard { - s, - data, - marker: marker::PhantomData, - #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, - } + guard } } diff --git a/third_party/rust/tokio/src/sync/rwlock/write_guard_mapped.rs b/third_party/rust/tokio/src/sync/rwlock/write_guard_mapped.rs index b5c644a9e83d..7705189e70f5 100644 --- a/third_party/rust/tokio/src/sync/rwlock/write_guard_mapped.rs +++ b/third_party/rust/tokio/src/sync/rwlock/write_guard_mapped.rs @@ -1,8 +1,6 @@ use crate::sync::batch_semaphore::Semaphore; -use std::fmt; -use std::marker; -use std::mem; -use std::ops; +use std::marker::PhantomData; +use std::{fmt, mem, ops}; /// RAII structure used to release the exclusive write access of a lock when /// dropped. @@ -13,16 +11,41 @@ use std::ops; /// /// [mapping]: method@crate::sync::RwLockWriteGuard::map /// [`RwLockWriteGuard`]: struct@crate::sync::RwLockWriteGuard +#[clippy::has_significant_drop] pub struct RwLockMappedWriteGuard<'a, T: ?Sized> { + // When changing the fields in this struct, make sure to update the + // `skip_drop` method. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) resource_span: tracing::Span, pub(super) permits_acquired: u32, pub(super) s: &'a Semaphore, pub(super) data: *mut T, - pub(super) marker: marker::PhantomData<&'a mut T>, + pub(super) marker: PhantomData<&'a mut T>, +} + +#[allow(dead_code)] // Unused fields are still used in Drop. +struct Inner<'a, T: ?Sized> { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: tracing::Span, + permits_acquired: u32, + s: &'a Semaphore, + data: *mut T, } impl<'a, T: ?Sized> RwLockMappedWriteGuard<'a, T> { + fn skip_drop(self) -> Inner<'a, T> { + let me = mem::ManuallyDrop::new(self); + // SAFETY: This duplicates the values in every field of the guard, then + // forgets the originals, so in the end no value is duplicated. + Inner { + #[cfg(all(tokio_unstable, feature = "tracing"))] + resource_span: unsafe { std::ptr::read(&me.resource_span) }, + permits_acquired: me.permits_acquired, + s: me.s, + data: me.data, + } + } + /// Makes a new `RwLockMappedWriteGuard` for a component of the locked data. /// /// This operation cannot fail as the `RwLockMappedWriteGuard` passed in already @@ -64,20 +87,15 @@ impl<'a, T: ?Sized> RwLockMappedWriteGuard<'a, T> { F: FnOnce(&mut T) -> &mut U, { let data = f(&mut *this) as *mut U; - let s = this.s; - let permits_acquired = this.permits_acquired; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); RwLockMappedWriteGuard { - permits_acquired, - s, + permits_acquired: this.permits_acquired, + s: this.s, data, - marker: marker::PhantomData, + marker: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, } } @@ -131,22 +149,20 @@ impl<'a, T: ?Sized> RwLockMappedWriteGuard<'a, T> { Some(data) => data as *mut U, None => return Err(this), }; - let s = this.s; - let permits_acquired = this.permits_acquired; - #[cfg(all(tokio_unstable, feature = "tracing"))] - let resource_span = this.resource_span.clone(); - // NB: Forget to avoid drop impl from being called. - mem::forget(this); + let this = this.skip_drop(); Ok(RwLockMappedWriteGuard { - permits_acquired, - s, + permits_acquired: this.permits_acquired, + s: this.s, data, - marker: marker::PhantomData, + marker: PhantomData, #[cfg(all(tokio_unstable, feature = "tracing"))] - resource_span, + resource_span: this.resource_span, }) } + + // Note: No `downgrade`, `downgrade_map` nor `try_downgrade_map` because they would be unsound, as we're already + // potentially been mapped with internal mutability. } impl ops::Deref for RwLockMappedWriteGuard<'_, T> { diff --git a/third_party/rust/tokio/src/sync/semaphore.rs b/third_party/rust/tokio/src/sync/semaphore.rs index 860f46f39987..e679d0e6b040 100644 --- a/third_party/rust/tokio/src/sync/semaphore.rs +++ b/third_party/rust/tokio/src/sync/semaphore.rs @@ -73,7 +73,7 @@ use std::sync::Arc; /// } /// ``` /// -/// [`PollSemaphore`]: https://docs.rs/tokio-util/0.6/tokio_util/sync/struct.PollSemaphore.html +/// [`PollSemaphore`]: https://docs.rs/tokio-util/latest/tokio_util/sync/struct.PollSemaphore.html /// [`Semaphore::acquire_owned`]: crate::sync::Semaphore::acquire_owned #[derive(Debug)] pub struct Semaphore { @@ -89,6 +89,7 @@ pub struct Semaphore { /// /// [`acquire`]: crate::sync::Semaphore::acquire() #[must_use] +#[clippy::has_significant_drop] #[derive(Debug)] pub struct SemaphorePermit<'a> { sem: &'a Semaphore, @@ -101,6 +102,7 @@ pub struct SemaphorePermit<'a> { /// /// [`acquire_owned`]: crate::sync::Semaphore::acquire_owned() #[must_use] +#[clippy::has_significant_drop] #[derive(Debug)] pub struct OwnedSemaphorePermit { sem: Arc, @@ -123,7 +125,14 @@ fn bounds() { } impl Semaphore { + /// The maximum number of permits which a semaphore can hold. It is `usize::MAX >> 3`. + /// + /// Exceeding this limit typically results in a panic. + pub const MAX_PERMITS: usize = super::batch_semaphore::Semaphore::MAX_PERMITS; + /// Creates a new semaphore with the initial number of permits. + /// + /// Panics if `permits` exceeds [`Semaphore::MAX_PERMITS`]. #[track_caller] pub fn new(permits: usize) -> Self { #[cfg(all(tokio_unstable, feature = "tracing"))] @@ -186,7 +195,7 @@ impl Semaphore { /// Adds `n` new permits to the semaphore. /// - /// The maximum number of permits is `usize::MAX >> 3`, and this function will panic if the limit is exceeded. + /// The maximum number of permits is [`Semaphore::MAX_PERMITS`], and this function will panic if the limit is exceeded. pub fn add_permits(&self, n: usize) { self.ll_sem.release(n); } @@ -620,6 +629,25 @@ impl<'a> SemaphorePermit<'a> { pub fn forget(mut self) { self.permits = 0; } + + /// Merge two [`SemaphorePermit`] instances together, consuming `other` + /// without releasing the permits it holds. + /// + /// Permits held by both `self` and `other` are released when `self` drops. + /// + /// # Panics + /// + /// This function panics if permits from different [`Semaphore`] instances + /// are merged. + #[track_caller] + pub fn merge(&mut self, mut other: Self) { + assert!( + std::ptr::eq(self.sem, other.sem), + "merging permits from different semaphore instances" + ); + self.permits += other.permits; + other.permits = 0; + } } impl OwnedSemaphorePermit { @@ -629,9 +657,33 @@ impl OwnedSemaphorePermit { pub fn forget(mut self) { self.permits = 0; } + + /// Merge two [`OwnedSemaphorePermit`] instances together, consuming `other` + /// without releasing the permits it holds. + /// + /// Permits held by both `self` and `other` are released when `self` drops. + /// + /// # Panics + /// + /// This function panics if permits from different [`Semaphore`] instances + /// are merged. + #[track_caller] + pub fn merge(&mut self, mut other: Self) { + assert!( + Arc::ptr_eq(&self.sem, &other.sem), + "merging permits from different semaphore instances" + ); + self.permits += other.permits; + other.permits = 0; + } + + /// Returns the [`Semaphore`] from which this permit was acquired. + pub fn semaphore(&self) -> &Arc { + &self.sem + } } -impl<'a> Drop for SemaphorePermit<'_> { +impl Drop for SemaphorePermit<'_> { fn drop(&mut self) { self.sem.add_permits(self.permits as usize); } diff --git a/third_party/rust/tokio/src/sync/tests/atomic_waker.rs b/third_party/rust/tokio/src/sync/tests/atomic_waker.rs index ec13cbd65858..8ebfb915f4ae 100644 --- a/third_party/rust/tokio/src/sync/tests/atomic_waker.rs +++ b/third_party/rust/tokio/src/sync/tests/atomic_waker.rs @@ -4,7 +4,7 @@ use tokio_test::task; use std::task::Waker; trait AssertSend: Send {} -trait AssertSync: Send {} +trait AssertSync: Sync {} impl AssertSend for AtomicWaker {} impl AssertSync for AtomicWaker {} @@ -12,7 +12,7 @@ impl AssertSync for AtomicWaker {} impl AssertSend for Waker {} impl AssertSync for Waker {} -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; #[test] @@ -37,7 +37,7 @@ fn wake_without_register() { } #[test] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn atomic_waker_panic_safe() { use std::panic; use std::ptr; diff --git a/third_party/rust/tokio/src/sync/tests/loom_notify.rs b/third_party/rust/tokio/src/sync/tests/loom_notify.rs index d484a7581723..a4ded1d35bc3 100644 --- a/third_party/rust/tokio/src/sync/tests/loom_notify.rs +++ b/third_party/rust/tokio/src/sync/tests/loom_notify.rs @@ -4,6 +4,11 @@ use loom::future::block_on; use loom::sync::Arc; use loom::thread; +use tokio_test::{assert_pending, assert_ready}; + +/// `util::wake_list::NUM_WAKERS` +const WAKE_LIST_SIZE: usize = 32; + #[test] fn notify_one() { loom::model(|| { @@ -138,3 +143,189 @@ fn notify_drop() { th2.join().unwrap(); }); } + +/// Polls two `Notified` futures and checks if poll results are consistent +/// with each other. If the first future is notified by a `notify_waiters` +/// call, then the second one must be notified as well. +#[test] +fn notify_waiters_poll_consistency() { + fn notify_waiters_poll_consistency_variant(poll_setting: [bool; 2]) { + let notify = Arc::new(Notify::new()); + let mut notified = [ + tokio_test::task::spawn(notify.notified()), + tokio_test::task::spawn(notify.notified()), + ]; + for i in 0..2 { + if poll_setting[i] { + assert_pending!(notified[i].poll()); + } + } + + let tx = notify.clone(); + let th = thread::spawn(move || { + tx.notify_waiters(); + }); + + let res1 = notified[0].poll(); + let res2 = notified[1].poll(); + + // If res1 is ready, then res2 must also be ready. + assert!(res1.is_pending() || res2.is_ready()); + + th.join().unwrap(); + } + + // We test different scenarios in which pending futures had or had not + // been polled before the call to `notify_waiters`. + loom::model(|| notify_waiters_poll_consistency_variant([false, false])); + loom::model(|| notify_waiters_poll_consistency_variant([true, false])); + loom::model(|| notify_waiters_poll_consistency_variant([false, true])); + loom::model(|| notify_waiters_poll_consistency_variant([true, true])); +} + +/// Polls two `Notified` futures and checks if poll results are consistent +/// with each other. If the first future is notified by a `notify_waiters` +/// call, then the second one must be notified as well. +/// +/// Here we also add other `Notified` futures in between to force the two +/// tested futures to end up in different chunks. +#[test] +fn notify_waiters_poll_consistency_many() { + fn notify_waiters_poll_consistency_many_variant(order: [usize; 2]) { + let notify = Arc::new(Notify::new()); + + let mut futs = (0..WAKE_LIST_SIZE + 1) + .map(|_| tokio_test::task::spawn(notify.notified())) + .collect::>(); + + assert_pending!(futs[order[0]].poll()); + for i in 2..futs.len() { + assert_pending!(futs[i].poll()); + } + assert_pending!(futs[order[1]].poll()); + + let tx = notify.clone(); + let th = thread::spawn(move || { + tx.notify_waiters(); + }); + + let res1 = futs[0].poll(); + let res2 = futs[1].poll(); + + // If res1 is ready, then res2 must also be ready. + assert!(res1.is_pending() || res2.is_ready()); + + th.join().unwrap(); + } + + // We test different scenarios in which futures are polled in different order. + loom::model(|| notify_waiters_poll_consistency_many_variant([0, 1])); + loom::model(|| notify_waiters_poll_consistency_many_variant([1, 0])); +} + +/// Checks if a call to `notify_waiters` is observed as atomic when combined +/// with a concurrent call to `notify_one`. +#[test] +fn notify_waiters_is_atomic() { + fn notify_waiters_is_atomic_variant(tested_fut_index: usize) { + let notify = Arc::new(Notify::new()); + + let mut futs = (0..WAKE_LIST_SIZE + 1) + .map(|_| tokio_test::task::spawn(notify.notified())) + .collect::>(); + + for fut in &mut futs { + assert_pending!(fut.poll()); + } + + let tx = notify.clone(); + let th = thread::spawn(move || { + tx.notify_waiters(); + }); + + block_on(async { + // If awaiting one of the futures completes, then we should be + // able to assume that all pending futures are notified. Therefore + // a notification from a subsequent `notify_one` call should not + // be consumed by an old future. + futs.remove(tested_fut_index).await; + + let mut new_fut = tokio_test::task::spawn(notify.notified()); + assert_pending!(new_fut.poll()); + + notify.notify_one(); + + // `new_fut` must consume the notification from `notify_one`. + assert_ready!(new_fut.poll()); + }); + + th.join().unwrap(); + } + + // We test different scenarios in which the tested future is at the beginning + // or at the end of the waiters queue used by `Notify`. + loom::model(|| notify_waiters_is_atomic_variant(0)); + loom::model(|| notify_waiters_is_atomic_variant(32)); +} + +/// Checks if a single call to `notify_waiters` does not get through two `Notified` +/// futures created and awaited sequentially like this: +/// ```ignore +/// notify.notified().await; +/// notify.notified().await; +/// ``` +#[test] +fn notify_waiters_sequential_notified_await() { + use crate::sync::oneshot; + + loom::model(|| { + let notify = Arc::new(Notify::new()); + + let (tx_fst, rx_fst) = oneshot::channel(); + let (tx_snd, rx_snd) = oneshot::channel(); + + let receiver = thread::spawn({ + let notify = notify.clone(); + move || { + block_on(async { + // Poll the first `Notified` to put it as the first waiter + // in the queue. + let mut first_notified = tokio_test::task::spawn(notify.notified()); + assert_pending!(first_notified.poll()); + + // Create additional waiters to force `notify_waiters` to + // release the lock at least once. + let _task_pile = (0..WAKE_LIST_SIZE + 1) + .map(|_| { + let mut fut = tokio_test::task::spawn(notify.notified()); + assert_pending!(fut.poll()); + fut + }) + .collect::>(); + + // We are ready for the notify_waiters call. + tx_fst.send(()).unwrap(); + + first_notified.await; + + // Poll the second `Notified` future to try to insert + // it to the waiters queue. + let mut second_notified = tokio_test::task::spawn(notify.notified()); + assert_pending!(second_notified.poll()); + + // Wait for the `notify_waiters` to end and check if we + // are woken up. + rx_snd.await.unwrap(); + assert_pending!(second_notified.poll()); + }); + } + }); + + // Wait for the signal and call `notify_waiters`. + block_on(rx_fst).unwrap(); + notify.notify_waiters(); + tx_snd.send(()).unwrap(); + + receiver.join().unwrap(); + }); +} diff --git a/third_party/rust/tokio/src/sync/tests/loom_watch.rs b/third_party/rust/tokio/src/sync/tests/loom_watch.rs index c575b5b66c5e..51589cd80425 100644 --- a/third_party/rust/tokio/src/sync/tests/loom_watch.rs +++ b/third_party/rust/tokio/src/sync/tests/loom_watch.rs @@ -2,6 +2,7 @@ use crate::sync::watch; use loom::future::block_on; use loom::thread; +use std::sync::Arc; #[test] fn smoke() { @@ -34,3 +35,56 @@ fn smoke() { th.join().unwrap(); }) } + +#[test] +fn wait_for_test() { + loom::model(move || { + let (tx, mut rx) = watch::channel(false); + + let tx_arc = Arc::new(tx); + let tx1 = tx_arc.clone(); + let tx2 = tx_arc.clone(); + + let th1 = thread::spawn(move || { + for _ in 0..2 { + tx1.send_modify(|_x| {}); + } + }); + + let th2 = thread::spawn(move || { + tx2.send(true).unwrap(); + }); + + assert_eq!(*block_on(rx.wait_for(|x| *x)).unwrap(), true); + + th1.join().unwrap(); + th2.join().unwrap(); + }); +} + +#[test] +fn wait_for_returns_correct_value() { + loom::model(move || { + let (tx, mut rx) = watch::channel(0); + + let jh = thread::spawn(move || { + tx.send(1).unwrap(); + tx.send(2).unwrap(); + tx.send(3).unwrap(); + }); + + // Stop at the first value we are called at. + let mut stopped_at = usize::MAX; + let returned = *block_on(rx.wait_for(|x| { + stopped_at = *x; + true + })) + .unwrap(); + + // Check that it returned the same value as the one we returned + // `true` for. + assert_eq!(stopped_at, returned); + + jh.join().unwrap(); + }); +} diff --git a/third_party/rust/tokio/src/sync/tests/notify.rs b/third_party/rust/tokio/src/sync/tests/notify.rs index 20153b7a5a81..4b5989597653 100644 --- a/third_party/rust/tokio/src/sync/tests/notify.rs +++ b/third_party/rust/tokio/src/sync/tests/notify.rs @@ -4,7 +4,7 @@ use std::mem::ManuallyDrop; use std::sync::Arc; use std::task::{Context, RawWaker, RawWakerVTable, Waker}; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; #[test] @@ -46,6 +46,45 @@ fn notify_clones_waker_before_lock() { let _ = future.poll(&mut cx); } +#[cfg(panic = "unwind")] +#[test] +fn notify_waiters_handles_panicking_waker() { + use futures::task::ArcWake; + + let notify = Arc::new(Notify::new()); + + struct PanickingWaker(Arc); + + impl ArcWake for PanickingWaker { + fn wake_by_ref(_arc_self: &Arc) { + panic!("waker panicked"); + } + } + + let bad_fut = notify.notified(); + pin!(bad_fut); + + let waker = futures::task::waker(Arc::new(PanickingWaker(notify.clone()))); + let mut cx = Context::from_waker(&waker); + let _ = bad_fut.poll(&mut cx); + + let mut futs = Vec::new(); + for _ in 0..32 { + let mut fut = tokio_test::task::spawn(notify.notified()); + assert!(fut.poll().is_pending()); + futs.push(fut); + } + + assert!(std::panic::catch_unwind(|| { + notify.notify_waiters(); + }) + .is_err()); + + for mut fut in futs { + assert!(fut.poll().is_ready()); + } +} + #[test] fn notify_simple() { let notify = Notify::new(); @@ -63,7 +102,7 @@ fn notify_simple() { } #[test] -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm))] fn watch_test() { let rt = crate::runtime::Builder::new_current_thread() .build() diff --git a/third_party/rust/tokio/src/sync/tests/semaphore_batch.rs b/third_party/rust/tokio/src/sync/tests/semaphore_batch.rs index d529a9e8869c..85089cd2213f 100644 --- a/third_party/rust/tokio/src/sync/tests/semaphore_batch.rs +++ b/third_party/rust/tokio/src/sync/tests/semaphore_batch.rs @@ -1,7 +1,9 @@ use crate::sync::batch_semaphore::Semaphore; use tokio_test::*; -#[cfg(target_arch = "wasm32")] +const MAX_PERMITS: usize = crate::sync::Semaphore::MAX_PERMITS; + +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; #[test] @@ -168,12 +170,16 @@ fn poll_acquire_one_zero_permits() { assert_ready_ok!(acquire.poll()); } +#[test] +fn max_permits_doesnt_panic() { + Semaphore::new(MAX_PERMITS); +} + #[test] #[should_panic] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn validates_max_permits() { - use std::usize; - Semaphore::new((usize::MAX >> 2) + 1); + Semaphore::new(MAX_PERMITS + 1); } #[test] @@ -252,3 +258,32 @@ fn cancel_acquire_releases_permits() { assert_eq!(6, s.available_permits()); assert_ok!(s.try_acquire(6)); } + +#[test] +fn release_permits_at_drop() { + use crate::sync::semaphore::*; + use futures::task::ArcWake; + use std::future::Future; + use std::sync::Arc; + + let sem = Arc::new(Semaphore::new(1)); + + struct ReleaseOnDrop(Option); + + impl ArcWake for ReleaseOnDrop { + fn wake_by_ref(_arc_self: &Arc) {} + } + + let mut fut = Box::pin(async { + let _permit = sem.acquire().await.unwrap(); + }); + + // Second iteration shouldn't deadlock. + for _ in 0..=1 { + let waker = futures::task::waker(Arc::new(ReleaseOnDrop( + sem.clone().try_acquire_owned().ok(), + ))); + let mut cx = std::task::Context::from_waker(&waker); + assert!(fut.as_mut().poll(&mut cx).is_pending()); + } +} diff --git a/third_party/rust/tokio/src/sync/watch.rs b/third_party/rust/tokio/src/sync/watch.rs index 5673e0fca78c..61049b71ef78 100644 --- a/third_party/rust/tokio/src/sync/watch.rs +++ b/third_party/rust/tokio/src/sync/watch.rs @@ -9,10 +9,10 @@ //! # Usage //! //! [`channel`] returns a [`Sender`] / [`Receiver`] pair. These are the producer -//! and sender halves of the channel. The channel is created with an initial +//! and consumer halves of the channel. The channel is created with an initial //! value. The **latest** value stored in the channel is accessed with //! [`Receiver::borrow()`]. Awaiting [`Receiver::changed()`] waits for a new -//! value to sent by the [`Sender`] half. +//! value to be sent by the [`Sender`] half. //! //! # Examples //! @@ -20,15 +20,15 @@ //! use tokio::sync::watch; //! //! # async fn dox() -> Result<(), Box> { -//! let (tx, mut rx) = watch::channel("hello"); +//! let (tx, mut rx) = watch::channel("hello"); //! -//! tokio::spawn(async move { -//! while rx.changed().await.is_ok() { -//! println!("received = {:?}", *rx.borrow()); -//! } -//! }); +//! tokio::spawn(async move { +//! while rx.changed().await.is_ok() { +//! println!("received = {:?}", *rx.borrow()); +//! } +//! }); //! -//! tx.send("world")?; +//! tx.send("world")?; //! # Ok(()) //! # } //! ``` @@ -39,6 +39,9 @@ //! when all [`Receiver`] handles have been dropped. This indicates that there //! is no further interest in the values being produced and work can be stopped. //! +//! The value in the channel will not be dropped until the sender and all receivers +//! have been dropped. +//! //! # Thread safety //! //! Both [`Sender`] and [`Receiver`] are thread safe. They can be moved to other @@ -58,8 +61,10 @@ use crate::sync::notify::Notify; use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::atomic::Ordering::Relaxed; use crate::loom::sync::{Arc, RwLock, RwLockReadGuard}; +use std::fmt; use std::mem; use std::ops; +use std::panic; /// Receives values from the associated [`Sender`](struct@Sender). /// @@ -89,8 +94,11 @@ pub struct Sender { /// Returns a reference to the inner value. /// /// Outstanding borrows hold a read lock on the inner value. This means that -/// long lived borrows could cause the produce half to block. It is recommended -/// to keep the borrow as short lived as possible. +/// long-lived borrows could cause the producer half to block. It is recommended +/// to keep the borrow as short-lived as possible. Additionally, if you are +/// running in an environment that allows `!Send` futures, you must ensure that +/// the returned `Ref` type is never held alive across an `.await` point, +/// otherwise, it can lead to a deadlock. /// /// The priority policy of the lock is dependent on the underlying lock /// implementation, and this type does not guarantee that any particular policy @@ -111,9 +119,57 @@ pub struct Sender { #[derive(Debug)] pub struct Ref<'a, T> { inner: RwLockReadGuard<'a, T>, + has_changed: bool, +} + +impl<'a, T> Ref<'a, T> { + /// Indicates if the borrowed value is considered as _changed_ since the last + /// time it has been marked as seen. + /// + /// Unlike [`Receiver::has_changed()`], this method does not fail if the channel is closed. + /// + /// When borrowed from the [`Sender`] this function will always return `false`. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// #[tokio::main] + /// async fn main() { + /// let (tx, mut rx) = watch::channel("hello"); + /// + /// tx.send("goodbye").unwrap(); + /// // The sender does never consider the value as changed. + /// assert!(!tx.borrow().has_changed()); + /// + /// // Drop the sender immediately, just for testing purposes. + /// drop(tx); + /// + /// // Even if the sender has already been dropped... + /// assert!(rx.has_changed().is_err()); + /// // ...the modified value is still readable and detected as changed. + /// assert_eq!(*rx.borrow(), "goodbye"); + /// assert!(rx.borrow().has_changed()); + /// + /// // Read the changed value and mark it as seen. + /// { + /// let received = rx.borrow_and_update(); + /// assert_eq!(*received, "goodbye"); + /// assert!(received.has_changed()); + /// // Release the read lock when leaving this scope. + /// } + /// + /// // Now the value has already been marked as seen and could + /// // never be modified again (after the sender has been dropped). + /// assert!(!rx.borrow().has_changed()); + /// } + /// ``` + pub fn has_changed(&self) -> bool { + self.has_changed + } } -#[derive(Debug)] struct Shared { /// The most recent value. value: RwLock, @@ -128,33 +184,51 @@ struct Shared { ref_count_rx: AtomicUsize, /// Notifies waiting receivers that the value changed. - notify_rx: Notify, + notify_rx: big_notify::BigNotify, /// Notifies any task listening for `Receiver` dropped events. notify_tx: Notify, } +impl fmt::Debug for Shared { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let state = self.state.load(); + f.debug_struct("Shared") + .field("value", &self.value) + .field("version", &state.version()) + .field("is_closed", &state.is_closed()) + .field("ref_count_rx", &self.ref_count_rx) + .finish() + } +} + pub mod error { //! Watch error types. use std::fmt; /// Error produced when sending a value fails. - #[derive(Debug)] + #[derive(PartialEq, Eq, Clone, Copy)] pub struct SendError(pub T); // ===== impl SendError ===== - impl fmt::Display for SendError { + impl fmt::Debug for SendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SendError").finish_non_exhaustive() + } + } + + impl fmt::Display for SendError { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { write!(fmt, "channel closed") } } - impl std::error::Error for SendError {} + impl std::error::Error for SendError {} /// Error produced when receiving a change notification. - #[derive(Debug)] + #[derive(Debug, Clone)] pub struct RecvError(pub(super) ()); // ===== impl RecvError ===== @@ -168,6 +242,62 @@ pub mod error { impl std::error::Error for RecvError {} } +mod big_notify { + use super::*; + use crate::sync::notify::Notified; + + // To avoid contention on the lock inside the `Notify`, we store multiple + // copies of it. Then, we use either circular access or randomness to spread + // out threads over different `Notify` objects. + // + // Some simple benchmarks show that randomness performs slightly better than + // circular access (probably due to contention on `next`), so we prefer to + // use randomness when Tokio is compiled with a random number generator. + // + // When the random number generator is not available, we fall back to + // circular access. + + pub(super) struct BigNotify { + #[cfg(not(all(not(loom), feature = "sync", any(feature = "rt", feature = "macros"))))] + next: AtomicUsize, + inner: [Notify; 8], + } + + impl BigNotify { + pub(super) fn new() -> Self { + Self { + #[cfg(not(all( + not(loom), + feature = "sync", + any(feature = "rt", feature = "macros") + )))] + next: AtomicUsize::new(0), + inner: Default::default(), + } + } + + pub(super) fn notify_waiters(&self) { + for notify in &self.inner { + notify.notify_waiters(); + } + } + + /// This function implements the case where randomness is not available. + #[cfg(not(all(not(loom), feature = "sync", any(feature = "rt", feature = "macros"))))] + pub(super) fn notified(&self) -> Notified<'_> { + let i = self.next.fetch_add(1, Relaxed) % 8; + self.inner[i].notified() + } + + /// This function implements the case where randomness is available. + #[cfg(all(not(loom), feature = "sync", any(feature = "rt", feature = "macros")))] + pub(super) fn notified(&self) -> Notified<'_> { + let i = crate::runtime::context::thread_rng_n(8) as usize; + self.inner[i].notified() + } + } +} + use self::state::{AtomicState, Version}; mod state { use crate::loom::sync::atomic::AtomicUsize; @@ -267,7 +397,7 @@ pub fn channel(init: T) -> (Sender, Receiver) { value: RwLock::new(init), state: AtomicState::new(), ref_count_rx: AtomicUsize::new(1), - notify_rx: Notify::new(), + notify_rx: big_notify::BigNotify::new(), notify_tx: Notify::new(), }); @@ -298,9 +428,12 @@ impl Receiver { /// [`changed`] may return immediately even if you have already seen the /// value with a call to `borrow`. /// - /// Outstanding borrows hold a read lock. This means that long lived borrows - /// could cause the send half to block. It is recommended to keep the borrow - /// as short lived as possible. + /// Outstanding borrows hold a read lock on the inner value. This means that + /// long-lived borrows could cause the producer half to block. It is recommended + /// to keep the borrow as short-lived as possible. Additionally, if you are + /// running in an environment that allows `!Send` futures, you must ensure that + /// the returned `Ref` type is never held alive across an `.await` point, + /// otherwise, it can lead to a deadlock. /// /// The priority policy of the lock is dependent on the underlying lock /// implementation, and this type does not guarantee that any particular policy @@ -331,19 +464,28 @@ impl Receiver { /// ``` pub fn borrow(&self) -> Ref<'_, T> { let inner = self.shared.value.read().unwrap(); - Ref { inner } + + // After obtaining a read-lock no concurrent writes could occur + // and the loaded version matches that of the borrowed reference. + let new_version = self.shared.state.load().version(); + let has_changed = self.version != new_version; + + Ref { inner, has_changed } } - /// Returns a reference to the most recently sent value and mark that value + /// Returns a reference to the most recently sent value and marks that value /// as seen. /// - /// This method marks the value as seen, so [`changed`] will not return - /// immediately if the newest value is one previously returned by - /// `borrow_and_update`. + /// This method marks the current value as seen. Subsequent calls to [`changed`] + /// will not return immediately until the [`Sender`] has modified the shared + /// value again. /// - /// Outstanding borrows hold a read lock. This means that long lived borrows - /// could cause the send half to block. It is recommended to keep the borrow - /// as short lived as possible. + /// Outstanding borrows hold a read lock on the inner value. This means that + /// long-lived borrows could cause the producer half to block. It is recommended + /// to keep the borrow as short-lived as possible. Additionally, if you are + /// running in an environment that allows `!Send` futures, you must ensure that + /// the returned `Ref` type is never held alive across an `.await` point, + /// otherwise, it can lead to a deadlock. /// /// The priority policy of the lock is dependent on the underlying lock /// implementation, and this type does not guarantee that any particular policy @@ -365,8 +507,16 @@ impl Receiver { /// [`changed`]: Receiver::changed pub fn borrow_and_update(&mut self) -> Ref<'_, T> { let inner = self.shared.value.read().unwrap(); - self.version = self.shared.state.load().version(); - Ref { inner } + + // After obtaining a read-lock no concurrent writes could occur + // and the loaded version matches that of the borrowed reference. + let new_version = self.shared.state.load().version(); + let has_changed = self.version != new_version; + + // Mark the shared value as seen by updating the version + self.version = new_version; + + Ref { inner, has_changed } } /// Checks if this channel contains a message that this receiver has not yet @@ -451,21 +601,112 @@ impl Receiver { /// } /// ``` pub async fn changed(&mut self) -> Result<(), error::RecvError> { - loop { - // In order to avoid a race condition, we first request a notification, - // **then** check the current value's version. If a new version exists, - // the notification request is dropped. - let notified = self.shared.notify_rx.notified(); + changed_impl(&self.shared, &mut self.version).await + } - if let Some(ret) = maybe_changed(&self.shared, &mut self.version) { - return ret; + /// Waits for a value that satisifes the provided condition. + /// + /// This method will call the provided closure whenever something is sent on + /// the channel. Once the closure returns `true`, this method will return a + /// reference to the value that was passed to the closure. + /// + /// Before `wait_for` starts waiting for changes, it will call the closure + /// on the current value. If the closure returns `true` when given the + /// current value, then `wait_for` will immediately return a reference to + /// the current value. This is the case even if the current value is already + /// considered seen. + /// + /// The watch channel only keeps track of the most recent value, so if + /// several messages are sent faster than `wait_for` is able to call the + /// closure, then it may skip some updates. Whenever the closure is called, + /// it will be called with the most recent value. + /// + /// When this function returns, the value that was passed to the closure + /// when it returned `true` will be considered seen. + /// + /// If the channel is closed, then `wait_for` will return a `RecvError`. + /// Once this happens, no more messages can ever be sent on the channel. + /// When an error is returned, it is guaranteed that the closure has been + /// called on the last value, and that it returned `false` for that value. + /// (If the closure returned `true`, then the last value would have been + /// returned instead of the error.) + /// + /// Like the `borrow` method, the returned borrow holds a read lock on the + /// inner value. This means that long-lived borrows could cause the producer + /// half to block. It is recommended to keep the borrow as short-lived as + /// possible. See the documentation of `borrow` for more information on + /// this. + /// + /// [`Receiver::changed()`]: crate::sync::watch::Receiver::changed + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// #[tokio::main] + /// + /// async fn main() { + /// let (tx, _rx) = watch::channel("hello"); + /// + /// tx.send("goodbye").unwrap(); + /// + /// // here we subscribe to a second receiver + /// // now in case of using `changed` we would have + /// // to first check the current value and then wait + /// // for changes or else `changed` would hang. + /// let mut rx2 = tx.subscribe(); + /// + /// // in place of changed we have use `wait_for` + /// // which would automatically check the current value + /// // and wait for changes until the closure returns true. + /// assert!(rx2.wait_for(|val| *val == "goodbye").await.is_ok()); + /// assert_eq!(*rx2.borrow(), "goodbye"); + /// } + /// ``` + pub async fn wait_for( + &mut self, + mut f: impl FnMut(&T) -> bool, + ) -> Result, error::RecvError> { + let mut closed = false; + loop { + { + let inner = self.shared.value.read().unwrap(); + + let new_version = self.shared.state.load().version(); + let has_changed = self.version != new_version; + self.version = new_version; + + if (!closed || has_changed) && f(&inner) { + return Ok(Ref { inner, has_changed }); + } } - notified.await; - // loop around again in case the wake-up was spurious + if closed { + return Err(error::RecvError(())); + } + + // Wait for the value to change. + closed = changed_impl(&self.shared, &mut self.version).await.is_err(); } } + /// Returns `true` if receivers belong to the same channel. + /// + /// # Examples + /// + /// ``` + /// let (tx, rx) = tokio::sync::watch::channel(true); + /// let rx2 = rx.clone(); + /// assert!(rx.same_channel(&rx2)); + /// + /// let (tx3, rx3) = tokio::sync::watch::channel(true); + /// assert!(!rx3.same_channel(&rx2)); + /// ``` + pub fn same_channel(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.shared, &other.shared) + } + cfg_process_driver! { pub(crate) fn try_has_changed(&mut self) -> Option> { maybe_changed(&self.shared, &mut self.version) @@ -495,6 +736,27 @@ fn maybe_changed( None } +async fn changed_impl( + shared: &Shared, + version: &mut Version, +) -> Result<(), error::RecvError> { + crate::trace::async_trace_leaf().await; + + loop { + // In order to avoid a race condition, we first request a notification, + // **then** check the current value's version. If a new version exists, + // the notification request is dropped. + let notified = shared.notify_rx.notified(); + + if let Some(ret) = maybe_changed(shared, version) { + return ret; + } + + notified.await; + // loop around again in case the wake-up was spurious + } +} + impl Clone for Receiver { fn clone(&self) -> Self { let version = self.version; @@ -518,8 +780,22 @@ impl Drop for Receiver { impl Sender { /// Sends a new value via the channel, notifying all receivers. /// - /// This method fails if the channel has been closed, which happens when - /// every receiver has been dropped. + /// This method fails if the channel is closed, which is the case when + /// every receiver has been dropped. It is possible to reopen the channel + /// using the [`subscribe`] method. However, when `send` fails, the value + /// isn't made available for future receivers (but returned with the + /// [`SendError`]). + /// + /// To always make a new value available for future receivers, even if no + /// receiver currently exists, one of the other send methods + /// ([`send_if_modified`], [`send_modify`], or [`send_replace`]) can be + /// used instead. + /// + /// [`subscribe`]: Sender::subscribe + /// [`SendError`]: error::SendError + /// [`send_if_modified`]: Sender::send_if_modified + /// [`send_modify`]: Sender::send_modify + /// [`send_replace`]: Sender::send_replace pub fn send(&self, value: T) -> Result<(), error::SendError> { // This is pretty much only useful as a hint anyway, so synchronization isn't critical. if 0 == self.receiver_count() { @@ -530,6 +806,145 @@ impl Sender { Ok(()) } + /// Modifies the watched value **unconditionally** in-place, + /// notifying all receivers. + /// + /// This can be useful for modifying the watched value, without + /// having to allocate a new instance. Additionally, this + /// method permits sending values even when there are no receivers. + /// + /// Prefer to use the more versatile function [`Self::send_if_modified()`] + /// if the value is only modified conditionally during the mutable borrow + /// to prevent unneeded change notifications for unmodified values. + /// + /// # Panics + /// + /// This function panics when the invocation of the `modify` closure panics. + /// No receivers are notified when panicking. All changes of the watched + /// value applied by the closure before panicking will be visible in + /// subsequent calls to `borrow`. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// struct State { + /// counter: usize, + /// } + /// let (state_tx, state_rx) = watch::channel(State { counter: 0 }); + /// state_tx.send_modify(|state| state.counter += 1); + /// assert_eq!(state_rx.borrow().counter, 1); + /// ``` + pub fn send_modify(&self, modify: F) + where + F: FnOnce(&mut T), + { + self.send_if_modified(|value| { + modify(value); + true + }); + } + + /// Modifies the watched value **conditionally** in-place, + /// notifying all receivers only if modified. + /// + /// This can be useful for modifying the watched value, without + /// having to allocate a new instance. Additionally, this + /// method permits sending values even when there are no receivers. + /// + /// The `modify` closure must return `true` if the value has actually + /// been modified during the mutable borrow. It should only return `false` + /// if the value is guaranteed to be unmodified despite the mutable + /// borrow. + /// + /// Receivers are only notified if the closure returned `true`. If the + /// closure has modified the value but returned `false` this results + /// in a *silent modification*, i.e. the modified value will be visible + /// in subsequent calls to `borrow`, but receivers will not receive + /// a change notification. + /// + /// Returns the result of the closure, i.e. `true` if the value has + /// been modified and `false` otherwise. + /// + /// # Panics + /// + /// This function panics when the invocation of the `modify` closure panics. + /// No receivers are notified when panicking. All changes of the watched + /// value applied by the closure before panicking will be visible in + /// subsequent calls to `borrow`. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// struct State { + /// counter: usize, + /// } + /// let (state_tx, mut state_rx) = watch::channel(State { counter: 1 }); + /// let inc_counter_if_odd = |state: &mut State| { + /// if state.counter % 2 == 1 { + /// state.counter += 1; + /// return true; + /// } + /// false + /// }; + /// + /// assert_eq!(state_rx.borrow().counter, 1); + /// + /// assert!(!state_rx.has_changed().unwrap()); + /// assert!(state_tx.send_if_modified(inc_counter_if_odd)); + /// assert!(state_rx.has_changed().unwrap()); + /// assert_eq!(state_rx.borrow_and_update().counter, 2); + /// + /// assert!(!state_rx.has_changed().unwrap()); + /// assert!(!state_tx.send_if_modified(inc_counter_if_odd)); + /// assert!(!state_rx.has_changed().unwrap()); + /// assert_eq!(state_rx.borrow_and_update().counter, 2); + /// ``` + pub fn send_if_modified(&self, modify: F) -> bool + where + F: FnOnce(&mut T) -> bool, + { + { + // Acquire the write lock and update the value. + let mut lock = self.shared.value.write().unwrap(); + + // Update the value and catch possible panic inside func. + let result = panic::catch_unwind(panic::AssertUnwindSafe(|| modify(&mut lock))); + match result { + Ok(modified) => { + if !modified { + // Abort, i.e. don't notify receivers if unmodified + return false; + } + // Continue if modified + } + Err(panicked) => { + // Drop the lock to avoid poisoning it. + drop(lock); + // Forward the panic to the caller. + panic::resume_unwind(panicked); + // Unreachable + } + }; + + self.shared.state.increment_version(); + + // Release the write lock. + // + // Incrementing the version counter while holding the lock ensures + // that receivers are able to figure out the version number of the + // value they are currently looking at. + drop(lock); + } + + self.shared.notify_rx.notify_waiters(); + + true + } + /// Sends a new value via the channel, notifying all receivers and returning /// the previous value in the channel. /// @@ -546,35 +961,21 @@ impl Sender { /// assert_eq!(tx.send_replace(2), 1); /// assert_eq!(tx.send_replace(3), 2); /// ``` - pub fn send_replace(&self, value: T) -> T { - let old = { - // Acquire the write lock and update the value. - let mut lock = self.shared.value.write().unwrap(); - let old = mem::replace(&mut *lock, value); + pub fn send_replace(&self, mut value: T) -> T { + // swap old watched value with the new one + self.send_modify(|old| mem::swap(old, &mut value)); - self.shared.state.increment_version(); - - // Release the write lock. - // - // Incrementing the version counter while holding the lock ensures - // that receivers are able to figure out the version number of the - // value they are currently looking at. - drop(lock); - - old - }; - - // Notify all watchers - self.shared.notify_rx.notify_waiters(); - - old + value } /// Returns a reference to the most recently sent value /// - /// Outstanding borrows hold a read lock. This means that long lived borrows - /// could cause the send half to block. It is recommended to keep the borrow - /// as short lived as possible. + /// Outstanding borrows hold a read lock on the inner value. This means that + /// long-lived borrows could cause the producer half to block. It is recommended + /// to keep the borrow as short-lived as possible. Additionally, if you are + /// running in an environment that allows `!Send` futures, you must ensure that + /// the returned `Ref` type is never held alive across an `.await` point, + /// otherwise, it can lead to a deadlock. /// /// # Examples /// @@ -586,7 +987,11 @@ impl Sender { /// ``` pub fn borrow(&self) -> Ref<'_, T> { let inner = self.shared.value.read().unwrap(); - Ref { inner } + + // The sender/producer always sees the current version + let has_changed = false; + + Ref { inner, has_changed } } /// Checks if the channel has been closed. This happens when all receivers @@ -635,6 +1040,8 @@ impl Sender { /// } /// ``` pub async fn closed(&self) { + crate::trace::async_trace_leaf().await; + while self.receiver_count() > 0 { let notified = self.shared.notify_tx.notified(); diff --git a/third_party/rust/tokio/src/task/blocking.rs b/third_party/rust/tokio/src/task/blocking.rs index 5fe358f3e50f..9bd15ebd5d8a 100644 --- a/third_party/rust/tokio/src/task/blocking.rs +++ b/third_party/rust/tokio/src/task/blocking.rs @@ -70,11 +70,12 @@ cfg_rt_multi_thread! { /// This function panics if called from a [`current_thread`] runtime. /// /// [`current_thread`]: fn@crate::runtime::Builder::new_current_thread + #[track_caller] pub fn block_in_place(f: F) -> R where F: FnOnce() -> R, { - crate::runtime::thread_pool::block_in_place(f) + crate::runtime::scheduler::multi_thread::block_in_place(f) } } @@ -102,14 +103,24 @@ cfg_rt! { /// their own. If you want to spawn an ordinary thread, you should use /// [`thread::spawn`] instead. /// - /// Closures spawned using `spawn_blocking` cannot be cancelled. When you shut - /// down the executor, it will wait indefinitely for all blocking operations to + /// Closures spawned using `spawn_blocking` cannot be cancelled abruptly; there + /// is no standard low level API to cause a thread to stop running. However, + /// a useful pattern is to pass some form of "cancellation token" into + /// the thread. This could be an [`AtomicBool`] that the task checks periodically. + /// Another approach is to have the thread primarily read or write from a channel, + /// and to exit when the channel closes; assuming the other side of the channel is dropped + /// when cancellation occurs, this will cause the blocking task thread to exit + /// soon after as well. + /// + /// When you shut down the executor, it will wait indefinitely for all blocking operations to /// finish. You can use [`shutdown_timeout`] to stop waiting for them after a /// certain timeout. Be aware that this will still not cancel the tasks — they - /// are simply allowed to keep running after the method returns. + /// are simply allowed to keep running after the method returns. It is possible + /// for a blocking task to be cancelled if it has not yet started running, but this + /// is not guaranteed. /// /// Note that if you are using the single threaded runtime, this function will - /// still spawn additional threads for blocking operations. The basic + /// still spawn additional threads for blocking operations. The current-thread /// scheduler's single thread is only used for asynchronous code. /// /// # Related APIs and patterns for bridging asynchronous and blocking code @@ -135,11 +146,12 @@ cfg_rt! { /// [blocking]: ../index.html#cpu-bound-tasks-and-blocking-code /// [rayon]: https://docs.rs/rayon /// [`mpsc channel`]: crate::sync::mpsc - /// [`SyncIoBridge`]: https://docs.rs/tokio-util/0.6/tokio_util/io/struct.SyncIoBridge.html + /// [`SyncIoBridge`]: https://docs.rs/tokio-util/latest/tokio_util/io/struct.SyncIoBridge.html /// [hyper]: https://docs.rs/hyper /// [`thread::spawn`]: fn@std::thread::spawn /// [`shutdown_timeout`]: fn@crate::runtime::Runtime::shutdown_timeout /// [bridgesync]: https://tokio.rs/tokio/topics/bridging + /// [`AtomicBool`]: struct@std::sync::atomic::AtomicBool /// /// # Examples /// diff --git a/third_party/rust/tokio/src/task/builder.rs b/third_party/rust/tokio/src/task/builder.rs index 2086302fb92a..306cc39d1eb2 100644 --- a/third_party/rust/tokio/src/task/builder.rs +++ b/third_party/rust/tokio/src/task/builder.rs @@ -1,6 +1,9 @@ #![allow(unreachable_pub)] -use crate::{runtime::context, task::JoinHandle}; -use std::future::Future; +use crate::{ + runtime::Handle, + task::{JoinHandle, LocalSet}, +}; +use std::{future::Future, io}; /// Factory which is used to configure the properties of a new task. /// @@ -45,7 +48,7 @@ use std::future::Future; /// .spawn(async move { /// // Process each socket concurrently. /// process(socket).await -/// }); +/// })?; /// } /// } /// ``` @@ -71,45 +74,128 @@ impl<'a> Builder<'a> { Self { name: Some(name) } } - /// Spawns a task on the executor. + /// Spawns a task with this builder's settings on the current runtime. /// - /// See [`task::spawn`](crate::task::spawn) for + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. + /// + /// See [`task::spawn`](crate::task::spawn()) for /// more details. #[track_caller] - pub fn spawn(self, future: Fut) -> JoinHandle + pub fn spawn(self, future: Fut) -> io::Result> where Fut: Future + Send + 'static, Fut::Output: Send + 'static, { - super::spawn::spawn_inner(future, self.name) + Ok(super::spawn::spawn_inner(future, self.name)) } - /// Spawns a task on the current thread. + /// Spawn a task with this builder's settings on the provided [runtime + /// handle]. /// - /// See [`task::spawn_local`](crate::task::spawn_local) - /// for more details. + /// See [`Handle::spawn`] for more details. + /// + /// [runtime handle]: crate::runtime::Handle + /// [`Handle::spawn`]: crate::runtime::Handle::spawn #[track_caller] - pub fn spawn_local(self, future: Fut) -> JoinHandle + pub fn spawn_on(self, future: Fut, handle: &Handle) -> io::Result> + where + Fut: Future + Send + 'static, + Fut::Output: Send + 'static, + { + Ok(handle.spawn_named(future, self.name)) + } + + /// Spawns `!Send` a task on the current [`LocalSet`] with this builder's + /// settings. + /// + /// The spawned future will be run on the same thread that called `spawn_local`. + /// This may only be called from the context of a [local task set][`LocalSet`]. + /// + /// # Panics + /// + /// This function panics if called outside of a [local task set][`LocalSet`]. + /// + /// See [`task::spawn_local`] for more details. + /// + /// [`task::spawn_local`]: crate::task::spawn_local + /// [`LocalSet`]: crate::task::LocalSet + #[track_caller] + pub fn spawn_local(self, future: Fut) -> io::Result> where Fut: Future + 'static, Fut::Output: 'static, { - super::local::spawn_local_inner(future, self.name) + Ok(super::local::spawn_local_inner(future, self.name)) + } + + /// Spawns `!Send` a task on the provided [`LocalSet`] with this builder's + /// settings. + /// + /// See [`LocalSet::spawn_local`] for more details. + /// + /// [`LocalSet::spawn_local`]: crate::task::LocalSet::spawn_local + /// [`LocalSet`]: crate::task::LocalSet + #[track_caller] + pub fn spawn_local_on( + self, + future: Fut, + local_set: &LocalSet, + ) -> io::Result> + where + Fut: Future + 'static, + Fut::Output: 'static, + { + Ok(local_set.spawn_named(future, self.name)) } /// Spawns blocking code on the blocking threadpool. /// + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. + /// /// See [`task::spawn_blocking`](crate::task::spawn_blocking) /// for more details. #[track_caller] - pub fn spawn_blocking(self, function: Function) -> JoinHandle + pub fn spawn_blocking( + self, + function: Function, + ) -> io::Result> + where + Function: FnOnce() -> Output + Send + 'static, + Output: Send + 'static, + { + let handle = Handle::current(); + self.spawn_blocking_on(function, &handle) + } + + /// Spawns blocking code on the provided [runtime handle]'s blocking threadpool. + /// + /// See [`Handle::spawn_blocking`] for more details. + /// + /// [runtime handle]: crate::runtime::Handle + /// [`Handle::spawn_blocking`]: crate::runtime::Handle::spawn_blocking + #[track_caller] + pub fn spawn_blocking_on( + self, + function: Function, + handle: &Handle, + ) -> io::Result> where Function: FnOnce() -> Output + Send + 'static, Output: Send + 'static, { use crate::runtime::Mandatory; - let (join_handle, _was_spawned) = - context::current().spawn_blocking_inner(function, Mandatory::NonMandatory, self.name); - join_handle + let (join_handle, spawn_result) = handle.inner.blocking_spawner().spawn_blocking_inner( + function, + Mandatory::NonMandatory, + self.name, + handle, + ); + + spawn_result?; + Ok(join_handle) } } diff --git a/third_party/rust/tokio/src/task/consume_budget.rs b/third_party/rust/tokio/src/task/consume_budget.rs new file mode 100644 index 000000000000..e7432ffe7d1f --- /dev/null +++ b/third_party/rust/tokio/src/task/consume_budget.rs @@ -0,0 +1,46 @@ +use std::task::Poll; + +/// Consumes a unit of budget and returns the execution back to the Tokio +/// runtime *if* the task's coop budget was exhausted. +/// +/// The task will only yield if its entire coop budget has been exhausted. +/// This function can be used in order to insert optional yield points into long +/// computations that do not use Tokio resources like sockets or semaphores, +/// without redundantly yielding to the runtime each time. +/// +/// **Note**: This is an [unstable API][unstable]. The public API of this type +/// may break in 1.x releases. See [the documentation on unstable +/// features][unstable] for details. +/// +/// # Examples +/// +/// Make sure that a function which returns a sum of (potentially lots of) +/// iterated values is cooperative. +/// +/// ``` +/// async fn sum_iterator(input: &mut impl std::iter::Iterator) -> i64 { +/// let mut sum: i64 = 0; +/// while let Some(i) = input.next() { +/// sum += i; +/// tokio::task::consume_budget().await +/// } +/// sum +/// } +/// ``` +/// [unstable]: crate#unstable-features +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "rt"))))] +pub async fn consume_budget() { + let mut status = Poll::Pending; + + crate::future::poll_fn(move |cx| { + ready!(crate::trace::trace_leaf(cx)); + if status.is_ready() { + return status; + } + status = crate::runtime::coop::poll_proceed(cx).map(|restore| { + restore.made_progress(); + }); + status + }) + .await +} diff --git a/third_party/rust/tokio/src/task/join_set.rs b/third_party/rust/tokio/src/task/join_set.rs index 8e8f74f66d17..4eb15a24d5f9 100644 --- a/third_party/rust/tokio/src/task/join_set.rs +++ b/third_party/rust/tokio/src/task/join_set.rs @@ -1,10 +1,18 @@ +//! A collection of tasks spawned on a Tokio runtime. +//! +//! This module provides the [`JoinSet`] type, a collection which stores a set +//! of spawned tasks and allows asynchronously awaiting the output of those +//! tasks as they complete. See the documentation for the [`JoinSet`] type for +//! details. use std::fmt; use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; use crate::runtime::Handle; -use crate::task::{JoinError, JoinHandle, LocalSet}; +#[cfg(tokio_unstable)] +use crate::task::Id; +use crate::task::{AbortHandle, JoinError, JoinHandle, LocalSet}; use crate::util::IdleNotifiedSet; /// A collection of tasks spawned on a Tokio runtime. @@ -17,10 +25,6 @@ use crate::util::IdleNotifiedSet; /// /// When the `JoinSet` is dropped, all tasks in the `JoinSet` are immediately aborted. /// -/// **Note**: This is an [unstable API][unstable]. The public API of this type -/// may break in 1.x releases. See [the documentation on unstable -/// features][unstable] for details. -/// /// # Examples /// /// Spawn multiple tasks and wait for them. @@ -37,8 +41,9 @@ use crate::util::IdleNotifiedSet; /// } /// /// let mut seen = [false; 10]; -/// while let Some(res) = set.join_one().await.unwrap() { -/// seen[res] = true; +/// while let Some(res) = set.join_next().await { +/// let idx = res.unwrap(); +/// seen[idx] = true; /// } /// /// for i in 0..10 { @@ -46,12 +51,23 @@ use crate::util::IdleNotifiedSet; /// } /// } /// ``` -/// -/// [unstable]: crate#unstable-features +#[cfg_attr(docsrs, doc(cfg(feature = "rt")))] pub struct JoinSet { inner: IdleNotifiedSet>, } +/// A variant of [`task::Builder`] that spawns tasks on a [`JoinSet`] rather +/// than on the current default runtime. +/// +/// [`task::Builder`]: crate::task::Builder +#[cfg(all(tokio_unstable, feature = "tracing"))] +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] +#[must_use = "builders do nothing unless used to spawn a task"] +pub struct Builder<'a, T> { + joinset: &'a mut JoinSet, + builder: super::Builder<'a>, +} + impl JoinSet { /// Create a new `JoinSet`. pub fn new() -> Self { @@ -72,61 +88,187 @@ impl JoinSet { } impl JoinSet { - /// Spawn the provided task on the `JoinSet`. + /// Returns a [`Builder`] that can be used to configure a task prior to + /// spawning it on this `JoinSet`. + /// + /// # Examples + /// + /// ``` + /// use tokio::task::JoinSet; + /// + /// #[tokio::main] + /// async fn main() -> std::io::Result<()> { + /// let mut set = JoinSet::new(); + /// + /// // Use the builder to configure a task's name before spawning it. + /// set.build_task() + /// .name("my_task") + /// .spawn(async { /* ... */ })?; + /// + /// Ok(()) + /// } + /// ``` + #[cfg(all(tokio_unstable, feature = "tracing"))] + #[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] + pub fn build_task(&mut self) -> Builder<'_, T> { + Builder { + builder: super::Builder::new(), + joinset: self, + } + } + + /// Spawn the provided task on the `JoinSet`, returning an [`AbortHandle`] + /// that can be used to remotely cancel the task. + /// + /// The provided future will start running in the background immediately + /// when this method is called, even if you don't await anything on this + /// `JoinSet`. /// /// # Panics /// /// This method panics if called outside of a Tokio runtime. - pub fn spawn(&mut self, task: F) + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn(&mut self, task: F) -> AbortHandle where F: Future, F: Send + 'static, T: Send, { - self.insert(crate::spawn(task)); + self.insert(crate::spawn(task)) } - /// Spawn the provided task on the provided runtime and store it in this `JoinSet`. - pub fn spawn_on(&mut self, task: F, handle: &Handle) + /// Spawn the provided task on the provided runtime and store it in this + /// `JoinSet` returning an [`AbortHandle`] that can be used to remotely + /// cancel the task. + /// + /// The provided future will start running in the background immediately + /// when this method is called, even if you don't await anything on this + /// `JoinSet`. + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_on(&mut self, task: F, handle: &Handle) -> AbortHandle where F: Future, F: Send + 'static, T: Send, { - self.insert(handle.spawn(task)); + self.insert(handle.spawn(task)) } - /// Spawn the provided task on the current [`LocalSet`] and store it in this `JoinSet`. + /// Spawn the provided task on the current [`LocalSet`] and store it in this + /// `JoinSet`, returning an [`AbortHandle`] that can be used to remotely + /// cancel the task. + /// + /// The provided future will start running in the background immediately + /// when this method is called, even if you don't await anything on this + /// `JoinSet`. /// /// # Panics /// /// This method panics if it is called outside of a `LocalSet`. /// /// [`LocalSet`]: crate::task::LocalSet - pub fn spawn_local(&mut self, task: F) + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_local(&mut self, task: F) -> AbortHandle where F: Future, F: 'static, { - self.insert(crate::task::spawn_local(task)); + self.insert(crate::task::spawn_local(task)) } - /// Spawn the provided task on the provided [`LocalSet`] and store it in this `JoinSet`. + /// Spawn the provided task on the provided [`LocalSet`] and store it in + /// this `JoinSet`, returning an [`AbortHandle`] that can be used to + /// remotely cancel the task. + /// + /// Unlike the [`spawn_local`] method, this method may be used to spawn local + /// tasks on a `LocalSet` that is _not_ currently running. The provided + /// future will start running whenever the `LocalSet` is next started. /// /// [`LocalSet`]: crate::task::LocalSet - pub fn spawn_local_on(&mut self, task: F, local_set: &LocalSet) + /// [`AbortHandle`]: crate::task::AbortHandle + /// [`spawn_local`]: Self::spawn_local + #[track_caller] + pub fn spawn_local_on(&mut self, task: F, local_set: &LocalSet) -> AbortHandle where F: Future, F: 'static, { - self.insert(local_set.spawn_local(task)); + self.insert(local_set.spawn_local(task)) } - fn insert(&mut self, jh: JoinHandle) { + /// Spawn the blocking code on the blocking threadpool and store + /// it in this `JoinSet`, returning an [`AbortHandle`] that can be + /// used to remotely cancel the task. + /// + /// # Examples + /// + /// Spawn multiple blocking tasks and wait for them. + /// + /// ``` + /// use tokio::task::JoinSet; + /// + /// #[tokio::main] + /// async fn main() { + /// let mut set = JoinSet::new(); + /// + /// for i in 0..10 { + /// set.spawn_blocking(move || { i }); + /// } + /// + /// let mut seen = [false; 10]; + /// while let Some(res) = set.join_next().await { + /// let idx = res.unwrap(); + /// seen[idx] = true; + /// } + /// + /// for i in 0..10 { + /// assert!(seen[i]); + /// } + /// } + /// ``` + /// + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_blocking(&mut self, f: F) -> AbortHandle + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send, + { + self.insert(crate::runtime::spawn_blocking(f)) + } + + /// Spawn the blocking code on the blocking threadpool of the + /// provided runtime and store it in this `JoinSet`, returning an + /// [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_blocking_on(&mut self, f: F, handle: &Handle) -> AbortHandle + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send, + { + self.insert(handle.spawn_blocking(f)) + } + + fn insert(&mut self, jh: JoinHandle) -> AbortHandle { + let abort = jh.abort_handle(); let mut entry = self.inner.insert_idle(jh); // Set the waker that is notified when the task completes. entry.with_value_and_context(|jh, ctx| jh.set_join_waker(ctx.waker())); + abort } /// Waits until one of the tasks in the set completes and returns its output. @@ -135,32 +277,54 @@ impl JoinSet { /// /// # Cancel Safety /// - /// This method is cancel safe. If `join_one` is used as the event in a `tokio::select!` + /// This method is cancel safe. If `join_next` is used as the event in a `tokio::select!` /// statement and some other branch completes first, it is guaranteed that no tasks were /// removed from this `JoinSet`. - pub async fn join_one(&mut self) -> Result, JoinError> { - crate::future::poll_fn(|cx| self.poll_join_one(cx)).await + pub async fn join_next(&mut self) -> Option> { + crate::future::poll_fn(|cx| self.poll_join_next(cx)).await + } + + /// Waits until one of the tasks in the set completes and returns its + /// output, along with the [task ID] of the completed task. + /// + /// Returns `None` if the set is empty. + /// + /// When this method returns an error, then the id of the task that failed can be accessed + /// using the [`JoinError::id`] method. + /// + /// # Cancel Safety + /// + /// This method is cancel safe. If `join_next_with_id` is used as the event in a `tokio::select!` + /// statement and some other branch completes first, it is guaranteed that no tasks were + /// removed from this `JoinSet`. + /// + /// [task ID]: crate::task::Id + /// [`JoinError::id`]: fn@crate::task::JoinError::id + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub async fn join_next_with_id(&mut self) -> Option> { + crate::future::poll_fn(|cx| self.poll_join_next_with_id(cx)).await } /// Aborts all tasks and waits for them to finish shutting down. /// - /// Calling this method is equivalent to calling [`abort_all`] and then calling [`join_one`] in - /// a loop until it returns `Ok(None)`. + /// Calling this method is equivalent to calling [`abort_all`] and then calling [`join_next`] in + /// a loop until it returns `None`. /// /// This method ignores any panics in the tasks shutting down. When this call returns, the /// `JoinSet` will be empty. /// /// [`abort_all`]: fn@Self::abort_all - /// [`join_one`]: fn@Self::join_one + /// [`join_next`]: fn@Self::join_next pub async fn shutdown(&mut self) { self.abort_all(); - while self.join_one().await.transpose().is_some() {} + while self.join_next().await.is_some() {} } /// Aborts all tasks on this `JoinSet`. /// /// This does not remove the tasks from the `JoinSet`. To wait for the tasks to complete - /// cancellation, you should call `join_one` in a loop until the `JoinSet` is empty. + /// cancellation, you should call `join_next` in a loop until the `JoinSet` is empty. pub fn abort_all(&mut self) { self.inner.for_each(|jh| jh.abort()); } @@ -175,12 +339,11 @@ impl JoinSet { /// Polls for one of the tasks in the set to complete. /// - /// If this returns `Poll::Ready(Ok(Some(_)))` or `Poll::Ready(Err(_))`, then the task that - /// completed is removed from the set. + /// If this returns `Poll::Ready(Some(_))`, then the task that completed is removed from the set. /// /// When the method returns `Poll::Pending`, the `Waker` in the provided `Context` is scheduled /// to receive a wakeup when a task in the `JoinSet` completes. Note that on multiple calls to - /// `poll_join_one`, only the `Waker` from the `Context` passed to the most recent call is + /// `poll_join_next`, only the `Waker` from the `Context` passed to the most recent call is /// scheduled to receive a wakeup. /// /// # Returns @@ -189,24 +352,24 @@ impl JoinSet { /// /// * `Poll::Pending` if the `JoinSet` is not empty but there is no task whose output is /// available right now. - /// * `Poll::Ready(Ok(Some(value)))` if one of the tasks in this `JoinSet` has completed. The - /// `value` is the return value of one of the tasks that completed. - /// * `Poll::Ready(Err(err))` if one of the tasks in this `JoinSet` has panicked or been - /// aborted. - /// * `Poll::Ready(Ok(None))` if the `JoinSet` is empty. + /// * `Poll::Ready(Some(Ok(value)))` if one of the tasks in this `JoinSet` has completed. + /// The `value` is the return value of one of the tasks that completed. + /// * `Poll::Ready(Some(Err(err)))` if one of the tasks in this `JoinSet` has panicked or been + /// aborted. The `err` is the `JoinError` from the panicked/aborted task. + /// * `Poll::Ready(None)` if the `JoinSet` is empty. /// /// Note that this method may return `Poll::Pending` even if one of the tasks has completed. /// This can happen if the [coop budget] is reached. /// /// [coop budget]: crate::task#cooperative-scheduling - fn poll_join_one(&mut self, cx: &mut Context<'_>) -> Poll, JoinError>> { + pub fn poll_join_next(&mut self, cx: &mut Context<'_>) -> Poll>> { // The call to `pop_notified` moves the entry to the `idle` list. It is moved back to // the `notified` list if the waker is notified in the `poll` call below. let mut entry = match self.inner.pop_notified(cx.waker()) { Some(entry) => entry, None => { if self.is_empty() { - return Poll::Ready(Ok(None)); + return Poll::Ready(None); } else { // The waker was set by `pop_notified`. return Poll::Pending; @@ -217,8 +380,71 @@ impl JoinSet { let res = entry.with_value_and_context(|jh, ctx| Pin::new(jh).poll(ctx)); if let Poll::Ready(res) = res { - entry.remove(); - Poll::Ready(Some(res).transpose()) + let _entry = entry.remove(); + Poll::Ready(Some(res)) + } else { + // A JoinHandle generally won't emit a wakeup without being ready unless + // the coop limit has been reached. We yield to the executor in this + // case. + cx.waker().wake_by_ref(); + Poll::Pending + } + } + + /// Polls for one of the tasks in the set to complete. + /// + /// If this returns `Poll::Ready(Some(_))`, then the task that completed is removed from the set. + /// + /// When the method returns `Poll::Pending`, the `Waker` in the provided `Context` is scheduled + /// to receive a wakeup when a task in the `JoinSet` completes. Note that on multiple calls to + /// `poll_join_next`, only the `Waker` from the `Context` passed to the most recent call is + /// scheduled to receive a wakeup. + /// + /// # Returns + /// + /// This function returns: + /// + /// * `Poll::Pending` if the `JoinSet` is not empty but there is no task whose output is + /// available right now. + /// * `Poll::Ready(Some(Ok((id, value))))` if one of the tasks in this `JoinSet` has completed. + /// The `value` is the return value of one of the tasks that completed, and + /// `id` is the [task ID] of that task. + /// * `Poll::Ready(Some(Err(err)))` if one of the tasks in this `JoinSet` has panicked or been + /// aborted. The `err` is the `JoinError` from the panicked/aborted task. + /// * `Poll::Ready(None)` if the `JoinSet` is empty. + /// + /// Note that this method may return `Poll::Pending` even if one of the tasks has completed. + /// This can happen if the [coop budget] is reached. + /// + /// [coop budget]: crate::task#cooperative-scheduling + /// [task ID]: crate::task::Id + #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] + pub fn poll_join_next_with_id( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + // The call to `pop_notified` moves the entry to the `idle` list. It is moved back to + // the `notified` list if the waker is notified in the `poll` call below. + let mut entry = match self.inner.pop_notified(cx.waker()) { + Some(entry) => entry, + None => { + if self.is_empty() { + return Poll::Ready(None); + } else { + // The waker was set by `pop_notified`. + return Poll::Pending; + } + } + }; + + let res = entry.with_value_and_context(|jh, ctx| Pin::new(jh).poll(ctx)); + + if let Poll::Ready(res) = res { + let entry = entry.remove(); + // If the task succeeded, add the task ID to the output. Otherwise, the + // `JoinError` will already have the task's ID. + Poll::Ready(Some(res.map(|output| (entry.id(), output)))) } else { // A JoinHandle generally won't emit a wakeup without being ready unless // the coop limit has been reached. We yield to the executor in this @@ -246,3 +472,113 @@ impl Default for JoinSet { Self::new() } } + +// === impl Builder === + +#[cfg(all(tokio_unstable, feature = "tracing"))] +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] +impl<'a, T: 'static> Builder<'a, T> { + /// Assigns a name to the task which will be spawned. + pub fn name(self, name: &'a str) -> Self { + let builder = self.builder.name(name); + Self { builder, ..self } + } + + /// Spawn the provided task with this builder's settings and store it in the + /// [`JoinSet`], returning an [`AbortHandle`] that can be used to remotely + /// cancel the task. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// # Panics + /// + /// This method panics if called outside of a Tokio runtime. + /// + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn(self, future: F) -> std::io::Result + where + F: Future, + F: Send + 'static, + T: Send, + { + Ok(self.joinset.insert(self.builder.spawn(future)?)) + } + + /// Spawn the provided task on the provided [runtime handle] with this + /// builder's settings, and store it in the [`JoinSet`]. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// + /// [`AbortHandle`]: crate::task::AbortHandle + /// [runtime handle]: crate::runtime::Handle + #[track_caller] + pub fn spawn_on(self, future: F, handle: &Handle) -> std::io::Result + where + F: Future, + F: Send + 'static, + T: Send, + { + Ok(self.joinset.insert(self.builder.spawn_on(future, handle)?)) + } + + /// Spawn the provided task on the current [`LocalSet`] with this builder's + /// settings, and store it in the [`JoinSet`]. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// # Panics + /// + /// This method panics if it is called outside of a `LocalSet`. + /// + /// [`LocalSet`]: crate::task::LocalSet + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_local(self, future: F) -> std::io::Result + where + F: Future, + F: 'static, + { + Ok(self.joinset.insert(self.builder.spawn_local(future)?)) + } + + /// Spawn the provided task on the provided [`LocalSet`] with this builder's + /// settings, and store it in the [`JoinSet`]. + /// + /// # Returns + /// + /// An [`AbortHandle`] that can be used to remotely cancel the task. + /// + /// [`LocalSet`]: crate::task::LocalSet + /// [`AbortHandle`]: crate::task::AbortHandle + #[track_caller] + pub fn spawn_local_on(self, future: F, local_set: &LocalSet) -> std::io::Result + where + F: Future, + F: 'static, + { + Ok(self + .joinset + .insert(self.builder.spawn_local_on(future, local_set)?)) + } +} + +// Manual `Debug` impl so that `Builder` is `Debug` regardless of whether `T` is +// `Debug`. +#[cfg(all(tokio_unstable, feature = "tracing"))] +#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))] +impl<'a, T> fmt::Debug for Builder<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("join_set::Builder") + .field("joinset", &self.joinset) + .field("builder", &self.builder) + .finish() + } +} diff --git a/third_party/rust/tokio/src/task/local.rs b/third_party/rust/tokio/src/task/local.rs index 2dbd9706047e..734b95587b9e 100644 --- a/third_party/rust/tokio/src/task/local.rs +++ b/third_party/rust/tokio/src/task/local.rs @@ -1,8 +1,10 @@ //! Runs `!Send` futures on the current thread. +use crate::loom::cell::UnsafeCell; use crate::loom::sync::{Arc, Mutex}; use crate::runtime::task::{self, JoinHandle, LocalOwnedTasks, Task}; +use crate::runtime::{context, ThreadId}; use crate::sync::AtomicWaker; -use crate::util::VecDequeCell; +use crate::util::RcCell; use std::cell::Cell; use std::collections::VecDeque; @@ -10,6 +12,7 @@ use std::fmt; use std::future::Future; use std::marker::PhantomData; use std::pin::Pin; +use std::rc::Rc; use std::task::Poll; use pin_project_lite::pin_project; @@ -31,14 +34,14 @@ cfg_rt! { /// async fn main() { /// // `Rc` does not implement `Send`, and thus may not be sent between /// // threads safely. - /// let unsend_data = Rc::new("my unsend data..."); + /// let nonsend_data = Rc::new("my nonsend data..."); /// - /// let unsend_data = unsend_data.clone(); - /// // Because the `async` block here moves `unsend_data`, the future is `!Send`. + /// let nonsend_data = nonsend_data.clone(); + /// // Because the `async` block here moves `nonsend_data`, the future is `!Send`. /// // Since `tokio::spawn` requires the spawned future to implement `Send`, this /// // will not compile. /// tokio::spawn(async move { - /// println!("{}", unsend_data); + /// println!("{}", nonsend_data); /// // ... /// }).await.unwrap(); /// } @@ -57,18 +60,18 @@ cfg_rt! { /// /// #[tokio::main] /// async fn main() { - /// let unsend_data = Rc::new("my unsend data..."); + /// let nonsend_data = Rc::new("my nonsend data..."); /// /// // Construct a local task set that can run `!Send` futures. /// let local = task::LocalSet::new(); /// /// // Run the local task set. /// local.run_until(async move { - /// let unsend_data = unsend_data.clone(); + /// let nonsend_data = nonsend_data.clone(); /// // `spawn_local` ensures that the future is spawned on the local /// // task set. /// task::spawn_local(async move { - /// println!("{}", unsend_data); + /// println!("{}", nonsend_data); /// // ... /// }).await.unwrap(); /// }).await; @@ -91,18 +94,18 @@ cfg_rt! { /// /// #[tokio::main] /// async fn main() { - /// let unsend_data = Rc::new("world"); + /// let nonsend_data = Rc::new("world"); /// let local = task::LocalSet::new(); /// - /// let unsend_data2 = unsend_data.clone(); + /// let nonsend_data2 = nonsend_data.clone(); /// local.spawn_local(async move { /// // ... - /// println!("hello {}", unsend_data2) + /// println!("hello {}", nonsend_data2) /// }); /// /// local.spawn_local(async move { /// time::sleep(time::Duration::from_millis(100)).await; - /// println!("goodbye {}", unsend_data) + /// println!("goodbye {}", nonsend_data) /// }); /// /// // ... @@ -159,7 +162,7 @@ cfg_rt! { /// tokio::task::spawn_local(run_task(new_task)); /// } /// // If the while loop returns, then all the LocalSpawner - /// // objects have have been dropped. + /// // objects have been dropped. /// }); /// /// // This will return once all senders are dropped and all @@ -215,7 +218,7 @@ cfg_rt! { tick: Cell, /// State available from thread-local. - context: Context, + context: Rc, /// This type should not be Send. _not_send: PhantomData<*const ()>, @@ -224,23 +227,44 @@ cfg_rt! { /// State available from the thread-local. struct Context { - /// Collection of all active tasks spawned onto this executor. - owned: LocalOwnedTasks>, - - /// Local run queue sender and receiver. - queue: VecDequeCell>>, - /// State shared between threads. shared: Arc, + + /// True if a task panicked without being handled and the local set is + /// configured to shutdown on unhandled panic. + unhandled_panic: Cell, } /// LocalSet state shared between threads. struct Shared { + /// # Safety + /// + /// This field must *only* be accessed from the thread that owns the + /// `LocalSet` (i.e., `Thread::current().id() == owner`). + local_state: LocalState, + /// Remote run queue sender. queue: Mutex>>>>, /// Wake the `LocalSet` task. waker: AtomicWaker, + + /// How to respond to unhandled task panics. + #[cfg(tokio_unstable)] + pub(crate) unhandled_panic: crate::runtime::UnhandledPanic, +} + +/// Tracks the `LocalSet` state that must only be accessed from the thread that +/// created the `LocalSet`. +struct LocalState { + /// The `ThreadId` of the thread that owns the `LocalSet`. + owner: ThreadId, + + /// Local run queue sender and receiver. + local_queue: UnsafeCell>>>, + + /// Collection of all active tasks spawned onto this executor. + owned: LocalOwnedTasks>, } pin_project! { @@ -252,17 +276,30 @@ pin_project! { } } -scoped_thread_local!(static CURRENT: Context); +tokio_thread_local!(static CURRENT: LocalData = const { LocalData { + ctx: RcCell::new(), +} }); + +struct LocalData { + ctx: RcCell, +} cfg_rt! { - /// Spawns a `!Send` future on the local task set. + /// Spawns a `!Send` future on the current [`LocalSet`]. /// - /// The spawned future will be run on the same thread that called `spawn_local.` - /// This may only be called from the context of a local task set. + /// The spawned future will run on the same thread that called `spawn_local`. + /// + /// The provided future will start running in the background immediately + /// when `spawn_local` is called, even if you don't await the returned + /// `JoinHandle`. /// /// # Panics /// - /// - This function panics if called outside of a local task set. + /// This function panics if called outside of a [`LocalSet`]. + /// + /// Note that if [`tokio::spawn`] is used from within a `LocalSet`, the + /// resulting new task will _not_ be inside the `LocalSet`, so you must use + /// `spawn_local` if you want to stay within the `LocalSet`. /// /// # Examples /// @@ -272,20 +309,23 @@ cfg_rt! { /// /// #[tokio::main] /// async fn main() { - /// let unsend_data = Rc::new("my unsend data..."); + /// let nonsend_data = Rc::new("my nonsend data..."); /// /// let local = task::LocalSet::new(); /// /// // Run the local task set. /// local.run_until(async move { - /// let unsend_data = unsend_data.clone(); + /// let nonsend_data = nonsend_data.clone(); /// task::spawn_local(async move { - /// println!("{}", unsend_data); + /// println!("{}", nonsend_data); /// // ... /// }).await.unwrap(); /// }).await; /// } /// ``` + /// + /// [`LocalSet`]: struct@crate::task::LocalSet + /// [`tokio::spawn`]: fn@crate::task::spawn #[track_caller] pub fn spawn_local(future: F) -> JoinHandle where @@ -301,19 +341,10 @@ cfg_rt! { where F: Future + 'static, F::Output: 'static { - let future = crate::util::trace::task(future, "local", name); - CURRENT.with(|maybe_cx| { - let cx = maybe_cx - .expect("`spawn_local` called from outside of a `task::LocalSet`"); - - let (handle, notified) = cx.owned.bind(future, cx.shared.clone()); - - if let Some(notified) = notified { - cx.shared.schedule(notified); - } - - handle - }) + match CURRENT.with(|LocalData { ctx, .. }| ctx.get()) { + None => panic!("`spawn_local` called from outside of a `task::LocalSet`"), + Some(cx) => cx.spawn(future, name) + } } } @@ -326,29 +357,72 @@ const MAX_TASKS_PER_TICK: usize = 61; /// How often it check the remote queue first. const REMOTE_FIRST_INTERVAL: u8 = 31; +/// Context guard for LocalSet +pub struct LocalEnterGuard(Option>); + +impl Drop for LocalEnterGuard { + fn drop(&mut self) { + CURRENT.with(|LocalData { ctx, .. }| { + ctx.set(self.0.take()); + }) + } +} + +impl fmt::Debug for LocalEnterGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalEnterGuard").finish() + } +} + impl LocalSet { /// Returns a new local task set. pub fn new() -> LocalSet { + let owner = context::thread_id().expect("cannot create LocalSet during thread shutdown"); + LocalSet { tick: Cell::new(0), - context: Context { - owned: LocalOwnedTasks::new(), - queue: VecDequeCell::with_capacity(INITIAL_CAPACITY), + context: Rc::new(Context { shared: Arc::new(Shared { + local_state: LocalState { + owner, + owned: LocalOwnedTasks::new(), + local_queue: UnsafeCell::new(VecDeque::with_capacity(INITIAL_CAPACITY)), + }, queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))), waker: AtomicWaker::new(), + #[cfg(tokio_unstable)] + unhandled_panic: crate::runtime::UnhandledPanic::Ignore, }), - }, + unhandled_panic: Cell::new(false), + }), _not_send: PhantomData, } } + /// Enters the context of this `LocalSet`. + /// + /// The [`spawn_local`] method will spawn tasks on the `LocalSet` whose + /// context you are inside. + /// + /// [`spawn_local`]: fn@crate::task::spawn_local + pub fn enter(&self) -> LocalEnterGuard { + CURRENT.with(|LocalData { ctx, .. }| { + let old = ctx.replace(Some(self.context.clone())); + LocalEnterGuard(old) + }) + } + /// Spawns a `!Send` task onto the local task set. /// /// This task is guaranteed to be run on the current thread. /// /// Unlike the free function [`spawn_local`], this method may be used to - /// spawn local tasks when the task set is _not_ running. For example: + /// spawn local tasks when the `LocalSet` is _not_ running. The provided + /// future will start running once the `LocalSet` is next started, even if + /// you don't await the returned `JoinHandle`. + /// + /// # Examples + /// /// ```rust /// use tokio::task; /// @@ -385,16 +459,7 @@ impl LocalSet { F: Future + 'static, F::Output: 'static, { - let future = crate::util::trace::task(future, "local", None); - - let (handle, notified) = self.context.owned.bind(future, self.context.shared.clone()); - - if let Some(notified) = notified { - self.context.shared.schedule(notified); - } - - self.context.shared.waker.wake(); - handle + self.spawn_named(future, None) } /// Runs a future to completion on the provided runtime, driving any local @@ -459,6 +524,7 @@ impl LocalSet { /// [`Runtime::block_on`]: method@crate::runtime::Runtime::block_on /// [in-place blocking]: fn@crate::task::block_in_place /// [`spawn_blocking`]: fn@crate::task::spawn_blocking + #[track_caller] #[cfg(feature = "rt")] #[cfg_attr(docsrs, doc(cfg(feature = "rt")))] pub fn block_on(&self, rt: &crate::runtime::Runtime, future: F) -> F::Output @@ -507,10 +573,36 @@ impl LocalSet { run_until.await } + pub(in crate::task) fn spawn_named( + &self, + future: F, + name: Option<&str>, + ) -> JoinHandle + where + F: Future + 'static, + F::Output: 'static, + { + let handle = self.context.spawn(future, name); + + // Because a task was spawned from *outside* the `LocalSet`, wake the + // `LocalSet` future to execute the new task, if it hasn't been woken. + // + // Spawning via the free fn `spawn` does not require this, as it can + // only be called from *within* a future executing on the `LocalSet` — + // in that case, the `LocalSet` must already be awake. + self.context.shared.waker.wake(); + handle + } + /// Ticks the scheduler, returning whether the local future needs to be /// notified again. fn tick(&self) -> bool { for _ in 0..MAX_TASKS_PER_TICK { + // Make sure we didn't hit an unhandled panic + if self.context.unhandled_panic.get() { + panic!("a spawned task panicked and the LocalSet is configured to shutdown on unhandled panic"); + } + match self.next_task() { // Run the task // @@ -520,7 +612,7 @@ impl LocalSet { // task initially. Because `LocalSet` itself is `!Send`, and // `spawn_local` spawns into the `LocalSet` on the current // thread, the invariant is maintained. - Some(task) => crate::coop::budget(|| task.run()), + Some(task) => crate::runtime::coop::budget(|| task.run()), // We have fully drained the queue of notified tasks, so the // local future doesn't need to be notified again — it can wait // until something else wakes a task in the local set. @@ -542,9 +634,9 @@ impl LocalSet { .lock() .as_mut() .and_then(|queue| queue.pop_front()) - .or_else(|| self.context.queue.pop_front()) + .or_else(|| self.pop_local()) } else { - self.context.queue.pop_front().or_else(|| { + self.pop_local().or_else(|| { self.context .shared .queue @@ -554,11 +646,145 @@ impl LocalSet { }) }; - task.map(|task| self.context.owned.assert_owner(task)) + task.map(|task| unsafe { + // Safety: because the `LocalSet` itself is `!Send`, we know we are + // on the same thread if we have access to the `LocalSet`, and can + // therefore access the local run queue. + self.context.shared.local_state.assert_owner(task) + }) + } + + fn pop_local(&self) -> Option>> { + unsafe { + // Safety: because the `LocalSet` itself is `!Send`, we know we are + // on the same thread if we have access to the `LocalSet`, and can + // therefore access the local run queue. + self.context.shared.local_state.task_pop_front() + } } fn with(&self, f: impl FnOnce() -> T) -> T { - CURRENT.set(&self.context, f) + CURRENT.with(|LocalData { ctx, .. }| { + struct Reset<'a> { + ctx_ref: &'a RcCell, + val: Option>, + } + impl<'a> Drop for Reset<'a> { + fn drop(&mut self) { + self.ctx_ref.set(self.val.take()); + } + } + let old = ctx.replace(Some(self.context.clone())); + + let _reset = Reset { + ctx_ref: ctx, + val: old, + }; + + f() + }) + } + + /// This method is like `with`, but it just calls `f` without setting the thread-local if that + /// fails. + fn with_if_possible(&self, f: impl FnOnce() -> T) -> T { + let mut f = Some(f); + + let res = CURRENT.try_with(|LocalData { ctx, .. }| { + struct Reset<'a> { + ctx_ref: &'a RcCell, + val: Option>, + } + impl<'a> Drop for Reset<'a> { + fn drop(&mut self) { + self.ctx_ref.replace(self.val.take()); + } + } + let old = ctx.replace(Some(self.context.clone())); + + let _reset = Reset { + ctx_ref: ctx, + val: old, + }; + + (f.take().unwrap())() + }); + + match res { + Ok(res) => res, + Err(_access_error) => (f.take().unwrap())(), + } + } +} + +cfg_unstable! { + impl LocalSet { + /// Configure how the `LocalSet` responds to an unhandled panic on a + /// spawned task. + /// + /// By default, an unhandled panic (i.e. a panic not caught by + /// [`std::panic::catch_unwind`]) has no impact on the `LocalSet`'s + /// execution. The panic is error value is forwarded to the task's + /// [`JoinHandle`] and all other spawned tasks continue running. + /// + /// The `unhandled_panic` option enables configuring this behavior. + /// + /// * `UnhandledPanic::Ignore` is the default behavior. Panics on + /// spawned tasks have no impact on the `LocalSet`'s execution. + /// * `UnhandledPanic::ShutdownRuntime` will force the `LocalSet` to + /// shutdown immediately when a spawned task panics even if that + /// task's `JoinHandle` has not been dropped. All other spawned tasks + /// will immediately terminate and further calls to + /// [`LocalSet::block_on`] and [`LocalSet::run_until`] will panic. + /// + /// # Panics + /// + /// This method panics if called after the `LocalSet` has started + /// running. + /// + /// # Unstable + /// + /// This option is currently unstable and its implementation is + /// incomplete. The API may change or be removed in the future. See + /// tokio-rs/tokio#4516 for more details. + /// + /// # Examples + /// + /// The following demonstrates a `LocalSet` configured to shutdown on + /// panic. The first spawned task panics and results in the `LocalSet` + /// shutting down. The second spawned task never has a chance to + /// execute. The call to `run_until` will panic due to the runtime being + /// forcibly shutdown. + /// + /// ```should_panic + /// use tokio::runtime::UnhandledPanic; + /// + /// # #[tokio::main] + /// # async fn main() { + /// tokio::task::LocalSet::new() + /// .unhandled_panic(UnhandledPanic::ShutdownRuntime) + /// .run_until(async { + /// tokio::task::spawn_local(async { panic!("boom"); }); + /// tokio::task::spawn_local(async { + /// // This task never completes + /// }); + /// + /// // Do some work, but `run_until` will panic before it completes + /// # loop { tokio::task::yield_now().await; } + /// }) + /// .await; + /// # } + /// ``` + /// + /// [`JoinHandle`]: struct@crate::task::JoinHandle + pub fn unhandled_panic(&mut self, behavior: crate::runtime::UnhandledPanic) -> &mut Self { + // TODO: This should be set as a builder + Rc::get_mut(&mut self.context) + .and_then(|ctx| Arc::get_mut(&mut ctx.shared)) + .expect("Unhandled Panic behavior modified after starting LocalSet") + .unhandled_panic = behavior; + self + } } } @@ -580,7 +806,10 @@ impl Future for LocalSet { // there are still tasks remaining in the run queue. cx.waker().wake_by_ref(); Poll::Pending - } else if self.context.owned.is_empty() { + + // Safety: called from the thread that owns `LocalSet`. Because + // `LocalSet` is `!Send`, this is safe. + } else if unsafe { self.context.shared.local_state.owned_is_empty() } { // If the scheduler has no remaining futures, we're done! Poll::Ready(()) } else { @@ -600,14 +829,34 @@ impl Default for LocalSet { impl Drop for LocalSet { fn drop(&mut self) { - self.with(|| { + self.with_if_possible(|| { // Shut down all tasks in the LocalOwnedTasks and close it to // prevent new tasks from ever being added. - self.context.owned.close_and_shutdown_all(); + unsafe { + // Safety: called from the thread that owns `LocalSet` + self.context.shared.local_state.close_and_shutdown_all(); + } // We already called shutdown on all tasks above, so there is no // need to call shutdown. - for task in self.context.queue.take() { + + // Safety: note that this *intentionally* bypasses the unsafe + // `Shared::local_queue()` method. This is in order to avoid the + // debug assertion that we are on the thread that owns the + // `LocalSet`, because on some systems (e.g. at least some macOS + // versions), attempting to get the current thread ID can panic due + // to the thread's local data that stores the thread ID being + // dropped *before* the `LocalSet`. + // + // Despite avoiding the assertion here, it is safe for us to access + // the local queue in `Drop`, because the `LocalSet` itself is + // `!Send`, so we can reasonably guarantee that it will not be + // `Drop`ped from another thread. + let local_queue = unsafe { + // Safety: called from the thread that owns `LocalSet` + self.context.shared.local_state.take_local_queue() + }; + for task in local_queue { drop(task); } @@ -618,11 +867,41 @@ impl Drop for LocalSet { drop(task); } - assert!(self.context.owned.is_empty()); + // Safety: called from the thread that owns `LocalSet` + assert!(unsafe { self.context.shared.local_state.owned_is_empty() }); }); } } +// === impl Context === + +impl Context { + #[track_caller] + fn spawn(&self, future: F, name: Option<&str>) -> JoinHandle + where + F: Future + 'static, + F::Output: 'static, + { + let id = crate::runtime::task::Id::next(); + let future = crate::util::trace::task(future, "local", name, id.as_u64()); + + // Safety: called from the thread that owns the `LocalSet` + let (handle, notified) = { + self.shared.local_state.assert_called_from_owner_thread(); + self.shared + .local_state + .owned + .bind(future, self.shared.clone(), id) + }; + + if let Some(notified) = notified { + self.shared.schedule(notified); + } + + handle + } +} + // === impl LocalFuture === impl Future for RunUntil<'_, T> { @@ -638,10 +917,10 @@ impl Future for RunUntil<'_, T> { .waker .register_by_ref(cx.waker()); - let _no_blocking = crate::runtime::enter::disallow_blocking(); + let _no_blocking = crate::runtime::context::disallow_block_in_place(); let f = me.future; - if let Poll::Ready(output) = crate::coop::budget(|| f.poll(cx)) { + if let Poll::Ready(output) = f.poll(cx) { return Poll::Ready(output); } @@ -659,21 +938,41 @@ impl Future for RunUntil<'_, T> { impl Shared { /// Schedule the provided task on the scheduler. fn schedule(&self, task: task::Notified>) { - CURRENT.with(|maybe_cx| match maybe_cx { - Some(cx) if cx.shared.ptr_eq(self) => { - cx.queue.push_back(task); - } - _ => { - // First check whether the queue is still there (if not, the - // LocalSet is dropped). Then push to it if so, and if not, - // do nothing. - let mut lock = self.queue.lock(); + CURRENT.with(|localdata| { + match localdata.ctx.get() { + Some(cx) if cx.shared.ptr_eq(self) => unsafe { + // Safety: if the current `LocalSet` context points to this + // `LocalSet`, then we are on the thread that owns it. + cx.shared.local_state.task_push_back(task); + }, - if let Some(queue) = lock.as_mut() { - queue.push_back(task); - drop(lock); + // We are on the thread that owns the `LocalSet`, so we can + // wake to the local queue. + _ if context::thread_id().ok() == Some(self.local_state.owner) => { + unsafe { + // Safety: we just checked that the thread ID matches + // the localset's owner, so this is safe. + self.local_state.task_push_back(task); + } + // We still have to wake the `LocalSet`, because it isn't + // currently being polled. self.waker.wake(); } + + // We are *not* on the thread that owns the `LocalSet`, so we + // have to wake to the remote queue. + _ => { + // First, check whether the queue is still there (if not, the + // LocalSet is dropped). Then push to it if so, and if not, + // do nothing. + let mut lock = self.queue.lock(); + + if let Some(queue) = lock.as_mut() { + queue.push_back(task); + drop(lock); + self.waker.wake(); + } + } } }); } @@ -683,16 +982,194 @@ impl Shared { } } +// This is safe because (and only because) we *pinky pwomise* to never touch the +// local run queue except from the thread that owns the `LocalSet`. +unsafe impl Sync for Shared {} + impl task::Schedule for Arc { fn release(&self, task: &Task) -> Option> { - CURRENT.with(|maybe_cx| { - let cx = maybe_cx.expect("scheduler context missing"); - assert!(cx.shared.ptr_eq(self)); - cx.owned.remove(task) - }) + // Safety, this is always called from the thread that owns `LocalSet` + unsafe { self.local_state.task_remove(task) } } fn schedule(&self, task: task::Notified) { Shared::schedule(self, task); } + + cfg_unstable! { + fn unhandled_panic(&self) { + use crate::runtime::UnhandledPanic; + + match self.unhandled_panic { + UnhandledPanic::Ignore => { + // Do nothing + } + UnhandledPanic::ShutdownRuntime => { + // This hook is only called from within the runtime, so + // `CURRENT` should match with `&self`, i.e. there is no + // opportunity for a nested scheduler to be called. + CURRENT.with(|LocalData { ctx, .. }| match ctx.get() { + Some(cx) if Arc::ptr_eq(self, &cx.shared) => { + cx.unhandled_panic.set(true); + // Safety: this is always called from the thread that owns `LocalSet` + unsafe { cx.shared.local_state.close_and_shutdown_all(); } + } + _ => unreachable!("runtime core not set in CURRENT thread-local"), + }) + } + } + } + } +} + +impl LocalState { + unsafe fn task_pop_front(&self) -> Option>> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.local_queue.with_mut(|ptr| (*ptr).pop_front()) + } + + unsafe fn task_push_back(&self, task: task::Notified>) { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.local_queue.with_mut(|ptr| (*ptr).push_back(task)) + } + + unsafe fn take_local_queue(&self) -> VecDeque>> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.local_queue.with_mut(|ptr| std::mem::take(&mut (*ptr))) + } + + unsafe fn task_remove(&self, task: &Task>) -> Option>> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.remove(task) + } + + /// Returns true if the `LocalSet` does not have any spawned tasks + unsafe fn owned_is_empty(&self) -> bool { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.is_empty() + } + + unsafe fn assert_owner( + &self, + task: task::Notified>, + ) -> task::LocalNotified> { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.assert_owner(task) + } + + unsafe fn close_and_shutdown_all(&self) { + // The caller ensures it is called from the same thread that owns + // the LocalSet. + self.assert_called_from_owner_thread(); + + self.owned.close_and_shutdown_all() + } + + #[track_caller] + fn assert_called_from_owner_thread(&self) { + // FreeBSD has some weirdness around thread-local destruction. + // TODO: remove this hack when thread id is cleaned up + #[cfg(not(any(target_os = "openbsd", target_os = "freebsd")))] + debug_assert!( + // if we couldn't get the thread ID because we're dropping the local + // data, skip the assertion --- the `Drop` impl is not going to be + // called from another thread, because `LocalSet` is `!Send` + context::thread_id() + .map(|id| id == self.owner) + .unwrap_or(true), + "`LocalSet`'s local run queue must not be accessed by another thread!" + ); + } +} + +// This is `Send` because it is stored in `Shared`. It is up to the caller to +// ensure they are on the same thread that owns the `LocalSet`. +unsafe impl Send for LocalState {} + +#[cfg(all(test, not(loom)))] +mod tests { + use super::*; + + // Does a `LocalSet` running on a current-thread runtime...basically work? + // + // This duplicates a test in `tests/task_local_set.rs`, but because this is + // a lib test, it wil run under Miri, so this is necessary to catch stacked + // borrows violations in the `LocalSet` implementation. + #[test] + fn local_current_thread_scheduler() { + let f = async { + LocalSet::new() + .run_until(async { + spawn_local(async {}).await.unwrap(); + }) + .await; + }; + crate::runtime::Builder::new_current_thread() + .build() + .expect("rt") + .block_on(f) + } + + // Tests that when a task on a `LocalSet` is woken by an io driver on the + // same thread, the task is woken to the localset's local queue rather than + // its remote queue. + // + // This test has to be defined in the `local.rs` file as a lib test, rather + // than in `tests/`, because it makes assertions about the local set's + // internal state. + #[test] + fn wakes_to_local_queue() { + use super::*; + use crate::sync::Notify; + let rt = crate::runtime::Builder::new_current_thread() + .build() + .expect("rt"); + rt.block_on(async { + let local = LocalSet::new(); + let notify = Arc::new(Notify::new()); + let task = local.spawn_local({ + let notify = notify.clone(); + async move { + notify.notified().await; + } + }); + let mut run_until = Box::pin(local.run_until(async move { + task.await.unwrap(); + })); + + // poll the run until future once + crate::future::poll_fn(|cx| { + let _ = run_until.as_mut().poll(cx); + Poll::Ready(()) + }) + .await; + + notify.notify_one(); + let task = unsafe { local.context.shared.local_state.task_pop_front() }; + // TODO(eliza): it would be nice to be able to assert that this is + // the local task. + assert!( + task.is_some(), + "task should have been notified to the LocalSet's local queue" + ); + }) + } } diff --git a/third_party/rust/tokio/src/task/mod.rs b/third_party/rust/tokio/src/task/mod.rs index d532155a1fe2..9b7537018542 100644 --- a/third_party/rust/tokio/src/task/mod.rs +++ b/third_party/rust/tokio/src/task/mod.rs @@ -243,7 +243,7 @@ //! //! #### unconstrained //! -//! If necessary, [`task::unconstrained`] lets you opt out a future of Tokio's cooperative +//! If necessary, [`task::unconstrained`] lets you opt a future out of of Tokio's cooperative //! scheduling. When a future is wrapped with `unconstrained`, it will never be forced to yield to //! Tokio. For example: //! @@ -278,8 +278,10 @@ cfg_rt! { pub use crate::runtime::task::{JoinError, JoinHandle}; - mod blocking; - pub use blocking::spawn_blocking; + cfg_not_wasi! { + mod blocking; + pub use blocking::spawn_blocking; + } mod spawn; pub use spawn::spawn; @@ -291,8 +293,13 @@ cfg_rt! { mod yield_now; pub use yield_now::yield_now; + cfg_unstable! { + mod consume_budget; + pub use consume_budget::consume_budget; + } + mod local; - pub use local::{spawn_local, LocalSet}; + pub use local::{spawn_local, LocalSet, LocalEnterGuard}; mod task_local; pub use task_local::LocalKey; @@ -300,9 +307,18 @@ cfg_rt! { mod unconstrained; pub use unconstrained::{unconstrained, Unconstrained}; + #[doc(inline)] + pub use join_set::JoinSet; + pub use crate::runtime::task::AbortHandle; + + // Uses #[cfg(...)] instead of macro since the macro adds docsrs annotations. + #[cfg(not(tokio_unstable))] + mod join_set; + #[cfg(tokio_unstable)] + pub mod join_set; + cfg_unstable! { - mod join_set; - pub use join_set::JoinSet; + pub use crate::runtime::task::{Id, id, try_id}; } cfg_trace! { diff --git a/third_party/rust/tokio/src/task/spawn.rs b/third_party/rust/tokio/src/task/spawn.rs index a9d736674c03..23a46d74562f 100644 --- a/third_party/rust/tokio/src/task/spawn.rs +++ b/third_party/rust/tokio/src/task/spawn.rs @@ -1,4 +1,4 @@ -use crate::{task::JoinHandle, util::error::CONTEXT_MISSING_ERROR}; +use crate::task::JoinHandle; use std::future::Future; @@ -6,11 +6,19 @@ cfg_rt! { /// Spawns a new asynchronous task, returning a /// [`JoinHandle`](super::JoinHandle) for it. /// + /// The provided future will start running in the background immediately + /// when `spawn` is called, even if you don't await the returned + /// `JoinHandle`. + /// /// Spawning a task enables the task to execute concurrently to other tasks. The /// spawned task may execute on the current thread, or it may be sent to a /// different thread to be executed. The specifics depend on the current /// [`Runtime`](crate::runtime::Runtime) configuration. /// + /// It is guaranteed that spawn will not synchronously poll the task being spawned. + /// This means that calling spawn while holding a lock does not pose a risk of + /// deadlocking with the spawned task. + /// /// There is no guarantee that a spawned task will execute to completion. /// When a runtime is shutdown, all outstanding tasks are dropped, /// regardless of the lifecycle of that task. @@ -49,6 +57,37 @@ cfg_rt! { /// } /// ``` /// + /// To run multiple tasks in parallel and receive their results, join + /// handles can be stored in a vector. + /// ``` + /// # #[tokio::main(flavor = "current_thread")] async fn main() { + /// async fn my_background_op(id: i32) -> String { + /// let s = format!("Starting background task {}.", id); + /// println!("{}", s); + /// s + /// } + /// + /// let ops = vec![1, 2, 3]; + /// let mut tasks = Vec::with_capacity(ops.len()); + /// for op in ops { + /// // This call will make them start running in the background + /// // immediately. + /// tasks.push(tokio::spawn(my_background_op(op))); + /// } + /// + /// let mut outputs = Vec::with_capacity(tasks.len()); + /// for task in tasks { + /// outputs.push(task.await.unwrap()); + /// } + /// println!("{:?}", outputs); + /// # } + /// ``` + /// This example pushes the tasks to `outputs` in the order they were + /// started in. If you do not care about the ordering of the outputs, then + /// you can also use a [`JoinSet`]. + /// + /// [`JoinSet`]: struct@crate::task::JoinSet + /// /// # Panics /// /// Panics if called from **outside** of the Tokio runtime. @@ -142,8 +181,26 @@ cfg_rt! { T: Future + Send + 'static, T::Output: Send + 'static, { - let spawn_handle = crate::runtime::context::spawn_handle().expect(CONTEXT_MISSING_ERROR); - let task = crate::util::trace::task(future, "task", name); - spawn_handle.spawn(task) + use crate::runtime::{context, task}; + + #[cfg(all( + tokio_unstable, + tokio_taskdump, + feature = "rt", + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64" + ) + ))] + let future = task::trace::Trace::root(future); + let id = task::Id::next(); + let task = crate::util::trace::task(future, "task", name, id.as_u64()); + + match context::with_current(|handle| handle.spawn(task, id)) { + Ok(join_handle) => join_handle, + Err(e) => panic!("{}", e), + } } } diff --git a/third_party/rust/tokio/src/task/task_local.rs b/third_party/rust/tokio/src/task/task_local.rs index 949bbca3eee9..eeadfbd3efc9 100644 --- a/third_party/rust/tokio/src/task/task_local.rs +++ b/third_party/rust/tokio/src/task/task_local.rs @@ -5,7 +5,7 @@ use std::future::Future; use std::marker::PhantomPinned; use std::pin::Pin; use std::task::{Context, Poll}; -use std::{fmt, thread}; +use std::{fmt, mem, thread}; /// Declares a new task-local key of type [`tokio::task::LocalKey`]. /// @@ -48,9 +48,27 @@ macro_rules! task_local { } #[doc(hidden)] +#[cfg(not(tokio_no_const_thread_local))] #[macro_export] macro_rules! __task_local_inner { ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty) => { + $(#[$attr])* + $vis static $name: $crate::task::LocalKey<$t> = { + std::thread_local! { + static __KEY: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; + } + + $crate::task::LocalKey { inner: __KEY } + }; + }; +} + +#[doc(hidden)] +#[cfg(tokio_no_const_thread_local)] +#[macro_export] +macro_rules! __task_local_inner { + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty) => { + $(#[$attr])* $vis static $name: $crate::task::LocalKey<$t> = { std::thread_local! { static __KEY: std::cell::RefCell> = std::cell::RefCell::new(None); @@ -63,7 +81,7 @@ macro_rules! __task_local_inner { /// A key for task-local data. /// -/// This type is generated by the `task_local!` macro. +/// This type is generated by the [`task_local!`] macro. /// /// Unlike [`std::thread::LocalKey`], `tokio::task::LocalKey` will /// _not_ lazily initialize the value on first access. Instead, the @@ -91,7 +109,9 @@ macro_rules! __task_local_inner { /// }).await; /// # } /// ``` +/// /// [`std::thread::LocalKey`]: struct@std::thread::LocalKey +/// [`task_local!`]: ../macro.task_local.html #[cfg_attr(docsrs, doc(cfg(feature = "rt")))] pub struct LocalKey { #[doc(hidden)] @@ -103,6 +123,11 @@ impl LocalKey { /// /// On completion of `scope`, the task-local will be dropped. /// + /// ### Panics + /// + /// If you poll the returned future inside a call to [`with`] or + /// [`try_with`] on the same `LocalKey`, then the call to `poll` will panic. + /// /// ### Examples /// /// ``` @@ -116,6 +141,9 @@ impl LocalKey { /// }).await; /// # } /// ``` + /// + /// [`with`]: fn@Self::with + /// [`try_with`]: fn@Self::try_with pub fn scope(&'static self, value: T, f: F) -> TaskLocalFuture where F: Future, @@ -123,14 +151,19 @@ impl LocalKey { TaskLocalFuture { local: self, slot: Some(value), - future: f, + future: Some(f), _pinned: PhantomPinned, } } /// Sets a value `T` as the task-local value for the closure `F`. /// - /// On completion of `scope`, the task-local will be dropped. + /// On completion of `sync_scope`, the task-local will be dropped. + /// + /// ### Panics + /// + /// This method panics if called inside a call to [`with`] or [`try_with`] + /// on the same `LocalKey`. /// /// ### Examples /// @@ -145,34 +178,80 @@ impl LocalKey { /// }); /// # } /// ``` + /// + /// [`with`]: fn@Self::with + /// [`try_with`]: fn@Self::try_with + #[track_caller] pub fn sync_scope(&'static self, value: T, f: F) -> R where F: FnOnce() -> R, { - let scope = TaskLocalFuture { - local: self, - slot: Some(value), - future: (), - _pinned: PhantomPinned, - }; - crate::pin!(scope); - scope.with_task(|_| f()) + let mut value = Some(value); + match self.scope_inner(&mut value, f) { + Ok(res) => res, + Err(err) => err.panic(), + } + } + + fn scope_inner(&'static self, slot: &mut Option, f: F) -> Result + where + F: FnOnce() -> R, + { + struct Guard<'a, T: 'static> { + local: &'static LocalKey, + slot: &'a mut Option, + } + + impl<'a, T: 'static> Drop for Guard<'a, T> { + fn drop(&mut self) { + // This should not panic. + // + // We know that the RefCell was not borrowed before the call to + // `scope_inner`, so the only way for this to panic is if the + // closure has created but not destroyed a RefCell guard. + // However, we never give user-code access to the guards, so + // there's no way for user-code to forget to destroy a guard. + // + // The call to `with` also should not panic, since the + // thread-local wasn't destroyed when we first called + // `scope_inner`, and it shouldn't have gotten destroyed since + // then. + self.local.inner.with(|inner| { + let mut ref_mut = inner.borrow_mut(); + mem::swap(self.slot, &mut *ref_mut); + }); + } + } + + self.inner.try_with(|inner| { + inner + .try_borrow_mut() + .map(|mut ref_mut| mem::swap(slot, &mut *ref_mut)) + })??; + + let guard = Guard { local: self, slot }; + + let res = f(); + + drop(guard); + + Ok(res) } /// Accesses the current task-local and runs the provided closure. /// /// # Panics /// - /// This function will panic if not called within the context - /// of a future containing a task-local with the corresponding key. + /// This function will panic if the task local doesn't have a value set. + #[track_caller] pub fn with(&'static self, f: F) -> R where F: FnOnce(&T) -> R, { - self.try_with(f).expect( - "cannot access a Task Local Storage value \ - without setting it via `LocalKey::set`", - ) + match self.try_with(f) { + Ok(res) => res, + Err(_) => panic!("cannot access a task-local storage value without setting it first"), + } } /// Accesses the current task-local and runs the provided closure. @@ -184,19 +263,32 @@ impl LocalKey { where F: FnOnce(&T) -> R, { - self.inner.with(|v| { - if let Some(val) = v.borrow().as_ref() { - Ok(f(val)) - } else { - Err(AccessError { _private: () }) - } - }) + // If called after the thread-local storing the task-local is destroyed, + // then we are outside of a closure where the task-local is set. + // + // Therefore, it is correct to return an AccessError if `try_with` + // returns an error. + let try_with_res = self.inner.try_with(|v| { + // This call to `borrow` cannot panic because no user-defined code + // runs while a `borrow_mut` call is active. + v.borrow().as_ref().map(f) + }); + + match try_with_res { + Ok(Some(res)) => Ok(res), + Ok(None) | Err(_) => Err(AccessError { _private: () }), + } } } impl LocalKey { /// Returns a copy of the task-local value /// if the task-local value implements `Copy`. + /// + /// # Panics + /// + /// This function will panic if the task local doesn't have a value set. + #[track_caller] pub fn get(&'static self) -> T { self.with(|v| *v) } @@ -232,52 +324,82 @@ pin_project! { /// ``` pub struct TaskLocalFuture where - T: 'static + T: 'static, { local: &'static LocalKey, slot: Option, #[pin] - future: F, + future: Option, #[pin] _pinned: PhantomPinned, } -} -impl TaskLocalFuture { - fn with_task) -> R, R>(self: Pin<&mut Self>, f: F2) -> R { - struct Guard<'a, T: 'static> { - local: &'static LocalKey, - slot: &'a mut Option, - prev: Option, - } - - impl Drop for Guard<'_, T> { - fn drop(&mut self) { - let value = self.local.inner.with(|c| c.replace(self.prev.take())); - *self.slot = value; + impl PinnedDrop for TaskLocalFuture { + fn drop(this: Pin<&mut Self>) { + let this = this.project(); + if mem::needs_drop::() && this.future.is_some() { + // Drop the future while the task-local is set, if possible. Otherwise + // the future is dropped normally when the `Option` field drops. + let mut future = this.future; + let _ = this.local.scope_inner(this.slot, || { + future.set(None); + }); } } - - let project = self.project(); - let val = project.slot.take(); - - let prev = project.local.inner.with(|c| c.replace(val)); - - let _guard = Guard { - prev, - slot: project.slot, - local: *project.local, - }; - - f(project.future) } } impl Future for TaskLocalFuture { type Output = F::Output; + #[track_caller] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.with_task(|f| f.poll(cx)) + let this = self.project(); + let mut future_opt = this.future; + + let res = this + .local + .scope_inner(this.slot, || match future_opt.as_mut().as_pin_mut() { + Some(fut) => { + let res = fut.poll(cx); + if res.is_ready() { + future_opt.set(None); + } + Some(res) + } + None => None, + }); + + match res { + Ok(Some(res)) => res, + Ok(None) => panic!("`TaskLocalFuture` polled after completion"), + Err(err) => err.panic(), + } + } +} + +impl fmt::Debug for TaskLocalFuture +where + T: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// Format the Option without Some. + struct TransparentOption<'a, T> { + value: &'a Option, + } + impl<'a, T: fmt::Debug> fmt::Debug for TransparentOption<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.value.as_ref() { + Some(value) => value.fmt(f), + // Hitting the None branch should not be possible. + None => f.pad(""), + } + } + } + + f.debug_struct("TaskLocalFuture") + .field("value", &TransparentOption { value: &self.slot }) + .finish() } } @@ -300,3 +422,30 @@ impl fmt::Display for AccessError { } impl Error for AccessError {} + +enum ScopeInnerErr { + BorrowError, + AccessError, +} + +impl ScopeInnerErr { + #[track_caller] + fn panic(&self) -> ! { + match self { + Self::BorrowError => panic!("cannot enter a task-local scope while the task-local storage is borrowed"), + Self::AccessError => panic!("cannot enter a task-local scope during or after destruction of the underlying thread-local"), + } + } +} + +impl From for ScopeInnerErr { + fn from(_: std::cell::BorrowMutError) -> Self { + Self::BorrowError + } +} + +impl From for ScopeInnerErr { + fn from(_: std::thread::AccessError) -> Self { + Self::AccessError + } +} diff --git a/third_party/rust/tokio/src/task/unconstrained.rs b/third_party/rust/tokio/src/task/unconstrained.rs index 31c732bfc954..40384c8709ed 100644 --- a/third_party/rust/tokio/src/task/unconstrained.rs +++ b/third_party/rust/tokio/src/task/unconstrained.rs @@ -22,7 +22,7 @@ where cfg_coop! { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let inner = self.project().inner; - crate::coop::with_unconstrained(|| inner.poll(cx)) + crate::runtime::coop::with_unconstrained(|| inner.poll(cx)) } } diff --git a/third_party/rust/tokio/src/task/yield_now.rs b/third_party/rust/tokio/src/task/yield_now.rs index 148e3dc0c878..428d124c3ba8 100644 --- a/third_party/rust/tokio/src/task/yield_now.rs +++ b/third_party/rust/tokio/src/task/yield_now.rs @@ -1,3 +1,5 @@ +use crate::runtime::context; + use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; @@ -44,12 +46,16 @@ pub async fn yield_now() { type Output = (); fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + ready!(crate::trace::trace_leaf(cx)); + if self.yielded { return Poll::Ready(()); } self.yielded = true; - cx.waker().wake_by_ref(); + + context::defer(cx.waker()); + Poll::Pending } } diff --git a/third_party/rust/tokio/src/time/clock.rs b/third_party/rust/tokio/src/time/clock.rs index 41be9bac48ce..091cf4b19d7a 100644 --- a/third_party/rust/tokio/src/time/clock.rs +++ b/third_party/rust/tokio/src/time/clock.rs @@ -29,26 +29,53 @@ cfg_not_test_util! { cfg_test_util! { use crate::time::{Duration, Instant}; - use crate::loom::sync::{Arc, Mutex}; + use crate::loom::sync::Mutex; + use crate::loom::sync::atomic::Ordering; + use std::sync::atomic::AtomicBool as StdAtomicBool; cfg_rt! { - fn clock() -> Option { - crate::runtime::context::clock() + #[track_caller] + fn with_clock(f: impl FnOnce(Option<&Clock>) -> Result) -> R { + use crate::runtime::Handle; + + let res = match Handle::try_current() { + Ok(handle) => f(Some(handle.inner.driver().clock())), + Err(ref e) if e.is_missing_context() => f(None), + Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR), + }; + + match res { + Ok(ret) => ret, + Err(msg) => panic!("{}", msg), + } } } cfg_not_rt! { - fn clock() -> Option { - None + #[track_caller] + fn with_clock(f: impl FnOnce(Option<&Clock>) -> Result) -> R { + match f(None) { + Ok(ret) => ret, + Err(msg) => panic!("{}", msg), + } } } /// A handle to a source of time. - #[derive(Debug, Clone)] + #[derive(Debug)] pub(crate) struct Clock { - inner: Arc>, + inner: Mutex, } + // Used to track if the clock was ever paused. This is an optimization to + // avoid touching the mutex if `test-util` was accidentally enabled in + // release mode. + // + // A static is used so we can avoid accessing the thread-local as well. The + // `std` AtomicBool is used directly because loom does not support static + // atomics. + static DID_PAUSE_CLOCK: StdAtomicBool = StdAtomicBool::new(false); + #[derive(Debug)] struct Inner { /// True if the ability to pause time is enabled. @@ -59,6 +86,9 @@ cfg_test_util! { /// Instant at which the clock was last unfrozen. unfrozen: Option, + + /// Number of `inhibit_auto_advance` calls still in effect. + auto_advance_inhibit_count: usize, } /// Pauses time. @@ -96,9 +126,14 @@ cfg_test_util! { /// /// [`Sleep`]: crate::time::Sleep /// [`advance`]: crate::time::advance + #[track_caller] pub fn pause() { - let clock = clock().expect("time cannot be frozen from outside the Tokio runtime"); - clock.pause(); + with_clock(|maybe_clock| { + match maybe_clock { + Some(clock) => clock.pause(), + None => Err("time cannot be frozen from outside the Tokio runtime"), + } + }) } /// Resumes time. @@ -110,15 +145,23 @@ cfg_test_util! { /// /// Panics if time is not frozen or if called from outside of the Tokio /// runtime. + #[track_caller] pub fn resume() { - let clock = clock().expect("time cannot be frozen from outside the Tokio runtime"); - let mut inner = clock.inner.lock(); + with_clock(|maybe_clock| { + let clock = match maybe_clock { + Some(clock) => clock, + None => return Err("time cannot be frozen from outside the Tokio runtime"), + }; - if inner.unfrozen.is_some() { - panic!("time is not frozen"); - } + let mut inner = clock.inner.lock(); - inner.unfrozen = Some(std::time::Instant::now()); + if inner.unfrozen.is_some() { + return Err("time is not frozen"); + } + + inner.unfrozen = Some(std::time::Instant::now()); + Ok(()) + }) } /// Advances time. @@ -153,19 +196,31 @@ cfg_test_util! { /// /// [`sleep`]: fn@crate::time::sleep pub async fn advance(duration: Duration) { - let clock = clock().expect("time cannot be frozen from outside the Tokio runtime"); - clock.advance(duration); + with_clock(|maybe_clock| { + let clock = match maybe_clock { + Some(clock) => clock, + None => return Err("time cannot be frozen from outside the Tokio runtime"), + }; + + clock.advance(duration) + }); crate::task::yield_now().await; } /// Returns the current instant, factoring in frozen time. pub(crate) fn now() -> Instant { - if let Some(clock) = clock() { - clock.now() - } else { - Instant::from_std(std::time::Instant::now()) + if !DID_PAUSE_CLOCK.load(Ordering::Acquire) { + return Instant::from_std(std::time::Instant::now()); } + + with_clock(|maybe_clock| { + Ok(if let Some(clock) = maybe_clock { + clock.now() + } else { + Instant::from_std(std::time::Instant::now()) + }) + }) } impl Clock { @@ -175,47 +230,70 @@ cfg_test_util! { let now = std::time::Instant::now(); let clock = Clock { - inner: Arc::new(Mutex::new(Inner { + inner: Mutex::new(Inner { enable_pausing, base: now, unfrozen: Some(now), - })), + auto_advance_inhibit_count: 0, + }), }; if start_paused { - clock.pause(); + if let Err(msg) = clock.pause() { + panic!("{}", msg); + } } clock } - pub(crate) fn pause(&self) { + pub(crate) fn pause(&self) -> Result<(), &'static str> { let mut inner = self.inner.lock(); if !inner.enable_pausing { drop(inner); // avoid poisoning the lock - panic!("`time::pause()` requires the `current_thread` Tokio runtime. \ + return Err("`time::pause()` requires the `current_thread` Tokio runtime. \ This is the default Runtime used by `#[tokio::test]."); } - let elapsed = inner.unfrozen.as_ref().expect("time is already frozen").elapsed(); + // Track that we paused the clock + DID_PAUSE_CLOCK.store(true, Ordering::Release); + + let elapsed = match inner.unfrozen.as_ref() { + Some(v) => v.elapsed(), + None => return Err("time is already frozen") + }; inner.base += elapsed; inner.unfrozen = None; + + Ok(()) } - pub(crate) fn is_paused(&self) -> bool { + /// Temporarily stop auto-advancing the clock (see `tokio::time::pause`). + pub(crate) fn inhibit_auto_advance(&self) { + let mut inner = self.inner.lock(); + inner.auto_advance_inhibit_count += 1; + } + + pub(crate) fn allow_auto_advance(&self) { + let mut inner = self.inner.lock(); + inner.auto_advance_inhibit_count -= 1; + } + + pub(crate) fn can_auto_advance(&self) -> bool { let inner = self.inner.lock(); - inner.unfrozen.is_none() + inner.unfrozen.is_none() && inner.auto_advance_inhibit_count == 0 } - pub(crate) fn advance(&self, duration: Duration) { + pub(crate) fn advance(&self, duration: Duration) -> Result<(), &'static str> { let mut inner = self.inner.lock(); if inner.unfrozen.is_some() { - panic!("time is not frozen"); + return Err("time is not frozen"); } inner.base += duration; + Ok(()) } pub(crate) fn now(&self) -> Instant { diff --git a/third_party/rust/tokio/src/time/driver/handle.rs b/third_party/rust/tokio/src/time/driver/handle.rs deleted file mode 100644 index b61c0476e156..000000000000 --- a/third_party/rust/tokio/src/time/driver/handle.rs +++ /dev/null @@ -1,94 +0,0 @@ -use crate::loom::sync::Arc; -use crate::time::driver::ClockTime; -use std::fmt; - -/// Handle to time driver instance. -#[derive(Clone)] -pub(crate) struct Handle { - time_source: ClockTime, - inner: Arc, -} - -impl Handle { - /// Creates a new timer `Handle` from a shared `Inner` timer state. - pub(super) fn new(inner: Arc) -> Self { - let time_source = inner.state.lock().time_source.clone(); - Handle { time_source, inner } - } - - /// Returns the time source associated with this handle. - pub(super) fn time_source(&self) -> &ClockTime { - &self.time_source - } - - /// Access the driver's inner structure. - pub(super) fn get(&self) -> &super::Inner { - &*self.inner - } - - /// Checks whether the driver has been shutdown. - pub(super) fn is_shutdown(&self) -> bool { - self.inner.is_shutdown() - } -} - -cfg_rt! { - impl Handle { - /// Tries to get a handle to the current timer. - /// - /// # Panics - /// - /// This function panics if there is no current timer set. - /// - /// It can be triggered when [`Builder::enable_time`] or - /// [`Builder::enable_all`] are not included in the builder. - /// - /// It can also panic whenever a timer is created outside of a - /// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic, - /// since the function is executed outside of the runtime. - /// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic. - /// And this is because wrapping the function on an async makes it lazy, - /// and so gets executed inside the runtime successfully without - /// panicking. - /// - /// [`Builder::enable_time`]: crate::runtime::Builder::enable_time - /// [`Builder::enable_all`]: crate::runtime::Builder::enable_all - pub(crate) fn current() -> Self { - crate::runtime::context::time_handle() - .expect("A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers.") - } - } -} - -cfg_not_rt! { - impl Handle { - /// Tries to get a handle to the current timer. - /// - /// # Panics - /// - /// This function panics if there is no current timer set. - /// - /// It can be triggered when [`Builder::enable_time`] or - /// [`Builder::enable_all`] are not included in the builder. - /// - /// It can also panic whenever a timer is created outside of a - /// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic, - /// since the function is executed outside of the runtime. - /// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic. - /// And this is because wrapping the function on an async makes it lazy, - /// and so gets executed inside the runtime successfully without - /// panicking. - /// - /// [`Builder::enable_time`]: crate::runtime::Builder::enable_time - /// [`Builder::enable_all`]: crate::runtime::Builder::enable_all - pub(crate) fn current() -> Self { - panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR) - } - } -} - -impl fmt::Debug for Handle { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Handle") - } -} diff --git a/third_party/rust/tokio/src/time/driver/wheel/stack.rs b/third_party/rust/tokio/src/time/driver/wheel/stack.rs deleted file mode 100644 index 80651c309e6c..000000000000 --- a/third_party/rust/tokio/src/time/driver/wheel/stack.rs +++ /dev/null @@ -1,112 +0,0 @@ -use super::{Item, OwnedItem}; -use crate::time::driver::Entry; - -use std::ptr; - -/// A doubly linked stack. -#[derive(Debug)] -pub(crate) struct Stack { - head: Option, -} - -impl Default for Stack { - fn default() -> Stack { - Stack { head: None } - } -} - -impl Stack { - pub(crate) fn is_empty(&self) -> bool { - self.head.is_none() - } - - pub(crate) fn push(&mut self, entry: OwnedItem) { - // Get a pointer to the entry to for the prev link - let ptr: *const Entry = &*entry as *const _; - - // Remove the old head entry - let old = self.head.take(); - - unsafe { - // Ensure the entry is not already in a stack. - debug_assert!((*entry.next_stack.get()).is_none()); - debug_assert!((*entry.prev_stack.get()).is_null()); - - if let Some(ref entry) = old.as_ref() { - debug_assert!({ - // The head is not already set to the entry - ptr != &***entry as *const _ - }); - - // Set the previous link on the old head - *entry.prev_stack.get() = ptr; - } - - // Set this entry's next pointer - *entry.next_stack.get() = old; - } - - // Update the head pointer - self.head = Some(entry); - } - - /// Pops an item from the stack. - pub(crate) fn pop(&mut self) -> Option { - let entry = self.head.take(); - - unsafe { - if let Some(entry) = entry.as_ref() { - self.head = (*entry.next_stack.get()).take(); - - if let Some(entry) = self.head.as_ref() { - *entry.prev_stack.get() = ptr::null(); - } - - *entry.prev_stack.get() = ptr::null(); - } - } - - entry - } - - pub(crate) fn remove(&mut self, entry: &Item) { - unsafe { - // Ensure that the entry is in fact contained by the stack - debug_assert!({ - // This walks the full linked list even if an entry is found. - let mut next = self.head.as_ref(); - let mut contains = false; - - while let Some(n) = next { - if entry as *const _ == &**n as *const _ { - debug_assert!(!contains); - contains = true; - } - - next = (*n.next_stack.get()).as_ref(); - } - - contains - }); - - // Unlink `entry` from the next node - let next = (*entry.next_stack.get()).take(); - - if let Some(next) = next.as_ref() { - (*next.prev_stack.get()) = *entry.prev_stack.get(); - } - - // Unlink `entry` from the prev node - - if let Some(prev) = (*entry.prev_stack.get()).as_ref() { - *prev.next_stack.get() = next; - } else { - // It is the head - self.head = next; - } - - // Unset the prev pointer - *entry.prev_stack.get() = ptr::null(); - } - } -} diff --git a/third_party/rust/tokio/src/time/error.rs b/third_party/rust/tokio/src/time/error.rs index 63f0a3b0bda6..71344d434871 100644 --- a/third_party/rust/tokio/src/time/error.rs +++ b/third_party/rust/tokio/src/time/error.rs @@ -41,7 +41,10 @@ impl From for Error { } /// Errors returned by `Timeout`. -#[derive(Debug, PartialEq)] +/// +/// This error is returned when a timeout expires before the function was able +/// to finish. +#[derive(Debug, PartialEq, Eq)] pub struct Elapsed(()); #[derive(Debug)] diff --git a/third_party/rust/tokio/src/time/instant.rs b/third_party/rust/tokio/src/time/instant.rs index f18492930a75..14cf6e567b54 100644 --- a/third_party/rust/tokio/src/time/instant.rs +++ b/third_party/rust/tokio/src/time/instant.rs @@ -188,7 +188,7 @@ impl ops::Sub for Instant { type Output = Instant; fn sub(self, rhs: Duration) -> Instant { - Instant::from_std(self.std - rhs) + Instant::from_std(std::time::Instant::sub(self.std, rhs)) } } diff --git a/third_party/rust/tokio/src/time/interval.rs b/third_party/rust/tokio/src/time/interval.rs index 8ecb15b389ed..2e153fdbe7ad 100644 --- a/third_party/rust/tokio/src/time/interval.rs +++ b/third_party/rust/tokio/src/time/interval.rs @@ -2,10 +2,10 @@ use crate::future::poll_fn; use crate::time::{sleep_until, Duration, Instant, Sleep}; use crate::util::trace; +use std::future::Future; use std::panic::Location; use std::pin::Pin; use std::task::{Context, Poll}; -use std::{convert::TryInto, future::Future}; /// Creates new [`Interval`] that yields with interval of `period`. The first /// tick completes immediately. The default [`MissedTickBehavior`] is @@ -207,6 +207,9 @@ pub enum MissedTickBehavior { /// # async fn main() { /// let mut interval = interval(Duration::from_millis(50)); /// + /// // First tick resolves immediately after creation + /// interval.tick().await; + /// /// task_that_takes_200_millis().await; /// // The `Interval` has missed a tick /// @@ -275,7 +278,7 @@ pub enum MissedTickBehavior { /// // 50ms after the call to `tick` up above. That is, in `tick`, when we /// // recognize that we missed a tick, we schedule the next tick to happen /// // 50ms (or whatever the `period` is) from right then, not from when - /// // were were *supposed* to tick + /// // were *supposed* to tick /// interval.tick().await; /// # } /// ``` @@ -384,7 +387,7 @@ impl Default for MissedTickBehavior { /// An `Interval` can be turned into a `Stream` with [`IntervalStream`]. /// /// [`IntervalStream`]: https://docs.rs/tokio-stream/latest/tokio_stream/wrappers/struct.IntervalStream.html -/// [`sleep`]: crate::time::sleep +/// [`sleep`]: crate::time::sleep() #[derive(Debug)] pub struct Interval { /// Future that completes the next time the `Interval` yields a value. @@ -420,6 +423,7 @@ impl Interval { /// let mut interval = time::interval(Duration::from_millis(10)); /// /// interval.tick().await; + /// // approximately 0ms have elapsed. The first tick completes immediately. /// interval.tick().await; /// interval.tick().await; /// @@ -478,7 +482,10 @@ impl Interval { timeout + self.period }; - self.delay.as_mut().reset(next); + // When we arrive here, the internal delay returned `Poll::Ready`. + // Reset the delay but do not register it. It should be registered with + // the next call to [`poll_tick`]. + self.delay.as_mut().reset_without_reregister(next); // Return the time when we were scheduled to tick Poll::Ready(timeout) diff --git a/third_party/rust/tokio/src/time/mod.rs b/third_party/rust/tokio/src/time/mod.rs index 281990ef9ac7..a1f27b839e92 100644 --- a/third_party/rust/tokio/src/time/mod.rs +++ b/third_party/rust/tokio/src/time/mod.rs @@ -82,17 +82,13 @@ //! ``` //! //! [`interval`]: crate::time::interval() +//! [`sleep`]: sleep() mod clock; pub(crate) use self::clock::Clock; #[cfg(feature = "test-util")] pub use clock::{advance, pause, resume}; -pub(crate) mod driver; - -#[doc(inline)] -pub use driver::sleep::{sleep, sleep_until, Sleep}; - pub mod error; mod instant; @@ -101,14 +97,13 @@ pub use self::instant::Instant; mod interval; pub use interval::{interval, interval_at, Interval, MissedTickBehavior}; +mod sleep; +pub use sleep::{sleep, sleep_until, Sleep}; + mod timeout; #[doc(inline)] pub use timeout::{timeout, timeout_at, Timeout}; -#[cfg(test)] -#[cfg(not(loom))] -mod tests; - // Re-export for convenience #[doc(no_inline)] pub use std::time::Duration; diff --git a/third_party/rust/tokio/src/time/driver/sleep.rs b/third_party/rust/tokio/src/time/sleep.rs similarity index 87% rename from third_party/rust/tokio/src/time/driver/sleep.rs rename to third_party/rust/tokio/src/time/sleep.rs index 7f27ef201f74..6c9b3379366c 100644 --- a/third_party/rust/tokio/src/time/driver/sleep.rs +++ b/third_party/rust/tokio/src/time/sleep.rs @@ -1,6 +1,4 @@ -#[cfg(all(tokio_unstable, feature = "tracing"))] -use crate::time::driver::ClockTime; -use crate::time::driver::{Handle, TimerEntry}; +use crate::runtime::time::TimerEntry; use crate::time::{error::Error, Duration, Instant}; use crate::util::trace; @@ -72,7 +70,9 @@ pub fn sleep_until(deadline: Instant) -> Sleep { /// /// No work is performed while awaiting on the sleep future to complete. `Sleep` /// operates at millisecond granularity and should not be used for tasks that -/// require high-resolution timers. +/// require high-resolution timers. The implementation is platform specific, +/// and some platforms (specifically Windows) will provide timers with a +/// larger resolution than 1 ms. /// /// To run something regularly on a schedule, see [`interval`]. /// @@ -235,33 +235,35 @@ pin_project! { cfg_trace! { #[derive(Debug)] struct Inner { - deadline: Instant, ctx: trace::AsyncOpTracingCtx, - time_source: ClockTime, } } cfg_not_trace! { #[derive(Debug)] struct Inner { - deadline: Instant, } } impl Sleep { #[cfg_attr(not(all(tokio_unstable, feature = "tracing")), allow(unused_variables))] + #[track_caller] pub(crate) fn new_timeout( deadline: Instant, location: Option<&'static Location<'static>>, ) -> Sleep { - let handle = Handle::current(); + use crate::runtime::scheduler; + + let handle = scheduler::Handle::current(); let entry = TimerEntry::new(&handle, deadline); #[cfg(all(tokio_unstable, feature = "tracing"))] let inner = { - let time_source = handle.time_source().clone(); + let clock = handle.driver().clock(); + let handle = &handle.driver().time(); + let time_source = handle.time_source(); let deadline_tick = time_source.deadline_to_tick(deadline); - let duration = deadline_tick.checked_sub(time_source.now()).unwrap_or(0); + let duration = deadline_tick.saturating_sub(time_source.now(clock)); let location = location.expect("should have location if tracing"); let resource_span = tracing::trace_span!( @@ -293,15 +295,11 @@ impl Sleep { resource_span, }; - Inner { - deadline, - ctx, - time_source, - } + Inner { ctx } }; #[cfg(not(all(tokio_unstable, feature = "tracing")))] - let inner = Inner { deadline }; + let inner = Inner {}; Sleep { inner, entry } } @@ -312,7 +310,7 @@ impl Sleep { /// Returns the instant at which the future will complete. pub fn deadline(&self) -> Instant { - self.inner.deadline + self.entry.deadline() } /// Returns `true` if `Sleep` has elapsed. @@ -355,10 +353,22 @@ impl Sleep { self.reset_inner(deadline) } + /// Resets the `Sleep` instance to a new deadline without reregistering it + /// to be woken up. + /// + /// Calling this function allows changing the instant at which the `Sleep` + /// future completes without having to create new associated state and + /// without having it registered. This is required in e.g. the + /// [crate::time::Interval] where we want to reset the internal [Sleep] + /// without having it wake up the last task that polled it. + pub(crate) fn reset_without_reregister(self: Pin<&mut Self>, deadline: Instant) { + let mut me = self.project(); + me.entry.as_mut().reset(deadline, false); + } + fn reset_inner(self: Pin<&mut Self>, deadline: Instant) { - let me = self.project(); - me.entry.reset(deadline); - (*me.inner).deadline = deadline; + let mut me = self.project(); + me.entry.as_mut().reset(deadline, true); #[cfg(all(tokio_unstable, feature = "tracing"))] { @@ -371,9 +381,11 @@ impl Sleep { tracing::trace_span!("runtime.resource.async_op.poll"); let duration = { - let now = me.inner.time_source.now(); - let deadline_tick = me.inner.time_source.deadline_to_tick(deadline); - deadline_tick.checked_sub(now).unwrap_or(0) + let clock = me.entry.clock(); + let time_source = me.entry.driver().time_source(); + let now = time_source.now(clock); + let deadline_tick = time_source.deadline_to_tick(deadline); + deadline_tick.saturating_sub(now) }; tracing::trace!( @@ -388,15 +400,17 @@ impl Sleep { fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll> { let me = self.project(); + ready!(crate::trace::trace_leaf(cx)); + // Keep track of task budget #[cfg(all(tokio_unstable, feature = "tracing"))] let coop = ready!(trace_poll_op!( "poll_elapsed", - crate::coop::poll_proceed(cx), + crate::runtime::coop::poll_proceed(cx), )); #[cfg(any(not(tokio_unstable), not(feature = "tracing")))] - let coop = ready!(crate::coop::poll_proceed(cx)); + let coop = ready!(crate::runtime::coop::poll_proceed(cx)); let result = me.entry.poll_elapsed(cx).map(move |r| { coop.made_progress(); diff --git a/third_party/rust/tokio/src/time/tests/mod.rs b/third_party/rust/tokio/src/time/tests/mod.rs deleted file mode 100644 index 35e1060aca0a..000000000000 --- a/third_party/rust/tokio/src/time/tests/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -mod test_sleep; - -use crate::time::{self, Instant}; -use std::time::Duration; - -fn assert_send() {} -fn assert_sync() {} - -#[test] -fn registration_is_send_and_sync() { - use crate::time::Sleep; - - assert_send::(); - assert_sync::(); -} - -#[test] -#[should_panic] -fn sleep_is_eager() { - let when = Instant::now() + Duration::from_millis(100); - let _ = time::sleep_until(when); -} diff --git a/third_party/rust/tokio/src/time/tests/test_sleep.rs b/third_party/rust/tokio/src/time/tests/test_sleep.rs deleted file mode 100644 index 77ca07e319ea..000000000000 --- a/third_party/rust/tokio/src/time/tests/test_sleep.rs +++ /dev/null @@ -1,443 +0,0 @@ -//use crate::time::driver::{Driver, Entry, Handle}; - -/* -macro_rules! poll { - ($e:expr) => { - $e.enter(|cx, e| e.poll_elapsed(cx)) - }; -} - -#[test] -fn frozen_utility_returns_correct_advanced_duration() { - let clock = Clock::new(); - clock.pause(); - let start = clock.now(); - - clock.advance(ms(10)); - assert_eq!(clock.now() - start, ms(10)); -} - -#[test] -fn immediate_sleep() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - let when = clock.now(); - let mut e = task::spawn(sleep_until(&handle, when)); - - assert_ready_ok!(poll!(e)); - - assert_ok!(driver.park_timeout(Duration::from_millis(1000))); - - // The time has not advanced. The `turn` completed immediately. - assert_eq!(clock.now() - start, ms(1000)); -} - -#[test] -fn delayed_sleep_level_0() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - for &i in &[1, 10, 60] { - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, start + ms(i))); - - // The sleep instance has not elapsed. - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(i)); - - assert_ready_ok!(poll!(e)); - } -} - -#[test] -fn sub_ms_delayed_sleep() { - let (mut driver, clock, handle) = setup(); - - for _ in 0..5 { - let deadline = clock.now() + ms(1) + Duration::new(0, 1); - - let mut e = task::spawn(sleep_until(&handle, deadline)); - - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_ready_ok!(poll!(e)); - - assert!(clock.now() >= deadline); - - clock.advance(Duration::new(0, 1)); - } -} - -#[test] -fn delayed_sleep_wrapping_level_0() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - assert_ok!(driver.park_timeout(ms(5))); - assert_eq!(clock.now() - start, ms(5)); - - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(60))); - - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(64)); - assert_pending!(poll!(e)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(65)); - - assert_ready_ok!(poll!(e)); -} - -#[test] -fn timer_wrapping_with_higher_levels() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Set sleep to hit level 1 - let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(64))); - assert_pending!(poll!(e1)); - - // Turn a bit - assert_ok!(driver.park_timeout(ms(5))); - - // Set timeout such that it will hit level 0, but wrap - let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(60))); - assert_pending!(poll!(e2)); - - // This should result in s1 firing - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(64)); - - assert_ready_ok!(poll!(e1)); - assert_pending!(poll!(e2)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(65)); - - assert_ready_ok!(poll!(e1)); -} - -#[test] -fn sleep_with_deadline_in_past() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create `Sleep` that elapsed immediately. - let mut e = task::spawn(sleep_until(&handle, clock.now() - ms(100))); - - // Even though the `Sleep` expires in the past, it is not ready yet - // because the timer must observe it. - assert_ready_ok!(poll!(e)); - - // Turn the timer, it runs for the elapsed time - assert_ok!(driver.park_timeout(ms(1000))); - - // The time has not advanced. The `turn` completed immediately. - assert_eq!(clock.now() - start, ms(1000)); -} - -#[test] -fn delayed_sleep_level_1() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(234))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer, this will wake up to cascade the timer down. - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(192)); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer again - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(234)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(234))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer with a smaller timeout than the cascade. - assert_ok!(driver.park_timeout(ms(100))); - assert_eq!(clock.now() - start, ms(100)); - - assert_pending!(poll!(e)); - - // Turn the timer, this will wake up to cascade the timer down. - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(192)); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer again - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(234)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); -} - -#[test] -fn concurrently_set_two_timers_second_one_shorter() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(500))); - let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(200))); - - // The sleep has not elapsed - assert_pending!(poll!(e1)); - assert_pending!(poll!(e2)); - - // Sleep until a cascade - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(192)); - - // Sleep until the second timer. - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(200)); - - // The shorter sleep fires - assert_ready_ok!(poll!(e2)); - assert_pending!(poll!(e1)); - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(448)); - - assert_pending!(poll!(e1)); - - // Turn again, this time the time will advance to the second sleep - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(500)); - - assert_ready_ok!(poll!(e1)); -} - -#[test] -fn short_sleep() { - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(1))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - // Turn the timer, but not enough time will go by. - assert_ok!(driver.park()); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); - - // The time has advanced to the point of the sleep elapsing. - assert_eq!(clock.now() - start, ms(1)); -} - -#[test] -fn sorta_long_sleep_until() { - const MIN_5: u64 = 5 * 60 * 1000; - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(MIN_5))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - let cascades = &[262_144, 262_144 + 9 * 4096, 262_144 + 9 * 4096 + 15 * 64]; - - for &elapsed in cascades { - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(elapsed)); - - assert_pending!(poll!(e)); - } - - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(MIN_5)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); -} - -#[test] -fn very_long_sleep() { - const MO_5: u64 = 5 * 30 * 24 * 60 * 60 * 1000; - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - // Create a `Sleep` that elapses in the future - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(MO_5))); - - // The sleep has not elapsed. - assert_pending!(poll!(e)); - - let cascades = &[ - 12_884_901_888, - 12_952_010_752, - 12_959_875_072, - 12_959_997_952, - ]; - - for &elapsed in cascades { - assert_ok!(driver.park()); - assert_eq!(clock.now() - start, ms(elapsed)); - - assert_pending!(poll!(e)); - } - - // Turn the timer, but not enough time will go by. - assert_ok!(driver.park()); - - // The time has advanced to the point of the sleep elapsing. - assert_eq!(clock.now() - start, ms(MO_5)); - - // The sleep has elapsed. - assert_ready_ok!(poll!(e)); -} - -#[test] -fn unpark_is_delayed() { - // A special park that will take much longer than the requested duration - struct MockPark(Clock); - - struct MockUnpark; - - impl Park for MockPark { - type Unpark = MockUnpark; - type Error = (); - - fn unpark(&self) -> Self::Unpark { - MockUnpark - } - - fn park(&mut self) -> Result<(), Self::Error> { - panic!("parking forever"); - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - assert_eq!(duration, ms(0)); - self.0.advance(ms(436)); - Ok(()) - } - - fn shutdown(&mut self) {} - } - - impl Unpark for MockUnpark { - fn unpark(&self) {} - } - - let clock = Clock::new(); - clock.pause(); - let start = clock.now(); - let mut driver = Driver::new(MockPark(clock.clone()), clock.clone()); - let handle = driver.handle(); - - let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(100))); - let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(101))); - let mut e3 = task::spawn(sleep_until(&handle, clock.now() + ms(200))); - - assert_pending!(poll!(e1)); - assert_pending!(poll!(e2)); - assert_pending!(poll!(e3)); - - assert_ok!(driver.park()); - - assert_eq!(clock.now() - start, ms(500)); - - assert_ready_ok!(poll!(e1)); - assert_ready_ok!(poll!(e2)); - assert_ready_ok!(poll!(e3)); -} - -#[test] -fn set_timeout_at_deadline_greater_than_max_timer() { - const YR_1: u64 = 365 * 24 * 60 * 60 * 1000; - const YR_5: u64 = 5 * YR_1; - - let (mut driver, clock, handle) = setup(); - let start = clock.now(); - - for _ in 0..5 { - assert_ok!(driver.park_timeout(ms(YR_1))); - } - - let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(1))); - assert_pending!(poll!(e)); - - assert_ok!(driver.park_timeout(ms(1000))); - assert_eq!(clock.now() - start, ms(YR_5) + ms(1)); - - assert_ready_ok!(poll!(e)); -} - -fn setup() -> (Driver, Clock, Handle) { - let clock = Clock::new(); - clock.pause(); - let driver = Driver::new(MockPark(clock.clone()), clock.clone()); - let handle = driver.handle(); - - (driver, clock, handle) -} - -fn sleep_until(handle: &Handle, when: Instant) -> Arc { - Entry::new(&handle, when, ms(0)) -} - -struct MockPark(Clock); - -struct MockUnpark; - -impl Park for MockPark { - type Unpark = MockUnpark; - type Error = (); - - fn unpark(&self) -> Self::Unpark { - MockUnpark - } - - fn park(&mut self) -> Result<(), Self::Error> { - panic!("parking forever"); - } - - fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> { - self.0.advance(duration); - Ok(()) - } - - fn shutdown(&mut self) {} -} - -impl Unpark for MockUnpark { - fn unpark(&self) {} -} - -fn ms(n: u64) -> Duration { - Duration::from_millis(n) -} -*/ diff --git a/third_party/rust/tokio/src/time/timeout.rs b/third_party/rust/tokio/src/time/timeout.rs index 4a93089e8e87..52ab9891c690 100644 --- a/third_party/rust/tokio/src/time/timeout.rs +++ b/third_party/rust/tokio/src/time/timeout.rs @@ -5,7 +5,7 @@ //! [`Timeout`]: struct@Timeout use crate::{ - coop, + runtime::coop, time::{error::Elapsed, sleep_until, Duration, Instant, Sleep}, util::trace, }; @@ -21,7 +21,22 @@ use std::task::{self, Poll}; /// value is returned. Otherwise, an error is returned and the future is /// canceled. /// -/// # Cancelation +/// Note that the timeout is checked before polling the future, so if the future +/// does not yield during execution then it is possible for the future to complete +/// and exceed the timeout _without_ returning an error. +/// +/// This function returns a future whose return type is [`Result`]``, where `T` is the +/// return type of the provided future. +/// +/// If the provided future completes immediately, then the future returned from +/// this function is guaranteed to complete immediately with an [`Ok`] variant +/// no matter the provided duration. +/// +/// [`Ok`]: std::result::Result::Ok +/// [`Result`]: std::result::Result +/// [`Elapsed`]: crate::time::error::Elapsed +/// +/// # Cancellation /// /// Cancelling a timeout is done by dropping the future. No additional cleanup /// or other work is required. @@ -68,9 +83,9 @@ use std::task::{self, Poll}; /// [`Builder::enable_time`]: crate::runtime::Builder::enable_time /// [`Builder::enable_all`]: crate::runtime::Builder::enable_all #[track_caller] -pub fn timeout(duration: Duration, future: T) -> Timeout +pub fn timeout(duration: Duration, future: F) -> Timeout where - T: Future, + F: Future, { let location = trace::caller_location(); @@ -87,7 +102,18 @@ where /// If the future completes before the instant is reached, then the completed /// value is returned. Otherwise, an error is returned. /// -/// # Cancelation +/// This function returns a future whose return type is [`Result`]``, where `T` is the +/// return type of the provided future. +/// +/// If the provided future completes immediately, then the future returned from +/// this function is guaranteed to complete immediately with an [`Ok`] variant +/// no matter the provided deadline. +/// +/// [`Ok`]: std::result::Result::Ok +/// [`Result`]: std::result::Result +/// [`Elapsed`]: crate::time::error::Elapsed +/// +/// # Cancellation /// /// Cancelling a timeout is done by dropping the future. No additional cleanup /// or other work is required. @@ -116,9 +142,9 @@ where /// } /// # } /// ``` -pub fn timeout_at(deadline: Instant, future: T) -> Timeout +pub fn timeout_at(deadline: Instant, future: F) -> Timeout where - T: Future, + F: Future, { let delay = sleep_until(deadline); diff --git a/third_party/rust/tokio/src/util/error.rs b/third_party/rust/tokio/src/util/error.rs index 8f252c0c9123..ebb27f6385fb 100644 --- a/third_party/rust/tokio/src/util/error.rs +++ b/third_party/rust/tokio/src/util/error.rs @@ -1,15 +1,14 @@ +// Some combinations of features may not use these constants. +#![cfg_attr(not(feature = "full"), allow(dead_code))] + /// Error string explaining that the Tokio context hasn't been instantiated. pub(crate) const CONTEXT_MISSING_ERROR: &str = "there is no reactor running, must be called from the context of a Tokio 1.x runtime"; -// some combinations of features might not use this -#[allow(dead_code)] /// Error string explaining that the Tokio context is shutting down and cannot drive timers. pub(crate) const RUNTIME_SHUTTING_DOWN_ERROR: &str = "A Tokio 1.x context was found, but it is being shutdown."; -// some combinations of features might not use this -#[allow(dead_code)] /// Error string explaining that the Tokio context is not available because the /// thread-local storing it has been destroyed. This usually only happens during /// destructors of other thread-locals. diff --git a/third_party/rust/tokio/src/util/idle_notified_set.rs b/third_party/rust/tokio/src/util/idle_notified_set.rs index 71f3a32a8530..19b81e28b15b 100644 --- a/third_party/rust/tokio/src/util/idle_notified_set.rs +++ b/third_party/rust/tokio/src/util/idle_notified_set.rs @@ -88,10 +88,6 @@ enum List { /// move it out from this entry to prevent it from getting leaked. (Since the /// two linked lists are emptied in the destructor of `IdleNotifiedSet`, the /// value should not be leaked.) -/// -/// This type is `#[repr(C)]` because its `linked_list::Link` implementation -/// requires that `pointers` is the first field. -#[repr(C)] struct ListEntry { /// The linked list pointers of the list this entry is in. pointers: linked_list::Pointers>, @@ -105,6 +101,14 @@ struct ListEntry { _pin: PhantomPinned, } +generate_addr_of_methods! { + impl ListEntry { + unsafe fn addr_of_pointers(self: NonNull) -> NonNull>> { + &self.pointers + } + } +} + // With mutable access to the `IdleNotifiedSet`, you can get mutable access to // the values. unsafe impl Send for IdleNotifiedSet {} @@ -417,7 +421,7 @@ impl Wake for ListEntry { // We move ourself to the notified list. let me = unsafe { // Safety: We just checked that we are in this particular list. - lock.idle.remove(NonNull::from(&**me)).unwrap() + lock.idle.remove(ListEntry::as_raw(me)).unwrap() }; lock.notified.push_front(me); @@ -453,11 +457,25 @@ unsafe impl linked_list::Link for ListEntry { unsafe fn pointers( target: NonNull>, ) -> NonNull>> { - // Safety: The pointers struct is the first field and ListEntry is - // `#[repr(C)]` so this cast is safe. - // - // We do this rather than doing a field access since `std::ptr::addr_of` - // is too new for our MSRV. - target.cast() + ListEntry::addr_of_pointers(target) + } +} + +#[cfg(all(test, not(loom)))] +mod tests { + use crate::runtime::Builder; + use crate::task::JoinSet; + + // A test that runs under miri. + // + // https://github.com/tokio-rs/tokio/pull/5693 + #[test] + fn join_set_test() { + let rt = Builder::new_current_thread().build().unwrap(); + + let mut set = JoinSet::new(); + set.spawn_on(futures::future::ready(()), rt.handle()); + + rt.block_on(set.join_next()).unwrap().unwrap(); } } diff --git a/third_party/rust/tokio/src/util/linked_list.rs b/third_party/rust/tokio/src/util/linked_list.rs index e6bdde68c7a6..ca6ef0e7b798 100644 --- a/third_party/rust/tokio/src/util/linked_list.rs +++ b/third_party/rust/tokio/src/util/linked_list.rs @@ -126,7 +126,7 @@ impl LinkedList { pub(crate) fn push_front(&mut self, val: L::Handle) { // The value should not be dropped, it is being inserted into the list let val = ManuallyDrop::new(val); - let ptr = L::as_raw(&*val); + let ptr = L::as_raw(&val); assert_ne!(self.head, Some(ptr)); unsafe { L::pointers(ptr).as_mut().set_next(self.head); @@ -178,8 +178,12 @@ impl LinkedList { /// /// # Safety /// - /// The caller **must** ensure that `node` is currently contained by - /// `self` or not contained by any other list. + /// The caller **must** ensure that exactly one of the following is true: + /// - `node` is currently contained by `self`, + /// - `node` is not contained by any list, + /// - `node` is currently contained by some other `GuardedLinkedList` **and** + /// the caller has an exclusive access to that list. This condition is + /// used by the linked list in `sync::Notify`. pub(crate) unsafe fn remove(&mut self, node: NonNull) -> Option { if let Some(prev) = L::pointers(node).as_ref().get_prev() { debug_assert_eq!(L::pointers(prev).as_ref().get_next(), Some(node)); @@ -224,6 +228,53 @@ impl fmt::Debug for LinkedList { } } +// ===== impl CountedLinkedList ==== + +// Delegates operations to the base LinkedList implementation, and adds a counter to the elements +// in the list. +pub(crate) struct CountedLinkedList { + list: LinkedList, + count: usize, +} + +impl CountedLinkedList { + pub(crate) fn new() -> CountedLinkedList { + CountedLinkedList { + list: LinkedList::new(), + count: 0, + } + } + + pub(crate) fn push_front(&mut self, val: L::Handle) { + self.list.push_front(val); + self.count += 1; + } + + pub(crate) fn pop_back(&mut self) -> Option { + let val = self.list.pop_back(); + if val.is_some() { + self.count -= 1; + } + val + } + + pub(crate) fn is_empty(&self) -> bool { + self.list.is_empty() + } + + pub(crate) unsafe fn remove(&mut self, node: NonNull) -> Option { + let val = self.list.remove(node); + if val.is_some() { + self.count -= 1; + } + val + } + + pub(crate) fn count(&self) -> usize { + self.count + } +} + #[cfg(any( feature = "fs", feature = "rt", @@ -290,6 +341,126 @@ cfg_io_readiness! { } } +cfg_taskdump! { + impl CountedLinkedList { + pub(crate) fn for_each(&mut self, f: F) + where + F: FnMut(&T::Handle), + { + self.list.for_each(f) + } + } + + impl LinkedList { + pub(crate) fn for_each(&mut self, mut f: F) + where + F: FnMut(&T::Handle), + { + use std::mem::ManuallyDrop; + + let mut next = self.head; + + while let Some(curr) = next { + unsafe { + let handle = ManuallyDrop::new(T::from_raw(curr)); + f(&handle); + next = T::pointers(curr).as_ref().get_next(); + } + } + } + } +} + +// ===== impl GuardedLinkedList ===== + +feature! { + #![any( + feature = "process", + feature = "sync", + feature = "rt", + feature = "signal", + )] + + /// An intrusive linked list, but instead of keeping pointers to the head + /// and tail nodes, it uses a special guard node linked with those nodes. + /// It means that the list is circular and every pointer of a node from + /// the list is not `None`, including pointers from the guard node. + /// + /// If a list is empty, then both pointers of the guard node are pointing + /// at the guard node itself. + pub(crate) struct GuardedLinkedList { + /// Pointer to the guard node. + guard: NonNull, + + /// Node type marker. + _marker: PhantomData<*const L>, + } + + impl>> LinkedList { + /// Turns a linked list into the guarded version by linking the guard node + /// with the head and tail nodes. Like with other nodes, you should guarantee + /// that the guard node is pinned in memory. + pub(crate) fn into_guarded(self, guard_handle: L::Handle) -> GuardedLinkedList { + // `guard_handle` is a NonNull pointer, we don't have to care about dropping it. + let guard = L::as_raw(&guard_handle); + + unsafe { + if let Some(head) = self.head { + debug_assert!(L::pointers(head).as_ref().get_prev().is_none()); + L::pointers(head).as_mut().set_prev(Some(guard)); + L::pointers(guard).as_mut().set_next(Some(head)); + + // The list is not empty, so the tail cannot be `None`. + let tail = self.tail.unwrap(); + debug_assert!(L::pointers(tail).as_ref().get_next().is_none()); + L::pointers(tail).as_mut().set_next(Some(guard)); + L::pointers(guard).as_mut().set_prev(Some(tail)); + } else { + // The list is empty. + L::pointers(guard).as_mut().set_prev(Some(guard)); + L::pointers(guard).as_mut().set_next(Some(guard)); + } + } + + GuardedLinkedList { guard, _marker: PhantomData } + } + } + + impl GuardedLinkedList { + fn tail(&self) -> Option> { + let tail_ptr = unsafe { + L::pointers(self.guard).as_ref().get_prev().unwrap() + }; + + // Compare the tail pointer with the address of the guard node itself. + // If the guard points at itself, then there are no other nodes and + // the list is considered empty. + if tail_ptr != self.guard { + Some(tail_ptr) + } else { + None + } + } + + /// Removes the last element from a list and returns it, or None if it is + /// empty. + pub(crate) fn pop_back(&mut self) -> Option { + unsafe { + let last = self.tail()?; + let before_last = L::pointers(last).as_ref().get_prev().unwrap(); + + L::pointers(self.guard).as_mut().set_prev(Some(before_last)); + L::pointers(before_last).as_mut().set_next(Some(self.guard)); + + L::pointers(last).as_mut().set_prev(None); + L::pointers(last).as_mut().set_next(None); + + Some(L::from_raw(last)) + } + } + } +} + // ===== impl Pointers ===== impl Pointers { @@ -352,9 +523,9 @@ impl fmt::Debug for Pointers { } } -#[cfg(test)] +#[cfg(any(test, fuzzing))] #[cfg(not(loom))] -mod tests { +pub(crate) mod tests { use super::*; use std::pin::Pin; @@ -413,6 +584,7 @@ mod tests { } } + #[cfg(test)] macro_rules! assert_clean { ($e:ident) => {{ assert!($e.pointers.get_next().is_none()); @@ -420,6 +592,7 @@ mod tests { }}; } + #[cfg(test)] macro_rules! assert_ptr_eq { ($a:expr, $b:expr) => {{ // Deal with mapping a Pin<&mut T> -> Option> @@ -623,30 +796,42 @@ mod tests { } } - #[cfg(not(target_arch = "wasm32"))] - proptest::proptest! { - #[test] - fn fuzz_linked_list(ops: Vec) { - run_fuzz(ops); + #[test] + fn count() { + let mut list = CountedLinkedList::<&Entry, <&Entry as Link>::Target>::new(); + assert_eq!(0, list.count()); + + let a = entry(5); + let b = entry(7); + list.push_front(a.as_ref()); + list.push_front(b.as_ref()); + assert_eq!(2, list.count()); + + list.pop_back(); + assert_eq!(1, list.count()); + + unsafe { + list.remove(ptr(&b)); } + assert_eq!(0, list.count()); } - fn run_fuzz(ops: Vec) { - use std::collections::VecDeque; - - #[derive(Debug)] + /// This is a fuzz test. You run it by entering `cargo fuzz run fuzz_linked_list` in CLI in `/tokio/` module. + #[cfg(fuzzing)] + pub fn fuzz_linked_list(ops: &[u8]) { enum Op { Push, Pop, Remove(usize), } + use std::collections::VecDeque; let ops = ops .iter() - .map(|i| match i % 3 { + .map(|i| match i % 3u8 { 0 => Op::Push, 1 => Op::Pop, - 2 => Op::Remove(i / 3), + 2 => Op::Remove((i / 3u8) as usize), _ => unreachable!(), }) .collect::>(); diff --git a/third_party/rust/tokio/src/util/markers.rs b/third_party/rust/tokio/src/util/markers.rs new file mode 100644 index 000000000000..7031fb6bcd19 --- /dev/null +++ b/third_party/rust/tokio/src/util/markers.rs @@ -0,0 +1,8 @@ +/// Marker for types that are `Sync` but not `Send` +pub(crate) struct SyncNotSend(*mut ()); + +unsafe impl Sync for SyncNotSend {} + +cfg_rt! { + pub(crate) struct NotSendOrSync(*mut ()); +} diff --git a/third_party/rust/tokio/src/util/memchr.rs b/third_party/rust/tokio/src/util/memchr.rs new file mode 100644 index 000000000000..44fd2da3719b --- /dev/null +++ b/third_party/rust/tokio/src/util/memchr.rs @@ -0,0 +1,74 @@ +//! Search for a byte in a byte array using libc. +//! +//! When nothing pulls in libc, then just use a trivial implementation. Note +//! that we only depend on libc on unix. + +#[cfg(not(all(unix, feature = "libc")))] +pub(crate) fn memchr(needle: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|val| needle == *val) +} + +#[cfg(all(unix, feature = "libc"))] +pub(crate) fn memchr(needle: u8, haystack: &[u8]) -> Option { + let start = haystack.as_ptr(); + + // SAFETY: `start` is valid for `haystack.len()` bytes. + let ptr = unsafe { libc::memchr(start.cast(), needle as _, haystack.len()) }; + + if ptr.is_null() { + None + } else { + Some(ptr as usize - start as usize) + } +} + +#[cfg(test)] +mod tests { + use super::memchr; + + #[test] + fn memchr_test() { + let haystack = b"123abc456\0\xffabc\n"; + + assert_eq!(memchr(b'1', haystack), Some(0)); + assert_eq!(memchr(b'2', haystack), Some(1)); + assert_eq!(memchr(b'3', haystack), Some(2)); + assert_eq!(memchr(b'4', haystack), Some(6)); + assert_eq!(memchr(b'5', haystack), Some(7)); + assert_eq!(memchr(b'6', haystack), Some(8)); + assert_eq!(memchr(b'7', haystack), None); + assert_eq!(memchr(b'a', haystack), Some(3)); + assert_eq!(memchr(b'b', haystack), Some(4)); + assert_eq!(memchr(b'c', haystack), Some(5)); + assert_eq!(memchr(b'd', haystack), None); + assert_eq!(memchr(b'A', haystack), None); + assert_eq!(memchr(0, haystack), Some(9)); + assert_eq!(memchr(0xff, haystack), Some(10)); + assert_eq!(memchr(0xfe, haystack), None); + assert_eq!(memchr(1, haystack), None); + assert_eq!(memchr(b'\n', haystack), Some(14)); + assert_eq!(memchr(b'\r', haystack), None); + } + + #[test] + fn memchr_all() { + let mut arr = Vec::new(); + for b in 0..=255 { + arr.push(b); + } + for b in 0..=255 { + assert_eq!(memchr(b, &arr), Some(b as usize)); + } + arr.reverse(); + for b in 0..=255 { + assert_eq!(memchr(b, &arr), Some(255 - b as usize)); + } + } + + #[test] + fn memchr_empty() { + for b in 0..=255 { + assert_eq!(memchr(b, b""), None); + } + } +} diff --git a/third_party/rust/tokio/src/util/mod.rs b/third_party/rust/tokio/src/util/mod.rs index 618f55438020..6a7d4b1037c0 100644 --- a/third_party/rust/tokio/src/util/mod.rs +++ b/third_party/rust/tokio/src/util/mod.rs @@ -6,6 +6,14 @@ cfg_io_driver! { #[cfg(feature = "rt")] pub(crate) mod atomic_cell; +#[cfg(any( + feature = "rt", + feature = "signal", + feature = "process", + tokio_no_const_mutex_new, +))] +pub(crate) mod once_cell; + #[cfg(any( // io driver uses `WakeList` directly feature = "net", @@ -40,14 +48,14 @@ pub(crate) use wake_list::WakeList; ))] pub(crate) mod linked_list; -#[cfg(any(feature = "rt-multi-thread", feature = "macros"))] -mod rand; +#[cfg(any(feature = "rt", feature = "macros"))] +pub(crate) mod rand; cfg_rt! { - cfg_unstable! { - mod idle_notified_set; - pub(crate) use idle_notified_set::IdleNotifiedSet; - } + mod idle_notified_set; + pub(crate) use idle_notified_set::IdleNotifiedSet; + + pub(crate) use self::rand::RngSeedGenerator; mod wake; pub(crate) use wake::WakerRef; @@ -56,28 +64,20 @@ cfg_rt! { mod sync_wrapper; pub(crate) use sync_wrapper::SyncWrapper; - mod vec_deque_cell; - pub(crate) use vec_deque_cell::VecDequeCell; + mod rc_cell; + pub(crate) use rc_cell::RcCell; } cfg_rt_multi_thread! { - pub(crate) use self::rand::FastRand; - mod try_lock; pub(crate) use try_lock::TryLock; } pub(crate) mod trace; -#[cfg(any(feature = "macros"))] -#[cfg_attr(not(feature = "macros"), allow(unreachable_pub))] -pub use self::rand::thread_rng_n; - -#[cfg(any( - feature = "rt", - feature = "time", - feature = "net", - feature = "process", - all(unix, feature = "signal") -))] pub(crate) mod error; + +#[cfg(feature = "io-util")] +pub(crate) mod memchr; + +pub(crate) mod markers; diff --git a/third_party/rust/tokio/src/util/once_cell.rs b/third_party/rust/tokio/src/util/once_cell.rs new file mode 100644 index 000000000000..71fc00758e18 --- /dev/null +++ b/third_party/rust/tokio/src/util/once_cell.rs @@ -0,0 +1,70 @@ +#![allow(dead_code)] +use std::cell::UnsafeCell; +use std::mem::MaybeUninit; +use std::sync::Once; + +pub(crate) struct OnceCell { + once: Once, + value: UnsafeCell>, +} + +unsafe impl Send for OnceCell {} +unsafe impl Sync for OnceCell {} + +impl OnceCell { + pub(crate) const fn new() -> Self { + Self { + once: Once::new(), + value: UnsafeCell::new(MaybeUninit::uninit()), + } + } + + /// Get the value inside this cell, initializing it using the provided + /// function if necessary. + /// + /// If the `init` closure panics, then the `OnceCell` is poisoned and all + /// future calls to `get` will panic. + #[inline] + pub(crate) fn get(&self, init: impl FnOnce() -> T) -> &T { + if !self.once.is_completed() { + self.do_init(init); + } + + // Safety: The `std::sync::Once` guarantees that we can only reach this + // line if a `call_once` closure has been run exactly once and without + // panicking. Thus, the value is not uninitialized. + // + // There is also no race because the only `&self` method that modifies + // `value` is `do_init`, but if the `call_once` closure is still + // running, then no thread has gotten past the `call_once`. + unsafe { &*(self.value.get() as *const T) } + } + + #[cold] + fn do_init(&self, init: impl FnOnce() -> T) { + let value_ptr = self.value.get() as *mut T; + + self.once.call_once(|| { + let set_to = init(); + + // Safety: The `std::sync::Once` guarantees that this initialization + // will run at most once, and that no thread can get past the + // `call_once` until it has run exactly once. Thus, we have + // exclusive access to `value`. + unsafe { + std::ptr::write(value_ptr, set_to); + } + }); + } +} + +impl Drop for OnceCell { + fn drop(&mut self) { + if self.once.is_completed() { + let value_ptr = self.value.get() as *mut T; + unsafe { + std::ptr::drop_in_place(value_ptr); + } + } + } +} diff --git a/third_party/rust/tokio/src/util/pad.rs b/third_party/rust/tokio/src/util/pad.rs deleted file mode 100644 index bf0913ca8537..000000000000 --- a/third_party/rust/tokio/src/util/pad.rs +++ /dev/null @@ -1,52 +0,0 @@ -use core::fmt; -use core::ops::{Deref, DerefMut}; - -#[derive(Clone, Copy, Default, Hash, PartialEq, Eq)] -// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache -// lines at a time, so we have to align to 128 bytes rather than 64. -// -// Sources: -// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf -// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107 -#[cfg_attr(target_arch = "x86_64", repr(align(128)))] -#[cfg_attr(not(target_arch = "x86_64"), repr(align(64)))] -pub(crate) struct CachePadded { - value: T, -} - -unsafe impl Send for CachePadded {} -unsafe impl Sync for CachePadded {} - -impl CachePadded { - pub(crate) fn new(t: T) -> CachePadded { - CachePadded:: { value: t } - } -} - -impl Deref for CachePadded { - type Target = T; - - fn deref(&self) -> &T { - &self.value - } -} - -impl DerefMut for CachePadded { - fn deref_mut(&mut self) -> &mut T { - &mut self.value - } -} - -impl fmt::Debug for CachePadded { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("CachePadded") - .field("value", &self.value) - .finish() - } -} - -impl From for CachePadded { - fn from(t: T) -> Self { - CachePadded::new(t) - } -} diff --git a/third_party/rust/tokio/src/util/rand.rs b/third_party/rust/tokio/src/util/rand.rs index 6b19c8be9578..d96c8d37e0a9 100644 --- a/third_party/rust/tokio/src/util/rand.rs +++ b/third_party/rust/tokio/src/util/rand.rs @@ -1,4 +1,22 @@ -use std::cell::Cell; +cfg_rt! { + mod rt; + pub(crate) use rt::RngSeedGenerator; + + cfg_unstable! { + mod rt_unstable; + } +} + +/// A seed for random number generation. +/// +/// In order to make certain functions within a runtime deterministic, a seed +/// can be specified at the time of creation. +#[allow(unreachable_pub)] +#[derive(Clone, Debug)] +pub struct RngSeed { + s: u32, + r: u32, +} /// Fast random number generate. /// @@ -7,15 +25,19 @@ use std::cell::Cell; /// Xorshift paper: /// This generator passes the SmallCrush suite, part of TestU01 framework: /// -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] pub(crate) struct FastRand { - one: Cell, - two: Cell, + one: u32, + two: u32, } -impl FastRand { - /// Initializes a new, thread-local, fast random number generator. - pub(crate) fn new(seed: u64) -> FastRand { +impl RngSeed { + /// Creates a random seed using loom internally. + pub(crate) fn new() -> Self { + Self::from_u64(crate::loom::rand::seed()) + } + + fn from_u64(seed: u64) -> Self { let one = (seed >> 32) as u32; let mut two = seed as u32; @@ -24,41 +46,50 @@ impl FastRand { two = 1; } + Self::from_pair(one, two) + } + + fn from_pair(s: u32, r: u32) -> Self { + Self { s, r } + } +} + +impl FastRand { + /// Initialize a new fast random number generator using the default source of entropy. + pub(crate) fn new() -> FastRand { + FastRand::from_seed(RngSeed::new()) + } + + /// Initializes a new, thread-local, fast random number generator. + pub(crate) fn from_seed(seed: RngSeed) -> FastRand { FastRand { - one: Cell::new(one), - two: Cell::new(two), + one: seed.s, + two: seed.r, } } - pub(crate) fn fastrand_n(&self, n: u32) -> u32 { + #[cfg(any( + feature = "macros", + feature = "rt-multi-thread", + all(feature = "sync", feature = "rt") + ))] + pub(crate) fn fastrand_n(&mut self, n: u32) -> u32 { // This is similar to fastrand() % n, but faster. // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ let mul = (self.fastrand() as u64).wrapping_mul(n as u64); (mul >> 32) as u32 } - fn fastrand(&self) -> u32 { - let mut s1 = self.one.get(); - let s0 = self.two.get(); + fn fastrand(&mut self) -> u32 { + let mut s1 = self.one; + let s0 = self.two; s1 ^= s1 << 17; s1 = s1 ^ s0 ^ s1 >> 7 ^ s0 >> 16; - self.one.set(s0); - self.two.set(s1); + self.one = s0; + self.two = s1; s0.wrapping_add(s1) } } - -// Used by the select macro and `StreamMap` -#[cfg(any(feature = "macros"))] -#[doc(hidden)] -#[cfg_attr(not(feature = "macros"), allow(unreachable_pub))] -pub fn thread_rng_n(n: u32) -> u32 { - thread_local! { - static THREAD_RNG: FastRand = FastRand::new(crate::loom::rand::seed()); - } - - THREAD_RNG.with(|rng| rng.fastrand_n(n)) -} diff --git a/third_party/rust/tokio/src/util/rand/rt.rs b/third_party/rust/tokio/src/util/rand/rt.rs new file mode 100644 index 000000000000..6584ba4f215b --- /dev/null +++ b/third_party/rust/tokio/src/util/rand/rt.rs @@ -0,0 +1,61 @@ +use super::{FastRand, RngSeed}; + +use std::sync::Mutex; + +/// A deterministic generator for seeds (and other generators). +/// +/// Given the same initial seed, the generator will output the same sequence of seeds. +/// +/// Since the seed generator will be kept in a runtime handle, we need to wrap `FastRand` +/// in a Mutex to make it thread safe. Different to the `FastRand` that we keep in a +/// thread local store, the expectation is that seed generation will not need to happen +/// very frequently, so the cost of the mutex should be minimal. +#[derive(Debug)] +pub(crate) struct RngSeedGenerator { + /// Internal state for the seed generator. We keep it in a Mutex so that we can safely + /// use it across multiple threads. + state: Mutex, +} + +impl RngSeedGenerator { + /// Returns a new generator from the provided seed. + pub(crate) fn new(seed: RngSeed) -> Self { + Self { + state: Mutex::new(FastRand::from_seed(seed)), + } + } + + /// Returns the next seed in the sequence. + pub(crate) fn next_seed(&self) -> RngSeed { + let mut rng = self + .state + .lock() + .expect("RNG seed generator is internally corrupt"); + + let s = rng.fastrand(); + let r = rng.fastrand(); + + RngSeed::from_pair(s, r) + } + + /// Directly creates a generator using the next seed. + pub(crate) fn next_generator(&self) -> Self { + RngSeedGenerator::new(self.next_seed()) + } +} + +impl FastRand { + /// Replaces the state of the random number generator with the provided seed, returning + /// the seed that represents the previous state of the random number generator. + /// + /// The random number generator will become equivalent to one created with + /// the same seed. + pub(crate) fn replace_seed(&mut self, seed: RngSeed) -> RngSeed { + let old_seed = RngSeed::from_pair(self.one, self.two); + + self.one = seed.s; + self.two = seed.r; + + old_seed + } +} diff --git a/third_party/rust/tokio/src/util/rand/rt_unstable.rs b/third_party/rust/tokio/src/util/rand/rt_unstable.rs new file mode 100644 index 000000000000..3a8613eb0f96 --- /dev/null +++ b/third_party/rust/tokio/src/util/rand/rt_unstable.rs @@ -0,0 +1,20 @@ +use super::RngSeed; + +use std::collections::hash_map::DefaultHasher; +use std::hash::Hasher; + +impl RngSeed { + /// Generates a seed from the provided byte slice. + /// + /// # Example + /// + /// ``` + /// # use tokio::runtime::RngSeed; + /// let seed = RngSeed::from_bytes(b"make me a seed"); + /// ``` + pub fn from_bytes(bytes: &[u8]) -> Self { + let mut hasher = DefaultHasher::default(); + hasher.write(bytes); + Self::from_u64(hasher.finish()) + } +} diff --git a/third_party/rust/tokio/src/util/rc_cell.rs b/third_party/rust/tokio/src/util/rc_cell.rs new file mode 100644 index 000000000000..447249268487 --- /dev/null +++ b/third_party/rust/tokio/src/util/rc_cell.rs @@ -0,0 +1,57 @@ +use crate::loom::cell::UnsafeCell; + +use std::rc::Rc; + +/// This is exactly like `Cell>>`, except that it provides a `get` +/// method even though `Rc` is not `Copy`. +pub(crate) struct RcCell { + inner: UnsafeCell>>, +} + +impl RcCell { + #[cfg(not(all(loom, test)))] + pub(crate) const fn new() -> Self { + Self { + inner: UnsafeCell::new(None), + } + } + + // The UnsafeCell in loom does not have a const `new` fn. + #[cfg(all(loom, test))] + pub(crate) fn new() -> Self { + Self { + inner: UnsafeCell::new(None), + } + } + + /// Safety: This method may not be called recursively. + #[inline] + unsafe fn with_inner(&self, f: F) -> R + where + F: FnOnce(&mut Option>) -> R, + { + // safety: This type is not Sync, so concurrent calls of this method + // cannot happen. Furthermore, the caller guarantees that the method is + // not called recursively. Finally, this is the only place that can + // create mutable references to the inner Rc. This ensures that any + // mutable references created here are exclusive. + self.inner.with_mut(|ptr| f(&mut *ptr)) + } + + pub(crate) fn get(&self) -> Option> { + // safety: The `Rc::clone` method will not call any unknown user-code, + // so it will not result in a recursive call to `with_inner`. + unsafe { self.with_inner(|rc| rc.clone()) } + } + + pub(crate) fn replace(&self, val: Option>) -> Option> { + // safety: No destructors or other unknown user-code will run inside the + // `with_inner` call, so no recursive call to `with_inner` can happen. + unsafe { self.with_inner(|rc| std::mem::replace(rc, val)) } + } + + pub(crate) fn set(&self, val: Option>) { + let old = self.replace(val); + drop(old); + } +} diff --git a/third_party/rust/tokio/src/util/slab.rs b/third_party/rust/tokio/src/util/slab.rs index 214fa08dc899..cc1208ebef42 100644 --- a/third_party/rust/tokio/src/util/slab.rs +++ b/third_party/rust/tokio/src/util/slab.rs @@ -551,17 +551,16 @@ impl Slots { fn index_for(&self, slot: *const Value) -> usize { use std::mem; - let base = &self.slots[0] as *const _ as usize; - - assert!(base != 0, "page is unallocated"); + assert_ne!(self.slots.capacity(), 0, "page is unallocated"); + let base = self.slots.as_ptr() as usize; let slot = slot as usize; let width = mem::size_of::>(); assert!(slot >= base, "unexpected pointer"); let idx = (slot - base) / width; - assert!(idx < self.slots.len() as usize); + assert!(idx < self.slots.len()); idx } diff --git a/third_party/rust/tokio/src/util/trace.rs b/third_party/rust/tokio/src/util/trace.rs index 6080e2358ae6..76e8a6cbf552 100644 --- a/third_party/rust/tokio/src/util/trace.rs +++ b/third_party/rust/tokio/src/util/trace.rs @@ -10,7 +10,7 @@ cfg_trace! { #[inline] #[track_caller] - pub(crate) fn task(task: F, kind: &'static str, name: Option<&str>) -> Instrumented { + pub(crate) fn task(task: F, kind: &'static str, name: Option<&str>, id: u64) -> Instrumented { use tracing::instrument::Instrument; let location = std::panic::Location::caller(); let span = tracing::trace_span!( @@ -18,6 +18,7 @@ cfg_trace! { "runtime.spawn", %kind, task.name = %name.unwrap_or_default(), + task.id = id, loc.file = location.file(), loc.line = location.line(), loc.col = location.column(), @@ -91,7 +92,7 @@ cfg_time! { cfg_not_trace! { cfg_rt! { #[inline] - pub(crate) fn task(task: F, _: &'static str, _name: Option<&str>) -> F { + pub(crate) fn task(task: F, _: &'static str, _name: Option<&str>, _: u64) -> F { // nop task } diff --git a/third_party/rust/tokio/src/util/vec_deque_cell.rs b/third_party/rust/tokio/src/util/vec_deque_cell.rs deleted file mode 100644 index b4e124c1519e..000000000000 --- a/third_party/rust/tokio/src/util/vec_deque_cell.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::loom::cell::UnsafeCell; - -use std::collections::VecDeque; -use std::marker::PhantomData; - -/// This type is like VecDeque, except that it is not Sync and can be modified -/// through immutable references. -pub(crate) struct VecDequeCell { - inner: UnsafeCell>, - _not_sync: PhantomData<*const ()>, -} - -// This is Send for the same reasons that RefCell> is Send. -unsafe impl Send for VecDequeCell {} - -impl VecDequeCell { - pub(crate) fn with_capacity(cap: usize) -> Self { - Self { - inner: UnsafeCell::new(VecDeque::with_capacity(cap)), - _not_sync: PhantomData, - } - } - - /// Safety: This method may not be called recursively. - #[inline] - unsafe fn with_inner(&self, f: F) -> R - where - F: FnOnce(&mut VecDeque) -> R, - { - // safety: This type is not Sync, so concurrent calls of this method - // cannot happen. Furthermore, the caller guarantees that the method is - // not called recursively. Finally, this is the only place that can - // create mutable references to the inner VecDeque. This ensures that - // any mutable references created here are exclusive. - self.inner.with_mut(|ptr| f(&mut *ptr)) - } - - pub(crate) fn pop_front(&self) -> Option { - unsafe { self.with_inner(VecDeque::pop_front) } - } - - pub(crate) fn push_back(&self, item: T) { - unsafe { - self.with_inner(|inner| inner.push_back(item)); - } - } - - /// Replaces the inner VecDeque with an empty VecDeque and return the current - /// contents. - pub(crate) fn take(&self) -> VecDeque { - unsafe { self.with_inner(|inner| std::mem::take(inner)) } - } -} diff --git a/third_party/rust/tokio/tests/_require_full.rs b/third_party/rust/tokio/tests/_require_full.rs index 3abda2629cf0..4b9698afeddc 100644 --- a/third_party/rust/tokio/tests/_require_full.rs +++ b/third_party/rust/tokio/tests/_require_full.rs @@ -1,2 +1,8 @@ -#![cfg(not(any(feature = "full", target_arch = "wasm32")))] +#[cfg(not(any(feature = "full", tokio_wasm)))] compile_error!("run main Tokio tests with `--features full`"); + +// CI sets `--cfg tokio_no_parking_lot` when trying to run tests with +// `parking_lot` disabled. This check prevents "silent failure" if `parking_lot` +// accidentally gets enabled. +#[cfg(all(tokio_no_parking_lot, feature = "parking_lot"))] +compile_error!("parking_lot feature enabled when it should not be"); diff --git a/third_party/rust/tokio/tests/async_send_sync.rs b/third_party/rust/tokio/tests/async_send_sync.rs index ea4445a27f11..32c03a54be2b 100644 --- a/third_party/rust/tokio/tests/async_send_sync.rs +++ b/third_party/rust/tokio/tests/async_send_sync.rs @@ -127,70 +127,111 @@ macro_rules! assert_value { }; } -assert_value!(tokio::fs::DirBuilder: Send & Sync & Unpin); -assert_value!(tokio::fs::DirEntry: Send & Sync & Unpin); -assert_value!(tokio::fs::File: Send & Sync & Unpin); -assert_value!(tokio::fs::OpenOptions: Send & Sync & Unpin); -assert_value!(tokio::fs::ReadDir: Send & Sync & Unpin); +macro_rules! cfg_not_wasi { + ($($item:item)*) => { + $( + #[cfg(not(tokio_wasi))] + $item + )* + } +} -async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::metadata(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read_link(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::write(&str, Vec): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::DirBuilder::create(_, &str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::open(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::create(&str): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync & !Unpin); +// Manually re-implementation of `async_assert_fn` for `poll_fn`. The macro +// doesn't work for this particular case because constructing the closure +// is too complicated. +const _: fn() = || { + let pinned = std::marker::PhantomPinned; + let f = tokio::macros::support::poll_fn(move |_| { + // Use `pinned` to take ownership of it. + let _ = &pinned; + std::task::Poll::Pending::<()> + }); + require_send(&f); + require_sync(&f); + AmbiguousIfUnpin::some_item(&f); +}; + +cfg_not_wasi! { + mod fs { + use super::*; + assert_value!(tokio::fs::DirBuilder: Send & Sync & Unpin); + assert_value!(tokio::fs::DirEntry: Send & Sync & Unpin); + assert_value!(tokio::fs::File: Send & Sync & Unpin); + assert_value!(tokio::fs::OpenOptions: Send & Sync & Unpin); + assert_value!(tokio::fs::ReadDir: Send & Sync & Unpin); + + async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::metadata(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read_link(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::write(&str, Vec): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::DirBuilder::create(_, &str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::open(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::create(&str): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync & !Unpin); + async_assert_fn!( + tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync & !Unpin + ); + } +} + +cfg_not_wasi! { + assert_value!(tokio::net::TcpSocket: Send & Sync & Unpin); + async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync & !Unpin); +} assert_value!(tokio::net::TcpListener: Send & Sync & Unpin); -assert_value!(tokio::net::TcpSocket: Send & Sync & Unpin); assert_value!(tokio::net::TcpStream: Send & Sync & Unpin); -assert_value!(tokio::net::UdpSocket: Send & Sync & Unpin); assert_value!(tokio::net::tcp::OwnedReadHalf: Send & Sync & Unpin); assert_value!(tokio::net::tcp::OwnedWriteHalf: Send & Sync & Unpin); assert_value!(tokio::net::tcp::ReadHalf<'_>: Send & Sync & Unpin); assert_value!(tokio::net::tcp::ReuniteError: Send & Sync & Unpin); assert_value!(tokio::net::tcp::WriteHalf<'_>: Send & Sync & Unpin); async_assert_fn!(tokio::net::TcpListener::accept(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::peek(_, &mut [u8]): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::readable(_): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::ready(_, tokio::io::Interest): Send & Sync & !Unpin); async_assert_fn!(tokio::net::TcpStream::writable(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::peek_from(_, &mut [u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::readable(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::ready(_, tokio::io::Interest): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync & !Unpin); -async_assert_fn!(tokio::net::UdpSocket::writable(_): Send & Sync & !Unpin); + +// Wasi does not support UDP +cfg_not_wasi! { + mod udp_socket { + use super::*; + assert_value!(tokio::net::UdpSocket: Send & Sync & Unpin); + async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::peek_from(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::readable(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync & !Unpin); + async_assert_fn!(tokio::net::UdpSocket::writable(_): Send & Sync & !Unpin); + } +} async_assert_fn!(tokio::net::lookup_host(SocketAddr): Send & Sync & !Unpin); async_assert_fn!(tokio::net::tcp::ReadHalf::peek(_, &mut [u8]): Send & Sync & !Unpin); @@ -222,6 +263,19 @@ mod unix_datagram { async_assert_fn!(UnixStream::writable(_): Send & Sync & !Unpin); } +#[cfg(unix)] +mod unix_pipe { + use super::*; + use tokio::net::unix::pipe::*; + assert_value!(OpenOptions: Send & Sync & Unpin); + assert_value!(Receiver: Send & Sync & Unpin); + assert_value!(Sender: Send & Sync & Unpin); + async_assert_fn!(Receiver::readable(_): Send & Sync & !Unpin); + async_assert_fn!(Receiver::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(Sender::ready(_, tokio::io::Interest): Send & Sync & !Unpin); + async_assert_fn!(Sender::writable(_): Send & Sync & !Unpin); +} + #[cfg(windows)] mod windows_named_pipe { use super::*; @@ -242,16 +296,22 @@ mod windows_named_pipe { async_assert_fn!(NamedPipeServer::writable(_): Send & Sync & !Unpin); } -assert_value!(tokio::process::Child: Send & Sync & Unpin); -assert_value!(tokio::process::ChildStderr: Send & Sync & Unpin); -assert_value!(tokio::process::ChildStdin: Send & Sync & Unpin); -assert_value!(tokio::process::ChildStdout: Send & Sync & Unpin); -assert_value!(tokio::process::Command: Send & Sync & Unpin); -async_assert_fn!(tokio::process::Child::kill(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::process::Child::wait(_): Send & Sync & !Unpin); -async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync & !Unpin); +cfg_not_wasi! { + mod test_process { + use super::*; + assert_value!(tokio::process::Child: Send & Sync & Unpin); + assert_value!(tokio::process::ChildStderr: Send & Sync & Unpin); + assert_value!(tokio::process::ChildStdin: Send & Sync & Unpin); + assert_value!(tokio::process::ChildStdout: Send & Sync & Unpin); + assert_value!(tokio::process::Command: Send & Sync & Unpin); + async_assert_fn!(tokio::process::Child::kill(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::process::Child::wait(_): Send & Sync & !Unpin); + async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync & !Unpin); + } + + async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync & !Unpin); +} -async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync & !Unpin); #[cfg(unix)] mod unix_signal { use super::*; @@ -287,6 +347,15 @@ assert_value!(tokio::sync::OnceCell: Send & Sync & Unpin); assert_value!(tokio::sync::OwnedMutexGuard: !Send & !Sync & Unpin); assert_value!(tokio::sync::OwnedMutexGuard: Send & !Sync & Unpin); assert_value!(tokio::sync::OwnedMutexGuard: Send & Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: !Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: Send & !Sync & Unpin); +assert_value!(tokio::sync::OwnedMappedMutexGuard: Send & Sync & Unpin); assert_value!(tokio::sync::OwnedRwLockMappedWriteGuard: !Send & !Sync & Unpin); assert_value!(tokio::sync::OwnedRwLockMappedWriteGuard: !Send & !Sync & Unpin); assert_value!(tokio::sync::OwnedRwLockMappedWriteGuard: Send & Sync & Unpin); @@ -362,6 +431,14 @@ assert_value!(tokio::sync::watch::Ref<'_, YY>: !Send & Sync & Unpin); assert_value!(tokio::sync::watch::Sender: !Send & !Sync & Unpin); assert_value!(tokio::sync::watch::Sender: !Send & !Sync & Unpin); assert_value!(tokio::sync::watch::Sender: Send & Sync & Unpin); +assert_value!(tokio::task::JoinError: Send & Sync & Unpin); +assert_value!(tokio::task::JoinHandle: !Send & !Sync & Unpin); +assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); +assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); +assert_value!(tokio::task::JoinSet: !Send & !Sync & Unpin); +assert_value!(tokio::task::JoinSet: Send & Sync & Unpin); +assert_value!(tokio::task::JoinSet: Send & Sync & Unpin); +assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin); async_assert_fn!(tokio::sync::Barrier::wait(_): Send & Sync & !Unpin); async_assert_fn!(tokio::sync::Mutex::lock(_): !Send & !Sync & !Unpin); async_assert_fn!(tokio::sync::Mutex::lock_owned(_): !Send & !Sync & !Unpin); @@ -434,6 +511,12 @@ async_assert_fn!(tokio::sync::watch::Receiver::changed(_): Send & Sync & !Un async_assert_fn!(tokio::sync::watch::Sender::closed(_): !Send & !Sync & !Unpin); async_assert_fn!(tokio::sync::watch::Sender::closed(_): !Send & !Sync & !Unpin); async_assert_fn!(tokio::sync::watch::Sender::closed(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet>::join_next(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet>::shutdown(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet>::join_next(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet>::shutdown(_): !Send & !Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet::join_next(_): Send & Sync & !Unpin); +async_assert_fn!(tokio::task::JoinSet::shutdown(_): Send & Sync & !Unpin); async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFuture<()>): !Send & !Sync & !Unpin); async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFutureSend<()>): Send & !Sync & !Unpin); async_assert_fn!(tokio::task::LocalKey>::scope(_, Cell, BoxFutureSync<()>): Send & !Sync & !Unpin); @@ -447,27 +530,9 @@ async_assert_fn!(tokio::task::LocalSet::run_until(_, BoxFutureSync<()>): !Send & async_assert_fn!(tokio::task::unconstrained(BoxFuture<()>): !Send & !Sync & Unpin); async_assert_fn!(tokio::task::unconstrained(BoxFutureSend<()>): Send & !Sync & Unpin); async_assert_fn!(tokio::task::unconstrained(BoxFutureSync<()>): Send & Sync & Unpin); -assert_value!(tokio::task::JoinError: Send & Sync & Unpin); -assert_value!(tokio::task::JoinHandle: !Send & !Sync & Unpin); -assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); -assert_value!(tokio::task::JoinHandle: Send & Sync & Unpin); -#[cfg(tokio_unstable)] -mod unstable { - use super::*; - async_assert_fn!(tokio::task::JoinSet>::join_one(_): Send & Sync & !Unpin); - async_assert_fn!(tokio::task::JoinSet>::shutdown(_): Send & Sync & !Unpin); - async_assert_fn!(tokio::task::JoinSet>::join_one(_): !Send & !Sync & !Unpin); - async_assert_fn!(tokio::task::JoinSet>::shutdown(_): !Send & !Sync & !Unpin); - async_assert_fn!(tokio::task::JoinSet::join_one(_): Send & Sync & !Unpin); - async_assert_fn!(tokio::task::JoinSet::shutdown(_): Send & Sync & !Unpin); - assert_value!(tokio::task::JoinSet: Send & Sync & Unpin); - assert_value!(tokio::task::JoinSet: Send & Sync & Unpin); - assert_value!(tokio::task::JoinSet: !Send & !Sync & Unpin); - assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin); -} assert_value!(tokio::runtime::Builder: Send & Sync & Unpin); -assert_value!(tokio::runtime::EnterGuard<'_>: Send & Sync & Unpin); +assert_value!(tokio::runtime::EnterGuard<'_>: !Send & Sync & Unpin); assert_value!(tokio::runtime::Handle: Send & Sync & Unpin); assert_value!(tokio::runtime::Runtime: Send & Sync & Unpin); diff --git a/third_party/rust/tokio/tests/buffered.rs b/third_party/rust/tokio/tests/buffered.rs index 98b6d5f3121c..4251c3fcc0b3 100644 --- a/third_party/rust/tokio/tests/buffered.rs +++ b/third_party/rust/tokio/tests/buffered.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind() use tokio::net::TcpListener; use tokio_test::assert_ok; @@ -18,10 +18,10 @@ async fn echo_server() { let msg = "foo bar baz"; let t = thread::spawn(move || { - let mut s = assert_ok!(TcpStream::connect(&addr)); + let mut s = assert_ok!(TcpStream::connect(addr)); let t2 = thread::spawn(move || { - let mut s = assert_ok!(TcpStream::connect(&addr)); + let mut s = assert_ok!(TcpStream::connect(addr)); let mut b = vec![0; msg.len() * N]; assert_ok!(s.read_exact(&mut b)); b diff --git a/third_party/rust/tokio/tests/dump.rs b/third_party/rust/tokio/tests/dump.rs new file mode 100644 index 000000000000..658ee4b9bfcd --- /dev/null +++ b/third_party/rust/tokio/tests/dump.rs @@ -0,0 +1,99 @@ +#![cfg(all( + tokio_unstable, + tokio_taskdump, + target_os = "linux", + any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64") +))] + +use std::hint::black_box; +use tokio::runtime::{self, Handle}; + +#[inline(never)] +async fn a() { + black_box(b()).await +} + +#[inline(never)] +async fn b() { + black_box(c()).await +} + +#[inline(never)] +async fn c() { + loop { + black_box(tokio::task::yield_now()).await + } +} + +#[test] +fn current_thread() { + let rt = runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + async fn dump() { + let handle = Handle::current(); + let dump = handle.dump().await; + + let tasks: Vec<_> = dump.tasks().iter().collect(); + + assert_eq!(tasks.len(), 3); + + for task in tasks { + let trace = task.trace().to_string(); + eprintln!("\n\n{trace}\n\n"); + assert!(trace.contains("dump::a")); + assert!(trace.contains("dump::b")); + assert!(trace.contains("dump::c")); + assert!(trace.contains("tokio::task::yield_now")); + } + } + + rt.block_on(async { + tokio::select!( + biased; + _ = tokio::spawn(a()) => {}, + _ = tokio::spawn(a()) => {}, + _ = tokio::spawn(a()) => {}, + _ = dump() => {}, + ); + }); +} + +#[test] +fn multi_thread() { + let rt = runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(3) + .build() + .unwrap(); + + async fn dump() { + let handle = Handle::current(); + let dump = handle.dump().await; + + let tasks: Vec<_> = dump.tasks().iter().collect(); + + assert_eq!(tasks.len(), 3); + + for task in tasks { + let trace = task.trace().to_string(); + eprintln!("\n\n{trace}\n\n"); + assert!(trace.contains("dump::a")); + assert!(trace.contains("dump::b")); + assert!(trace.contains("dump::c")); + assert!(trace.contains("tokio::task::yield_now")); + } + } + + rt.block_on(async { + tokio::select!( + biased; + _ = tokio::spawn(a()) => {}, + _ = tokio::spawn(a()) => {}, + _ = tokio::spawn(a()) => {}, + _ = dump() => {}, + ); + }); +} diff --git a/third_party/rust/tokio/tests/fs.rs b/third_party/rust/tokio/tests/fs.rs index 13c44c08d6ab..ba38b719f2c1 100644 --- a/third_party/rust/tokio/tests/fs.rs +++ b/third_party/rust/tokio/tests/fs.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations use tokio::fs; use tokio_test::assert_ok; diff --git a/third_party/rust/tokio/tests/fs_canonicalize_dir.rs b/third_party/rust/tokio/tests/fs_canonicalize_dir.rs new file mode 100644 index 000000000000..83f0d8137371 --- /dev/null +++ b/third_party/rust/tokio/tests/fs_canonicalize_dir.rs @@ -0,0 +1,22 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations + +use tokio::fs; + +#[tokio::test] +#[cfg(unix)] +async fn canonicalize_root_dir_unix() { + assert_eq!(fs::canonicalize("/.").await.unwrap().to_str().unwrap(), "/"); +} + +#[tokio::test] +#[cfg(windows)] +async fn canonicalize_root_dir_windows() { + // 2-step let bindings due to Rust memory semantics + let dir_path = fs::canonicalize("C:\\.\\").await.unwrap(); + + let dir_name = dir_path.to_str().unwrap(); + + assert!(dir_name.starts_with("\\\\")); + assert!(dir_name.ends_with("C:\\")); +} diff --git a/third_party/rust/tokio/tests/fs_copy.rs b/third_party/rust/tokio/tests/fs_copy.rs index 8d1632013ea7..e6e5b666f4a7 100644 --- a/third_party/rust/tokio/tests/fs_copy.rs +++ b/third_party/rust/tokio/tests/fs_copy.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations use tempfile::tempdir; use tokio::fs; diff --git a/third_party/rust/tokio/tests/fs_dir.rs b/third_party/rust/tokio/tests/fs_dir.rs index 21efe8c0eebe..5f653cb2135f 100644 --- a/third_party/rust/tokio/tests/fs_dir.rs +++ b/third_party/rust/tokio/tests/fs_dir.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations use tokio::fs; use tokio_test::{assert_err, assert_ok}; @@ -45,6 +45,27 @@ async fn build_dir() { ); } +#[tokio::test] +#[cfg(unix)] +async fn build_dir_mode_read_only() { + let base_dir = tempdir().unwrap(); + let new_dir = base_dir.path().join("abc"); + + assert_ok!( + fs::DirBuilder::new() + .recursive(true) + .mode(0o444) + .create(&new_dir) + .await + ); + + assert!(fs::metadata(new_dir) + .await + .expect("metadata result") + .permissions() + .readonly()); +} + #[tokio::test] async fn remove() { let base_dir = tempdir().unwrap(); @@ -85,3 +106,21 @@ async fn read_inherent() { vec!["aa".to_string(), "bb".to_string(), "cc".to_string()] ); } + +#[tokio::test] +async fn read_dir_entry_info() { + let temp_dir = tempdir().unwrap(); + + let file_path = temp_dir.path().join("a.txt"); + + fs::write(&file_path, b"Hello File!").await.unwrap(); + + let mut dir = fs::read_dir(temp_dir.path()).await.unwrap(); + + let first_entry = dir.next_entry().await.unwrap().unwrap(); + + assert_eq!(first_entry.path(), file_path); + assert_eq!(first_entry.file_name(), "a.txt"); + assert!(first_entry.metadata().await.unwrap().is_file()); + assert!(first_entry.file_type().await.unwrap().is_file()); +} diff --git a/third_party/rust/tokio/tests/fs_file.rs b/third_party/rust/tokio/tests/fs_file.rs index f645e61aeda0..94d872a57aa3 100644 --- a/third_party/rust/tokio/tests/fs_file.rs +++ b/third_party/rust/tokio/tests/fs_file.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations use std::io::prelude::*; use tempfile::NamedTempFile; @@ -73,7 +73,7 @@ async fn coop() { let mut buf = [0; 1024]; loop { - file.read(&mut buf).await.unwrap(); + let _ = file.read(&mut buf).await.unwrap(); file.seek(std::io::SeekFrom::Start(0)).await.unwrap(); } }); @@ -87,13 +87,93 @@ async fn coop() { panic!("did not yield"); } +#[tokio::test] +async fn write_to_clone() { + let tempfile = tempfile(); + + let file = File::create(tempfile.path()).await.unwrap(); + let mut clone = file.try_clone().await.unwrap(); + + clone.write_all(HELLO).await.unwrap(); + clone.flush().await.unwrap(); + + let contents = std::fs::read(tempfile.path()).unwrap(); + assert_eq!(contents, HELLO); +} + +#[tokio::test] +async fn write_into_std() { + let tempfile = tempfile(); + + let file = File::create(tempfile.path()).await.unwrap(); + let mut std_file = file.into_std().await; + + std_file.write_all(HELLO).unwrap(); + + let contents = std::fs::read(tempfile.path()).unwrap(); + assert_eq!(contents, HELLO); +} + +#[tokio::test] +async fn write_into_std_immediate() { + let tempfile = tempfile(); + + let file = File::create(tempfile.path()).await.unwrap(); + let mut std_file = file.try_into_std().unwrap(); + + std_file.write_all(HELLO).unwrap(); + + let contents = std::fs::read(tempfile.path()).unwrap(); + assert_eq!(contents, HELLO); +} + +#[tokio::test] +async fn read_file_from_std() { + let mut tempfile = tempfile(); + tempfile.write_all(HELLO).unwrap(); + + let std_file = std::fs::File::open(tempfile.path()).unwrap(); + let mut file = File::from(std_file); + + let mut buf = [0; 1024]; + let n = file.read(&mut buf).await.unwrap(); + assert_eq!(n, HELLO.len()); + assert_eq!(&buf[..n], HELLO); +} + fn tempfile() -> NamedTempFile { NamedTempFile::new().unwrap() } #[tokio::test] #[cfg(unix)] -async fn unix_fd() { +async fn file_debug_fmt() { + let tempfile = tempfile(); + + let file = File::open(tempfile.path()).await.unwrap(); + + assert_eq!( + &format!("{:?}", file)[0..33], + "tokio::fs::File { std: File { fd:" + ); +} + +#[tokio::test] +#[cfg(windows)] +async fn file_debug_fmt() { + let tempfile = tempfile(); + + let file = File::open(tempfile.path()).await.unwrap(); + + assert_eq!( + &format!("{:?}", file)[0..37], + "tokio::fs::File { std: File { handle:" + ); +} + +#[tokio::test] +#[cfg(unix)] +async fn unix_fd_is_valid() { use std::os::unix::io::AsRawFd; let tempfile = tempfile(); @@ -101,6 +181,26 @@ async fn unix_fd() { assert!(file.as_raw_fd() as u64 > 0); } +#[tokio::test] +#[cfg(unix)] +async fn read_file_from_unix_fd() { + use std::os::unix::io::{FromRawFd, IntoRawFd}; + + let mut tempfile = tempfile(); + tempfile.write_all(HELLO).unwrap(); + + let file1 = File::open(tempfile.path()).await.unwrap(); + let raw_fd = file1.into_std().await.into_raw_fd(); + assert!(raw_fd > 0); + + let mut file2 = unsafe { File::from_raw_fd(raw_fd) }; + + let mut buf = [0; 1024]; + let n = file2.read(&mut buf).await.unwrap(); + assert_eq!(n, HELLO.len()); + assert_eq!(&buf[..n], HELLO); +} + #[tokio::test] #[cfg(windows)] async fn windows_handle() { diff --git a/third_party/rust/tokio/tests/fs_link.rs b/third_party/rust/tokio/tests/fs_link.rs index 2ef666fb2f68..681b566607ed 100644 --- a/third_party/rust/tokio/tests/fs_link.rs +++ b/third_party/rust/tokio/tests/fs_link.rs @@ -1,10 +1,9 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations use tokio::fs; -use std::io::prelude::*; -use std::io::BufReader; +use std::io::Write; use tempfile::tempdir; #[tokio::test] @@ -13,24 +12,23 @@ async fn test_hard_link() { let src = dir.path().join("src.txt"); let dst = dir.path().join("dst.txt"); - { - let mut file = std::fs::File::create(&src).unwrap(); - file.write_all(b"hello").unwrap(); - } + std::fs::File::create(&src) + .unwrap() + .write_all(b"hello") + .unwrap(); - let dst_2 = dst.clone(); + fs::hard_link(&src, &dst).await.unwrap(); - assert!(fs::hard_link(src, dst_2.clone()).await.is_ok()); + std::fs::File::create(&src) + .unwrap() + .write_all(b"new-data") + .unwrap(); - let mut content = String::new(); + let content = fs::read(&dst).await.unwrap(); + assert_eq!(content, b"new-data"); - { - let file = std::fs::File::open(dst).unwrap(); - let mut reader = BufReader::new(file); - reader.read_to_string(&mut content).unwrap(); - } - - assert!(content == "hello"); + // test that this is not a symlink: + assert!(fs::read_link(&dst).await.is_err()); } #[cfg(unix)] @@ -40,25 +38,20 @@ async fn test_symlink() { let src = dir.path().join("src.txt"); let dst = dir.path().join("dst.txt"); - { - let mut file = std::fs::File::create(&src).unwrap(); - file.write_all(b"hello").unwrap(); - } + std::fs::File::create(&src) + .unwrap() + .write_all(b"hello") + .unwrap(); - let src_2 = src.clone(); - let dst_2 = dst.clone(); + fs::symlink(&src, &dst).await.unwrap(); - assert!(fs::symlink(src_2.clone(), dst_2.clone()).await.is_ok()); + std::fs::File::create(&src) + .unwrap() + .write_all(b"new-data") + .unwrap(); - let mut content = String::new(); - - { - let file = std::fs::File::open(dst.clone()).unwrap(); - let mut reader = BufReader::new(file); - reader.read_to_string(&mut content).unwrap(); - } - - assert!(content == "hello"); + let content = fs::read(&dst).await.unwrap(); + assert_eq!(content, b"new-data"); let read = fs::read_link(dst.clone()).await.unwrap(); assert!(read == src); diff --git a/third_party/rust/tokio/tests/fs_open_options.rs b/third_party/rust/tokio/tests/fs_open_options.rs new file mode 100644 index 000000000000..e8d8b51b39f1 --- /dev/null +++ b/third_party/rust/tokio/tests/fs_open_options.rs @@ -0,0 +1,80 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations + +use std::io::Write; +use tempfile::NamedTempFile; +use tokio::fs::OpenOptions; +use tokio::io::AsyncReadExt; + +const HELLO: &[u8] = b"hello world..."; + +#[tokio::test] +async fn open_with_open_options_and_read() { + let mut tempfile = NamedTempFile::new().unwrap(); + tempfile.write_all(HELLO).unwrap(); + + let mut file = OpenOptions::new().read(true).open(tempfile).await.unwrap(); + + let mut buf = [0; 1024]; + let n = file.read(&mut buf).await.unwrap(); + + assert_eq!(n, HELLO.len()); + assert_eq!(&buf[..n], HELLO); +} + +#[tokio::test] +async fn open_options_write() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().write(true)).contains("write: true")); +} + +#[tokio::test] +async fn open_options_append() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().append(true)).contains("append: true")); +} + +#[tokio::test] +async fn open_options_truncate() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().truncate(true)).contains("truncate: true")); +} + +#[tokio::test] +async fn open_options_create() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().create(true)).contains("create: true")); +} + +#[tokio::test] +async fn open_options_create_new() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().create_new(true)).contains("create_new: true")); +} + +#[tokio::test] +#[cfg(unix)] +async fn open_options_mode() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().mode(0o644)).contains("mode: 420 ")); +} + +#[tokio::test] +#[cfg(target_os = "linux")] +async fn open_options_custom_flags_linux() { + // TESTING HACK: use Debug output to check the stored data + assert!( + format!("{:?}", OpenOptions::new().custom_flags(libc::O_TRUNC)) + .contains("custom_flags: 512,") + ); +} + +#[tokio::test] +#[cfg(any(target_os = "freebsd", target_os = "macos"))] +async fn open_options_custom_flags_bsd_family() { + // TESTING HACK: use Debug output to check the stored data + assert!( + format!("{:?}", OpenOptions::new().custom_flags(libc::O_NOFOLLOW)) + .contains("custom_flags: 256,") + ); +} diff --git a/third_party/rust/tokio/tests/fs_open_options_windows.rs b/third_party/rust/tokio/tests/fs_open_options_windows.rs new file mode 100644 index 000000000000..398e6a12b36a --- /dev/null +++ b/third_party/rust/tokio/tests/fs_open_options_windows.rs @@ -0,0 +1,51 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations +#![cfg(windows)] + +use tokio::fs::OpenOptions; +use windows_sys::Win32::Storage::FileSystem; + +#[tokio::test] +#[cfg(windows)] +async fn open_options_windows_access_mode() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().access_mode(0)).contains("access_mode: Some(0)")); +} + +#[tokio::test] +#[cfg(windows)] +async fn open_options_windows_share_mode() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!("{:?}", OpenOptions::new().share_mode(0)).contains("share_mode: 0,")); +} + +#[tokio::test] +#[cfg(windows)] +async fn open_options_windows_custom_flags() { + // TESTING HACK: use Debug output to check the stored data + assert!(format!( + "{:?}", + OpenOptions::new().custom_flags(FileSystem::FILE_FLAG_DELETE_ON_CLOSE) + ) + .contains("custom_flags: 67108864,")); +} + +#[tokio::test] +#[cfg(windows)] +async fn open_options_windows_attributes() { + assert!(format!( + "{:?}", + OpenOptions::new().attributes(FileSystem::FILE_ATTRIBUTE_HIDDEN) + ) + .contains("attributes: 2,")); +} + +#[tokio::test] +#[cfg(windows)] +async fn open_options_windows_security_qos_flags() { + assert!(format!( + "{:?}", + OpenOptions::new().security_qos_flags(FileSystem::SECURITY_IDENTIFICATION) + ) + .contains("security_qos_flags: 1114112,")); +} diff --git a/third_party/rust/tokio/tests/fs_remove_dir_all.rs b/third_party/rust/tokio/tests/fs_remove_dir_all.rs new file mode 100644 index 000000000000..53a2fd34f36c --- /dev/null +++ b/third_party/rust/tokio/tests/fs_remove_dir_all.rs @@ -0,0 +1,31 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations + +use tempfile::tempdir; +use tokio::fs; + +#[tokio::test] +async fn remove_dir_all() { + let temp_dir = tempdir().unwrap(); + + let test_dir = temp_dir.path().join("test"); + fs::create_dir(&test_dir).await.unwrap(); + + let file_path = test_dir.as_path().join("a.txt"); + + fs::write(&file_path, b"Hello File!").await.unwrap(); + + fs::remove_dir_all(test_dir.as_path()).await.unwrap(); + + // test dir should no longer exist + match fs::try_exists(test_dir).await { + Ok(exists) => assert!(!exists), + Err(_) => println!("ignored try_exists error after remove_dir_all"), + }; + + // contents should no longer exist + match fs::try_exists(file_path).await { + Ok(exists) => assert!(!exists), + Err(_) => println!("ignored try_exists error after remove_dir_all"), + }; +} diff --git a/third_party/rust/tokio/tests/fs_remove_file.rs b/third_party/rust/tokio/tests/fs_remove_file.rs new file mode 100644 index 000000000000..14fd680dabeb --- /dev/null +++ b/third_party/rust/tokio/tests/fs_remove_file.rs @@ -0,0 +1,24 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations + +use tempfile::tempdir; +use tokio::fs; + +#[tokio::test] +async fn remove_file() { + let temp_dir = tempdir().unwrap(); + + let file_path = temp_dir.path().join("a.txt"); + + fs::write(&file_path, b"Hello File!").await.unwrap(); + + assert!(fs::try_exists(&file_path).await.unwrap()); + + fs::remove_file(&file_path).await.unwrap(); + + // should no longer exist + match fs::try_exists(file_path).await { + Ok(exists) => assert!(!exists), + Err(_) => println!("ignored try_exists error after remove_file"), + }; +} diff --git a/third_party/rust/tokio/tests/fs_rename.rs b/third_party/rust/tokio/tests/fs_rename.rs new file mode 100644 index 000000000000..382fea079bde --- /dev/null +++ b/third_party/rust/tokio/tests/fs_rename.rs @@ -0,0 +1,28 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations + +use tempfile::tempdir; +use tokio::fs; + +#[tokio::test] +async fn rename_file() { + let temp_dir = tempdir().unwrap(); + + let file_path = temp_dir.path().join("a.txt"); + + fs::write(&file_path, b"Hello File!").await.unwrap(); + + assert!(fs::try_exists(&file_path).await.unwrap()); + + let new_file_path = temp_dir.path().join("b.txt"); + + fs::rename(&file_path, &new_file_path).await.unwrap(); + + assert!(fs::try_exists(new_file_path).await.unwrap()); + + // original file should no longer exist + match fs::try_exists(file_path).await { + Ok(exists) => assert!(!exists), + Err(_) => println!("ignored try_exists error after rename"), + }; +} diff --git a/third_party/rust/tokio/tests/fs_symlink_dir_windows.rs b/third_party/rust/tokio/tests/fs_symlink_dir_windows.rs new file mode 100644 index 000000000000..d80354268a27 --- /dev/null +++ b/third_party/rust/tokio/tests/fs_symlink_dir_windows.rs @@ -0,0 +1,31 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations +#![cfg(windows)] + +use tempfile::tempdir; +use tokio::fs; + +#[tokio::test] +async fn symlink_file_windows() { + const FILE_NAME: &str = "abc.txt"; + + let temp_dir = tempdir().unwrap(); + + let dir1 = temp_dir.path().join("a"); + fs::create_dir(&dir1).await.unwrap(); + + let file1 = dir1.as_path().join(FILE_NAME); + fs::write(&file1, b"Hello File!").await.unwrap(); + + let dir2 = temp_dir.path().join("b"); + fs::symlink_dir(&dir1, &dir2).await.unwrap(); + + fs::write(&file1, b"new data!").await.unwrap(); + + let file2 = dir2.as_path().join(FILE_NAME); + + let from = fs::read(&file1).await.unwrap(); + let to = fs::read(&file2).await.unwrap(); + + assert_eq!(from, to); +} diff --git a/third_party/rust/tokio/tests/fs_symlink_file_windows.rs b/third_party/rust/tokio/tests/fs_symlink_file_windows.rs new file mode 100644 index 000000000000..50eaf8aab752 --- /dev/null +++ b/third_party/rust/tokio/tests/fs_symlink_file_windows.rs @@ -0,0 +1,24 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations +#![cfg(windows)] + +use tempfile::tempdir; +use tokio::fs; + +#[tokio::test] +async fn symlink_file_windows() { + let dir = tempdir().unwrap(); + + let source_path = dir.path().join("foo.txt"); + let dest_path = dir.path().join("bar.txt"); + + fs::write(&source_path, b"Hello File!").await.unwrap(); + fs::symlink_file(&source_path, &dest_path).await.unwrap(); + + fs::write(&source_path, b"new data!").await.unwrap(); + + let from = fs::read(&source_path).await.unwrap(); + let to = fs::read(&dest_path).await.unwrap(); + + assert_eq!(from, to); +} diff --git a/third_party/rust/tokio/tests/fs_try_exists.rs b/third_party/rust/tokio/tests/fs_try_exists.rs new file mode 100644 index 000000000000..00d236500153 --- /dev/null +++ b/third_party/rust/tokio/tests/fs_try_exists.rs @@ -0,0 +1,44 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // WASI does not support all fs operations + +use tempfile::tempdir; +use tokio::fs; + +#[tokio::test] +async fn try_exists() { + let dir = tempdir().unwrap(); + + let existing_path = dir.path().join("foo.txt"); + fs::write(&existing_path, b"Hello File!").await.unwrap(); + let nonexisting_path = dir.path().join("bar.txt"); + + assert!(fs::try_exists(existing_path).await.unwrap()); + assert!(!fs::try_exists(nonexisting_path).await.unwrap()); + // FreeBSD root user always has permission to stat. + #[cfg(all(unix, not(target_os = "freebsd")))] + { + use std::os::unix::prelude::PermissionsExt; + let permission_denied_directory_path = dir.path().join("baz"); + fs::create_dir(&permission_denied_directory_path) + .await + .unwrap(); + let permission_denied_file_path = permission_denied_directory_path.join("baz.txt"); + fs::write(&permission_denied_file_path, b"Hello File!") + .await + .unwrap(); + let mut perms = tokio::fs::metadata(&permission_denied_directory_path) + .await + .unwrap() + .permissions(); + + perms.set_mode(0o244); + fs::set_permissions(&permission_denied_directory_path, perms) + .await + .unwrap(); + let permission_denied_result = fs::try_exists(permission_denied_file_path).await; + assert_eq!( + permission_denied_result.err().unwrap().kind(), + std::io::ErrorKind::PermissionDenied + ); + } +} diff --git a/third_party/rust/tokio/tests/io_async_fd.rs b/third_party/rust/tokio/tests/io_async_fd.rs index 5a6875e3c2ed..cdbfbacd0dbc 100644 --- a/third_party/rust/tokio/tests/io_async_fd.rs +++ b/third_party/rust/tokio/tests/io_async_fd.rs @@ -599,3 +599,89 @@ fn driver_shutdown_wakes_poll_race() { assert_err!(futures::executor::block_on(poll_writable(&afd_a))); } } + +#[tokio::test] +#[cfg(any(target_os = "linux", target_os = "android"))] +async fn priority_event_on_oob_data() { + use std::net::SocketAddr; + + use tokio::io::Interest; + + let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); + + let listener = std::net::TcpListener::bind(addr).unwrap(); + let addr = listener.local_addr().unwrap(); + + let client = std::net::TcpStream::connect(addr).unwrap(); + let client = AsyncFd::with_interest(client, Interest::PRIORITY).unwrap(); + + let (stream, _) = listener.accept().unwrap(); + + // Sending out of band data should trigger priority event. + send_oob_data(&stream, b"hello").unwrap(); + + let _ = client.ready(Interest::PRIORITY).await.unwrap(); +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn send_oob_data(stream: &S, data: &[u8]) -> io::Result { + unsafe { + let res = libc::send( + stream.as_raw_fd(), + data.as_ptr().cast(), + data.len(), + libc::MSG_OOB, + ); + if res == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(res as usize) + } + } +} + +#[tokio::test] +async fn clear_ready_matching_clears_ready() { + use tokio::io::{Interest, Ready}; + + let (a, mut b) = socketpair(); + + let afd_a = AsyncFd::new(a).unwrap(); + b.write_all(b"0").unwrap(); + + let mut guard = afd_a + .ready(Interest::READABLE | Interest::WRITABLE) + .await + .unwrap(); + + assert_eq!(guard.ready(), Ready::READABLE | Ready::WRITABLE); + + guard.clear_ready_matching(Ready::READABLE); + assert_eq!(guard.ready(), Ready::WRITABLE); + + guard.clear_ready_matching(Ready::WRITABLE); + assert_eq!(guard.ready(), Ready::EMPTY); +} + +#[tokio::test] +async fn clear_ready_matching_clears_ready_mut() { + use tokio::io::{Interest, Ready}; + + let (a, mut b) = socketpair(); + + let mut afd_a = AsyncFd::new(a).unwrap(); + b.write_all(b"0").unwrap(); + + let mut guard = afd_a + .ready_mut(Interest::READABLE | Interest::WRITABLE) + .await + .unwrap(); + + assert_eq!(guard.ready(), Ready::READABLE | Ready::WRITABLE); + + guard.clear_ready_matching(Ready::READABLE); + assert_eq!(guard.ready(), Ready::WRITABLE); + + guard.clear_ready_matching(Ready::WRITABLE); + assert_eq!(guard.ready(), Ready::EMPTY); +} diff --git a/third_party/rust/tokio/tests/io_buf_writer.rs b/third_party/rust/tokio/tests/io_buf_writer.rs index 47a0d466f494..d3acf62c784c 100644 --- a/third_party/rust/tokio/tests/io_buf_writer.rs +++ b/third_party/rust/tokio/tests/io_buf_writer.rs @@ -70,15 +70,15 @@ where async fn buf_writer() { let mut writer = BufWriter::with_capacity(2, Vec::new()); - writer.write(&[0, 1]).await.unwrap(); + assert_eq!(writer.write(&[0, 1]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1]); - writer.write(&[2]).await.unwrap(); + assert_eq!(writer.write(&[2]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2]); assert_eq!(*writer.get_ref(), [0, 1]); - writer.write(&[3]).await.unwrap(); + assert_eq!(writer.write(&[3]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2, 3]); assert_eq!(*writer.get_ref(), [0, 1]); @@ -86,20 +86,20 @@ async fn buf_writer() { assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); - writer.write(&[4]).await.unwrap(); - writer.write(&[5]).await.unwrap(); + assert_eq!(writer.write(&[4]).await.unwrap(), 1); + assert_eq!(writer.write(&[5]).await.unwrap(), 1); assert_eq!(writer.buffer(), [4, 5]); assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); - writer.write(&[6]).await.unwrap(); + assert_eq!(writer.write(&[6]).await.unwrap(), 1); assert_eq!(writer.buffer(), [6]); assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5]); - writer.write(&[7, 8]).await.unwrap(); + assert_eq!(writer.write(&[7, 8]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8]); - writer.write(&[9, 10, 11]).await.unwrap(); + assert_eq!(writer.write(&[9, 10, 11]).await.unwrap(), 3); assert_eq!(writer.buffer(), []); assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); @@ -111,7 +111,7 @@ async fn buf_writer() { #[tokio::test] async fn buf_writer_inner_flushes() { let mut w = BufWriter::with_capacity(3, Vec::new()); - w.write(&[0, 1]).await.unwrap(); + assert_eq!(w.write(&[0, 1]).await.unwrap(), 2); assert_eq!(*w.get_ref(), []); w.flush().await.unwrap(); let w = w.into_inner(); @@ -135,15 +135,15 @@ async fn buf_writer_seek() { async fn maybe_pending_buf_writer() { let mut writer = BufWriter::with_capacity(2, MaybePending::new(Vec::new())); - writer.write(&[0, 1]).await.unwrap(); + assert_eq!(writer.write(&[0, 1]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(&writer.get_ref().inner, &[0, 1]); - writer.write(&[2]).await.unwrap(); + assert_eq!(writer.write(&[2]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2]); assert_eq!(&writer.get_ref().inner, &[0, 1]); - writer.write(&[3]).await.unwrap(); + assert_eq!(writer.write(&[3]).await.unwrap(), 1); assert_eq!(writer.buffer(), [2, 3]); assert_eq!(&writer.get_ref().inner, &[0, 1]); @@ -151,20 +151,20 @@ async fn maybe_pending_buf_writer() { assert_eq!(writer.buffer(), []); assert_eq!(&writer.get_ref().inner, &[0, 1, 2, 3]); - writer.write(&[4]).await.unwrap(); - writer.write(&[5]).await.unwrap(); + assert_eq!(writer.write(&[4]).await.unwrap(), 1); + assert_eq!(writer.write(&[5]).await.unwrap(), 1); assert_eq!(writer.buffer(), [4, 5]); assert_eq!(&writer.get_ref().inner, &[0, 1, 2, 3]); - writer.write(&[6]).await.unwrap(); + assert_eq!(writer.write(&[6]).await.unwrap(), 1); assert_eq!(writer.buffer(), [6]); assert_eq!(writer.get_ref().inner, &[0, 1, 2, 3, 4, 5]); - writer.write(&[7, 8]).await.unwrap(); + assert_eq!(writer.write(&[7, 8]).await.unwrap(), 2); assert_eq!(writer.buffer(), []); assert_eq!(writer.get_ref().inner, &[0, 1, 2, 3, 4, 5, 6, 7, 8]); - writer.write(&[9, 10, 11]).await.unwrap(); + assert_eq!(writer.write(&[9, 10, 11]).await.unwrap(), 3); assert_eq!(writer.buffer(), []); assert_eq!( writer.get_ref().inner, @@ -182,7 +182,7 @@ async fn maybe_pending_buf_writer() { #[tokio::test] async fn maybe_pending_buf_writer_inner_flushes() { let mut w = BufWriter::with_capacity(3, MaybePending::new(Vec::new())); - w.write(&[0, 1]).await.unwrap(); + assert_eq!(w.write(&[0, 1]).await.unwrap(), 2); assert_eq!(&w.get_ref().inner, &[]); w.flush().await.unwrap(); let w = w.into_inner().inner; diff --git a/third_party/rust/tokio/tests/io_copy_bidirectional.rs b/third_party/rust/tokio/tests/io_copy_bidirectional.rs index 0e82b29ea69c..c54967595500 100644 --- a/third_party/rust/tokio/tests/io_copy_bidirectional.rs +++ b/third_party/rust/tokio/tests/io_copy_bidirectional.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind() use std::time::Duration; use tokio::io::{self, copy_bidirectional, AsyncReadExt, AsyncWriteExt}; @@ -111,18 +111,30 @@ async fn blocking_one_side_does_not_block_other() { } #[tokio::test] -async fn immediate_exit_on_error() { - symmetric(|handle, mut a, mut b| async move { - block_write(&mut a).await; +async fn immediate_exit_on_write_error() { + let payload = b"here, take this"; + let error = || io::Error::new(io::ErrorKind::Other, "no thanks!"); - // Fill up the b->copy->a path. We expect that this will _not_ drain - // before we exit the copy task. - let _bytes_written = block_write(&mut b).await; + let mut a = tokio_test::io::Builder::new() + .read(payload) + .write_error(error()) + .build(); - // Drop b. We should not wait for a to consume the data buffered in the - // copy loop, since b will be failing writes. - drop(b); - assert!(handle.await.unwrap().is_err()); - }) - .await + let mut b = tokio_test::io::Builder::new() + .read(payload) + .write_error(error()) + .build(); + + assert!(copy_bidirectional(&mut a, &mut b).await.is_err()); +} + +#[tokio::test] +async fn immediate_exit_on_read_error() { + let error = || io::Error::new(io::ErrorKind::Other, "got nothing!"); + + let mut a = tokio_test::io::Builder::new().read_error(error()).build(); + + let mut b = tokio_test::io::Builder::new().read_error(error()).build(); + + assert!(copy_bidirectional(&mut a, &mut b).await.is_err()); } diff --git a/third_party/rust/tokio/tests/io_driver.rs b/third_party/rust/tokio/tests/io_driver.rs index 6fb566de582c..97018e0f9675 100644 --- a/third_party/rust/tokio/tests/io_driver.rs +++ b/third_party/rust/tokio/tests/io_driver.rs @@ -1,5 +1,6 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +// Wasi does not support panic recovery or threading +#![cfg(all(feature = "full", not(tokio_wasi)))] use tokio::net::TcpListener; use tokio::runtime; @@ -79,7 +80,7 @@ fn test_drop_on_notify() { drop(task); // Establish a connection to the acceptor - let _s = TcpStream::connect(&addr).unwrap(); + let _s = TcpStream::connect(addr).unwrap(); // Force the reactor to turn rt.block_on(async {}); diff --git a/third_party/rust/tokio/tests/io_driver_drop.rs b/third_party/rust/tokio/tests/io_driver_drop.rs index 631e66e9fbe4..0a384d4196db 100644 --- a/third_party/rust/tokio/tests/io_driver_drop.rs +++ b/third_party/rust/tokio/tests/io_driver_drop.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind use tokio::net::TcpListener; use tokio::runtime; diff --git a/third_party/rust/tokio/tests/io_fill_buf.rs b/third_party/rust/tokio/tests/io_fill_buf.rs index 0b2ebd78fc75..62c3f1b370d5 100644 --- a/third_party/rust/tokio/tests/io_fill_buf.rs +++ b/third_party/rust/tokio/tests/io_fill_buf.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations use tempfile::NamedTempFile; use tokio::fs::File; diff --git a/third_party/rust/tokio/tests/io_mem_stream.rs b/third_party/rust/tokio/tests/io_mem_stream.rs index a2c2dadfc901..e79ec4e9d772 100644 --- a/third_party/rust/tokio/tests/io_mem_stream.rs +++ b/third_party/rust/tokio/tests/io_mem_stream.rs @@ -113,7 +113,7 @@ async fn duplex_is_cooperative() { let buf = [3u8; 4096]; tx.write_all(&buf).await.unwrap(); let mut buf = [0u8; 4096]; - rx.read(&mut buf).await.unwrap(); + let _ = rx.read(&mut buf).await.unwrap(); } } => {}, _ = tokio::task::yield_now() => {} diff --git a/third_party/rust/tokio/tests/io_panic.rs b/third_party/rust/tokio/tests/io_panic.rs new file mode 100644 index 000000000000..922d7c076e72 --- /dev/null +++ b/third_party/rust/tokio/tests/io_panic.rs @@ -0,0 +1,176 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery + +use std::task::{Context, Poll}; +use std::{error::Error, pin::Pin}; +use tokio::io::{self, split, AsyncRead, AsyncWrite, ReadBuf}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +struct RW; + +impl AsyncRead for RW { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + buf.put_slice(&[b'z']); + Poll::Ready(Ok(())) + } +} + +impl AsyncWrite for RW { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + Poll::Ready(Ok(1)) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +#[cfg(unix)] +mod unix { + use std::os::unix::prelude::{AsRawFd, RawFd}; + + pub struct MockFd; + + impl AsRawFd for MockFd { + fn as_raw_fd(&self) -> RawFd { + 0 + } + } +} + +#[test] +fn read_buf_initialize_unfilled_to_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + read_buf.initialize_unfilled_to(2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn read_buf_advance_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + read_buf.advance(2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn read_buf_set_filled_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + read_buf.set_filled(2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn read_buf_put_slice_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let mut buffer = Vec::::new(); + let mut read_buf = ReadBuf::new(&mut buffer); + + let new_slice = [0x40_u8, 0x41_u8]; + + read_buf.put_slice(&new_slice); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn unsplit_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let (r1, _w1) = split(RW); + let (_r2, w2) = split(RW); + r1.unsplit(w2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn async_fd_new_panic_caller() -> Result<(), Box> { + use tokio::io::unix::AsyncFd; + use tokio::runtime::Builder; + + let panic_location_file = test_panic(|| { + // Runtime without `enable_io` so it has no IO driver set. + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(async { + let fd = unix::MockFd; + + let _ = AsyncFd::new(fd); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn async_fd_with_interest_panic_caller() -> Result<(), Box> { + use tokio::io::unix::AsyncFd; + use tokio::io::Interest; + use tokio::runtime::Builder; + + let panic_location_file = test_panic(|| { + // Runtime without `enable_io` so it has no IO driver set. + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(async { + let fd = unix::MockFd; + + let _ = AsyncFd::with_interest(fd, Interest::READABLE); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/third_party/rust/tokio/tests/io_read.rs b/third_party/rust/tokio/tests/io_read.rs index cb1aa705231a..7f74c5e857a7 100644 --- a/third_party/rust/tokio/tests/io_read.rs +++ b/third_party/rust/tokio/tests/io_read.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; @@ -8,6 +8,11 @@ use std::io; use std::pin::Pin; use std::task::{Context, Poll}; +mod support { + pub(crate) mod leaked_buffers; +} +use support::leaked_buffers::LeakedBuffers; + #[tokio::test] async fn read() { #[derive(Default)] @@ -37,16 +42,27 @@ async fn read() { assert_eq!(buf[..], b"hello world"[..]); } -struct BadAsyncRead; +struct BadAsyncRead { + leaked_buffers: LeakedBuffers, +} + +impl BadAsyncRead { + fn new() -> Self { + Self { + leaked_buffers: LeakedBuffers::new(), + } + } +} impl AsyncRead for BadAsyncRead { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll> { - *buf = ReadBuf::new(Box::leak(vec![0; buf.capacity()].into_boxed_slice())); + *buf = ReadBuf::new(unsafe { self.leaked_buffers.create(buf.capacity()) }); buf.advance(buf.capacity()); + Poll::Ready(Ok(())) } } @@ -55,5 +71,5 @@ impl AsyncRead for BadAsyncRead { #[should_panic] async fn read_buf_bad_async_read() { let mut buf = Vec::with_capacity(10); - BadAsyncRead.read_buf(&mut buf).await.unwrap(); + BadAsyncRead::new().read_buf(&mut buf).await.unwrap(); } diff --git a/third_party/rust/tokio/tests/io_read_buf.rs b/third_party/rust/tokio/tests/io_read_buf.rs index 0328168d7abf..49a4f86f8ad9 100644 --- a/third_party/rust/tokio/tests/io_read_buf.rs +++ b/third_party/rust/tokio/tests/io_read_buf.rs @@ -34,3 +34,61 @@ async fn read_buf() { assert_eq!(n, 11); assert_eq!(buf[..], b"hello world"[..]); } + +#[tokio::test] +#[cfg(feature = "io-util")] +async fn issue_5588() { + use bytes::BufMut; + + // steps to zero + let mut buf = [0; 8]; + let mut read_buf = ReadBuf::new(&mut buf); + assert_eq!(read_buf.remaining_mut(), 8); + assert_eq!(read_buf.chunk_mut().len(), 8); + unsafe { + read_buf.advance_mut(1); + } + assert_eq!(read_buf.remaining_mut(), 7); + assert_eq!(read_buf.chunk_mut().len(), 7); + unsafe { + read_buf.advance_mut(5); + } + assert_eq!(read_buf.remaining_mut(), 2); + assert_eq!(read_buf.chunk_mut().len(), 2); + unsafe { + read_buf.advance_mut(2); + } + assert_eq!(read_buf.remaining_mut(), 0); + assert_eq!(read_buf.chunk_mut().len(), 0); + + // directly to zero + let mut buf = [0; 8]; + let mut read_buf = ReadBuf::new(&mut buf); + assert_eq!(read_buf.remaining_mut(), 8); + assert_eq!(read_buf.chunk_mut().len(), 8); + unsafe { + read_buf.advance_mut(8); + } + assert_eq!(read_buf.remaining_mut(), 0); + assert_eq!(read_buf.chunk_mut().len(), 0); + + // uninit + let mut buf = [std::mem::MaybeUninit::new(1); 8]; + let mut uninit = ReadBuf::uninit(&mut buf); + assert_eq!(uninit.remaining_mut(), 8); + assert_eq!(uninit.chunk_mut().len(), 8); + + let mut buf = [std::mem::MaybeUninit::uninit(); 8]; + let mut uninit = ReadBuf::uninit(&mut buf); + unsafe { + uninit.advance_mut(4); + } + assert_eq!(uninit.remaining_mut(), 4); + assert_eq!(uninit.chunk_mut().len(), 4); + uninit.put_u8(1); + assert_eq!(uninit.remaining_mut(), 3); + assert_eq!(uninit.chunk_mut().len(), 3); + uninit.put_slice(&[1, 2, 3]); + assert_eq!(uninit.remaining_mut(), 0); + assert_eq!(uninit.chunk_mut().len(), 0); +} diff --git a/third_party/rust/tokio/tests/io_read_to_end.rs b/third_party/rust/tokio/tests/io_read_to_end.rs index 171e6d6480ee..29b60becec47 100644 --- a/third_party/rust/tokio/tests/io_read_to_end.rs +++ b/third_party/rust/tokio/tests/io_read_to_end.rs @@ -5,6 +5,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; +use tokio_test::io::Builder; #[tokio::test] async fn read_to_end() { @@ -76,3 +77,48 @@ async fn read_to_end_uninit() { test.read_to_end(&mut buf).await.unwrap(); assert_eq!(buf.len(), 33); } + +#[tokio::test] +async fn read_to_end_doesnt_grow_with_capacity() { + let arr: Vec = (0..100).collect(); + + // We only test from 32 since we allocate at least 32 bytes each time + for len in 32..100 { + let bytes = &arr[..len]; + for split in 0..len { + for cap in 0..101 { + let mut mock = if split == 0 { + Builder::new().read(bytes).build() + } else { + Builder::new() + .read(&bytes[..split]) + .read(&bytes[split..]) + .build() + }; + let mut buf = Vec::with_capacity(cap); + AsyncReadExt::read_to_end(&mut mock, &mut buf) + .await + .unwrap(); + // It has the right data. + assert_eq!(buf.as_slice(), bytes); + // Unless cap was smaller than length, then we did not reallocate. + if cap >= len { + assert_eq!(buf.capacity(), cap); + } + } + } + } +} + +#[tokio::test] +async fn read_to_end_grows_capacity_if_unfit() { + let bytes = b"the_vector_startingcap_will_be_smaller"; + let mut mock = Builder::new().read(bytes).build(); + let initial_capacity = bytes.len() - 4; + let mut buf = Vec::with_capacity(initial_capacity); + AsyncReadExt::read_to_end(&mut mock, &mut buf) + .await + .unwrap(); + // *4 since it doubles when it doesn't fit and again when reaching EOF + assert_eq!(buf.capacity(), initial_capacity * 4); +} diff --git a/third_party/rust/tokio/tests/io_split.rs b/third_party/rust/tokio/tests/io_split.rs index a0121667f75b..4cbd2a2d1fb7 100644 --- a/third_party/rust/tokio/tests/io_split.rs +++ b/third_party/rust/tokio/tests/io_split.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery use tokio::io::{split, AsyncRead, AsyncWrite, ReadBuf, ReadHalf, WriteHalf}; diff --git a/third_party/rust/tokio/tests/io_take.rs b/third_party/rust/tokio/tests/io_take.rs index 684e041a674e..fba01d2e0c60 100644 --- a/third_party/rust/tokio/tests/io_take.rs +++ b/third_party/rust/tokio/tests/io_take.rs @@ -1,11 +1,16 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery use std::pin::Pin; use std::task::{Context, Poll}; use tokio::io::{self, AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; +mod support { + pub(crate) mod leaked_buffers; +} +use support::leaked_buffers::LeakedBuffers; + #[tokio::test] async fn take() { let mut buf = [0; 6]; @@ -34,17 +39,25 @@ async fn issue_4435() { assert_eq!(&buf, &b"ABhell\0\0"[..]); } -struct BadReader; +struct BadReader { + leaked_buffers: LeakedBuffers, +} + +impl BadReader { + fn new() -> Self { + Self { + leaked_buffers: LeakedBuffers::new(), + } + } +} impl AsyncRead for BadReader { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, _cx: &mut Context<'_>, read_buf: &mut ReadBuf<'_>, ) -> Poll> { - let vec = vec![0; 10]; - - let mut buf = ReadBuf::new(vec.leak()); + let mut buf = ReadBuf::new(unsafe { self.leaked_buffers.create(10) }); buf.put_slice(&[123; 10]); *read_buf = buf; @@ -57,5 +70,5 @@ impl AsyncRead for BadReader { async fn bad_reader_fails() { let mut buf = Vec::with_capacity(10); - BadReader.take(10).read_buf(&mut buf).await.unwrap(); + BadReader::new().take(10).read_buf(&mut buf).await.unwrap(); } diff --git a/third_party/rust/tokio/tests/join_handle_panic.rs b/third_party/rust/tokio/tests/join_handle_panic.rs index f7de92d41782..bc34fd0ea595 100644 --- a/third_party/rust/tokio/tests/join_handle_panic.rs +++ b/third_party/rust/tokio/tests/join_handle_panic.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery struct PanicsOnDrop; diff --git a/third_party/rust/tokio/tests/macros_join.rs b/third_party/rust/tokio/tests/macros_join.rs index d4f20b3862c2..7866ac086faf 100644 --- a/third_party/rust/tokio/tests/macros_join.rs +++ b/third_party/rust/tokio/tests/macros_join.rs @@ -1,15 +1,17 @@ #![cfg(feature = "macros")] -#![allow(clippy::blacklisted_name)] +#![allow(clippy::disallowed_names)] +use std::sync::Arc; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] +#[cfg(target_pointer_width = "64")] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, Semaphore}; use tokio_test::{assert_pending, assert_ready, task}; #[maybe_tokio_test] @@ -63,6 +65,7 @@ async fn two_await() { } #[test] +#[cfg(target_pointer_width = "64")] fn join_size() { use futures::future; use std::mem; @@ -71,12 +74,88 @@ fn join_size() { let ready = future::ready(0i32); tokio::join!(ready) }; - assert_eq!(mem::size_of_val(&fut), 16); + assert_eq!(mem::size_of_val(&fut), 32); let fut = async { let ready1 = future::ready(0i32); let ready2 = future::ready(0i32); tokio::join!(ready1, ready2) }; - assert_eq!(mem::size_of_val(&fut), 28); + assert_eq!(mem::size_of_val(&fut), 40); +} + +async fn non_cooperative_task(permits: Arc) -> usize { + let mut exceeded_budget = 0; + + for _ in 0..5 { + // Another task should run after this task uses its whole budget + for _ in 0..128 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + } + + exceeded_budget += 1; + } + + exceeded_budget +} + +async fn poor_little_task(permits: Arc) -> usize { + let mut how_many_times_i_got_to_run = 0; + + for _ in 0..5 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + how_many_times_i_got_to_run += 1; + } + + how_many_times_i_got_to_run +} + +#[tokio::test] +async fn join_does_not_allow_tasks_to_starve() { + let permits = Arc::new(Semaphore::new(1)); + + // non_cooperative_task should yield after its budget is exceeded and then poor_little_task should run. + let (non_cooperative_result, little_task_result) = tokio::join!( + non_cooperative_task(Arc::clone(&permits)), + poor_little_task(permits) + ); + + assert_eq!(5, non_cooperative_result); + assert_eq!(5, little_task_result); +} + +#[tokio::test] +async fn a_different_future_is_polled_first_every_time_poll_fn_is_polled() { + let poll_order = Arc::new(std::sync::Mutex::new(vec![])); + + let fut = |x, poll_order: Arc>>| async move { + for _ in 0..4 { + { + let mut guard = poll_order.lock().unwrap(); + + guard.push(x); + } + + tokio::task::yield_now().await; + } + }; + + tokio::join!( + fut(1, Arc::clone(&poll_order)), + fut(2, Arc::clone(&poll_order)), + fut(3, Arc::clone(&poll_order)), + ); + + // Each time the future created by join! is polled, it should start + // by polling a different future first. + assert_eq!( + vec![1, 2, 3, 2, 3, 1, 3, 1, 2, 1, 2, 3], + *poll_order.lock().unwrap() + ); +} + +#[tokio::test] +#[allow(clippy::unit_cmp)] +async fn empty_join() { + assert_eq!(tokio::join!(), ()); } diff --git a/third_party/rust/tokio/tests/macros_pin.rs b/third_party/rust/tokio/tests/macros_pin.rs index 70c95a1c8235..e99a3ee2017a 100644 --- a/third_party/rust/tokio/tests/macros_pin.rs +++ b/third_party/rust/tokio/tests/macros_pin.rs @@ -1,9 +1,9 @@ #![cfg(feature = "macros")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; async fn one() {} diff --git a/third_party/rust/tokio/tests/macros_rename_test.rs b/third_party/rust/tokio/tests/macros_rename_test.rs new file mode 100644 index 000000000000..6c2ce2f57aa7 --- /dev/null +++ b/third_party/rust/tokio/tests/macros_rename_test.rs @@ -0,0 +1,35 @@ +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threading + +#[allow(unused_imports)] +use std as tokio; + +use ::tokio as tokio1; + +mod test { + pub use ::tokio; +} + +async fn compute() -> usize { + let join = tokio1::spawn(async { 1 }); + join.await.unwrap() +} + +#[tokio1::main(crate = "tokio1")] +async fn compute_main() -> usize { + compute().await +} + +#[test] +fn crate_rename_main() { + assert_eq!(1, compute_main()); +} + +#[tokio1::test(crate = "tokio1")] +async fn crate_rename_test() { + assert_eq!(1, compute().await); +} + +#[test::tokio::test(crate = "test::tokio")] +async fn crate_path_test() { + assert_eq!(1, compute().await); +} diff --git a/third_party/rust/tokio/tests/macros_select.rs b/third_party/rust/tokio/tests/macros_select.rs index 755365affbc5..0514c864d5e4 100644 --- a/third_party/rust/tokio/tests/macros_select.rs +++ b/third_party/rust/tokio/tests/macros_select.rs @@ -1,10 +1,10 @@ #![cfg(feature = "macros")] -#![allow(clippy::blacklisted_name)] +#![allow(clippy::disallowed_names)] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; use tokio::sync::oneshot; @@ -206,46 +206,56 @@ async fn nested() { assert_eq!(res, 3); } -#[maybe_tokio_test] -async fn struct_size() { +#[cfg(target_pointer_width = "64")] +mod pointer_64_tests { + use super::maybe_tokio_test; use futures::future; use std::mem; - let fut = async { - let ready = future::ready(0i32); + #[maybe_tokio_test] + async fn struct_size_1() { + let fut = async { + let ready = future::ready(0i32); - tokio::select! { - _ = ready => {}, - } - }; + tokio::select! { + _ = ready => {}, + } + }; - assert!(mem::size_of_val(&fut) <= 32); + assert_eq!(mem::size_of_val(&fut), 32); + } - let fut = async { - let ready1 = future::ready(0i32); - let ready2 = future::ready(0i32); + #[maybe_tokio_test] + async fn struct_size_2() { + let fut = async { + let ready1 = future::ready(0i32); + let ready2 = future::ready(0i32); - tokio::select! { - _ = ready1 => {}, - _ = ready2 => {}, - } - }; + tokio::select! { + _ = ready1 => {}, + _ = ready2 => {}, + } + }; - assert!(mem::size_of_val(&fut) <= 40); + assert_eq!(mem::size_of_val(&fut), 40); + } - let fut = async { - let ready1 = future::ready(0i32); - let ready2 = future::ready(0i32); - let ready3 = future::ready(0i32); + #[maybe_tokio_test] + async fn struct_size_3() { + let fut = async { + let ready1 = future::ready(0i32); + let ready2 = future::ready(0i32); + let ready3 = future::ready(0i32); - tokio::select! { - _ = ready1 => {}, - _ = ready2 => {}, - _ = ready3 => {}, - } - }; + tokio::select! { + _ = ready1 => {}, + _ = ready2 => {}, + _ = ready3 => {}, + } + }; - assert!(mem::size_of_val(&fut) <= 48); + assert_eq!(mem::size_of_val(&fut), 48); + } } #[maybe_tokio_test] @@ -461,6 +471,7 @@ async fn many_branches() { x = async { 1 } => x, x = async { 1 } => x, x = async { 1 } => x, + x = async { 1 } => x, }; assert_eq!(1, num); @@ -598,3 +609,66 @@ async fn mut_ref_patterns() { }, }; } + +#[cfg(tokio_unstable)] +mod unstable { + use tokio::runtime::RngSeed; + + #[test] + fn deterministic_select_current_thread() { + let seed = b"bytes used to generate seed"; + let rt1 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values = rt1.block_on(async { (select_0_to_9().await, select_0_to_9().await) }); + + let rt2 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values = rt2.block_on(async { (select_0_to_9().await, select_0_to_9().await) }); + + assert_eq!(rt1_values, rt2_values); + } + + #[test] + #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] + fn deterministic_select_multi_thread() { + let seed = b"bytes used to generate seed"; + let rt1 = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values = rt1.block_on(async { + let _ = tokio::spawn(async { (select_0_to_9().await, select_0_to_9().await) }).await; + }); + + let rt2 = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values = rt2.block_on(async { + let _ = tokio::spawn(async { (select_0_to_9().await, select_0_to_9().await) }).await; + }); + + assert_eq!(rt1_values, rt2_values); + } + + async fn select_0_to_9() -> u32 { + tokio::select!( + x = async { 0 } => x, + x = async { 1 } => x, + x = async { 2 } => x, + x = async { 3 } => x, + x = async { 4 } => x, + x = async { 5 } => x, + x = async { 6 } => x, + x = async { 7 } => x, + x = async { 8 } => x, + x = async { 9 } => x, + ) + } +} diff --git a/third_party/rust/tokio/tests/macros_test.rs b/third_party/rust/tokio/tests/macros_test.rs index c5d9d9f9b016..85279b7edc03 100644 --- a/third_party/rust/tokio/tests/macros_test.rs +++ b/third_party/rust/tokio/tests/macros_test.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threading use tokio::test; @@ -50,6 +50,7 @@ pub async fn issue_4175_test() -> std::io::Result<()> { } // https://github.com/tokio-rs/tokio/issues/4175 +#[allow(clippy::let_unit_value)] pub mod clippy_semicolon_if_nothing_returned { #![deny(clippy::semicolon_if_nothing_returned)] @@ -70,3 +71,18 @@ pub mod clippy_semicolon_if_nothing_returned { // To trigger clippy::semicolon_if_nothing_returned lint, the block needs to contain newline. } } + +// https://github.com/tokio-rs/tokio/issues/5243 +pub mod issue_5243 { + macro_rules! mac { + (async fn $name:ident() $b:block) => { + #[::tokio::test] + async fn $name() { + $b + } + }; + } + mac!( + async fn foo() {} + ); +} diff --git a/third_party/rust/tokio/tests/macros_try_join.rs b/third_party/rust/tokio/tests/macros_try_join.rs index 60a726b659ae..74b1c9f9481c 100644 --- a/third_party/rust/tokio/tests/macros_try_join.rs +++ b/third_party/rust/tokio/tests/macros_try_join.rs @@ -1,13 +1,15 @@ #![cfg(feature = "macros")] -#![allow(clippy::blacklisted_name)] +#![allow(clippy::disallowed_names)] -use tokio::sync::oneshot; +use std::sync::Arc; + +use tokio::sync::{oneshot, Semaphore}; use tokio_test::{assert_pending, assert_ready, task}; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; #[maybe_tokio_test] @@ -86,6 +88,7 @@ async fn err_abort_early() { } #[test] +#[cfg(target_pointer_width = "64")] fn join_size() { use futures::future; use std::mem; @@ -94,16 +97,94 @@ fn join_size() { let ready = future::ready(ok(0i32)); tokio::try_join!(ready) }; - assert_eq!(mem::size_of_val(&fut), 16); + assert_eq!(mem::size_of_val(&fut), 32); let fut = async { let ready1 = future::ready(ok(0i32)); let ready2 = future::ready(ok(0i32)); tokio::try_join!(ready1, ready2) }; - assert_eq!(mem::size_of_val(&fut), 28); + assert_eq!(mem::size_of_val(&fut), 48); } fn ok(val: T) -> Result { Ok(val) } + +async fn non_cooperative_task(permits: Arc) -> Result { + let mut exceeded_budget = 0; + + for _ in 0..5 { + // Another task should run after this task uses its whole budget + for _ in 0..128 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + } + + exceeded_budget += 1; + } + + Ok(exceeded_budget) +} + +async fn poor_little_task(permits: Arc) -> Result { + let mut how_many_times_i_got_to_run = 0; + + for _ in 0..5 { + let _permit = permits.clone().acquire_owned().await.unwrap(); + + how_many_times_i_got_to_run += 1; + } + + Ok(how_many_times_i_got_to_run) +} + +#[tokio::test] +async fn try_join_does_not_allow_tasks_to_starve() { + let permits = Arc::new(Semaphore::new(10)); + + // non_cooperative_task should yield after its budget is exceeded and then poor_little_task should run. + let result = tokio::try_join!( + non_cooperative_task(Arc::clone(&permits)), + poor_little_task(permits) + ); + + let (non_cooperative_result, little_task_result) = result.unwrap(); + + assert_eq!(5, non_cooperative_result); + assert_eq!(5, little_task_result); +} + +#[tokio::test] +async fn a_different_future_is_polled_first_every_time_poll_fn_is_polled() { + let poll_order = Arc::new(std::sync::Mutex::new(vec![])); + + let fut = |x, poll_order: Arc>>| async move { + for _ in 0..4 { + { + let mut guard = poll_order.lock().unwrap(); + + guard.push(x); + } + + tokio::task::yield_now().await; + } + }; + + tokio::join!( + fut(1, Arc::clone(&poll_order)), + fut(2, Arc::clone(&poll_order)), + fut(3, Arc::clone(&poll_order)), + ); + + // Each time the future created by join! is polled, it should start + // by polling a different future first. + assert_eq!( + vec![1, 2, 3, 2, 3, 1, 3, 1, 2, 1, 2, 3], + *poll_order.lock().unwrap() + ); +} + +#[tokio::test] +async fn empty_try_join() { + assert_eq!(tokio::try_join!() as Result<_, ()>, Ok(())); +} diff --git a/third_party/rust/tokio/tests/net_bind_resource.rs b/third_party/rust/tokio/tests/net_bind_resource.rs index d4a0b8dab08f..76378b3ea1eb 100644 --- a/third_party/rust/tokio/tests/net_bind_resource.rs +++ b/third_party/rust/tokio/tests/net_bind_resource.rs @@ -1,9 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery or bind use tokio::net::TcpListener; -use std::convert::TryFrom; use std::net; #[test] diff --git a/third_party/rust/tokio/tests/net_lookup_host.rs b/third_party/rust/tokio/tests/net_lookup_host.rs index 44c8e19e0d87..2b346e65e373 100644 --- a/third_party/rust/tokio/tests/net_lookup_host.rs +++ b/third_party/rust/tokio/tests/net_lookup_host.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support direct socket operations use tokio::net; use tokio_test::assert_ok; diff --git a/third_party/rust/tokio/tests/named_pipe.rs b/third_party/rust/tokio/tests/net_named_pipe.rs similarity index 78% rename from third_party/rust/tokio/tests/named_pipe.rs rename to third_party/rust/tokio/tests/net_named_pipe.rs index 2055c3ce5be3..02eda48e57b4 100644 --- a/third_party/rust/tokio/tests/named_pipe.rs +++ b/third_party/rust/tokio/tests/net_named_pipe.rs @@ -2,13 +2,11 @@ #![cfg(all(windows))] use std::io; -use std::mem; -use std::os::windows::io::AsRawHandle; use std::time::Duration; -use tokio::io::AsyncWriteExt; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::windows::named_pipe::{ClientOptions, PipeMode, ServerOptions}; use tokio::time; -use winapi::shared::winerror; +use windows_sys::Win32::Foundation::{ERROR_NO_DATA, ERROR_PIPE_BUSY}; #[tokio::test] async fn test_named_pipe_client_drop() -> io::Result<()> { @@ -16,8 +14,6 @@ async fn test_named_pipe_client_drop() -> io::Result<()> { let mut server = ServerOptions::new().create(PIPE_NAME)?; - assert_eq!(num_instances("test-named-pipe-client-drop")?, 1); - let client = ClientOptions::new().open(PIPE_NAME)?; server.connect().await?; @@ -25,7 +21,7 @@ async fn test_named_pipe_client_drop() -> io::Result<()> { // instance will be broken because client is gone match server.write_all(b"ping").await { - Err(e) if e.raw_os_error() == Some(winerror::ERROR_NO_DATA as i32) => (), + Err(e) if e.raw_os_error() == Some(ERROR_NO_DATA as i32) => (), x => panic!("{:?}", x), } @@ -120,7 +116,7 @@ async fn test_named_pipe_multi_client() -> io::Result<()> { let client = loop { match ClientOptions::new().open(PIPE_NAME) { Ok(client) => break client, - Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), + Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), Err(e) if e.kind() == io::ErrorKind::NotFound => (), Err(e) => return Err(e), } @@ -249,7 +245,7 @@ async fn test_named_pipe_multi_client_ready() -> io::Result<()> { let client = loop { match ClientOptions::new().open(PIPE_NAME) { Ok(client) => break client, - Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (), + Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (), Err(e) if e.kind() == io::ErrorKind::NotFound => (), Err(e) => return Err(e), } @@ -327,67 +323,77 @@ async fn test_named_pipe_multi_client_ready() -> io::Result<()> { Ok(()) } -// This tests what happens when a client tries to disconnect. +// This tests that message mode works as expected. #[tokio::test] async fn test_named_pipe_mode_message() -> io::Result<()> { - const PIPE_NAME: &str = r"\\.\pipe\test-named-pipe-mode-message"; + // it's easy to accidentally get a seemingly working test here because byte pipes + // often return contents at write boundaries. to make sure we're doing the right thing we + // explicitly test that it doesn't work in byte mode. + _named_pipe_mode_message(PipeMode::Message).await?; + _named_pipe_mode_message(PipeMode::Byte).await +} - let server = ServerOptions::new() - .pipe_mode(PipeMode::Message) - .create(PIPE_NAME)?; +async fn _named_pipe_mode_message(mode: PipeMode) -> io::Result<()> { + let pipe_name = format!( + r"\\.\pipe\test-named-pipe-mode-message-{}", + matches!(mode, PipeMode::Message) + ); + let mut buf = [0u8; 32]; + + let mut server = ServerOptions::new() + .first_pipe_instance(true) + .pipe_mode(mode) + .create(&pipe_name)?; + + let mut client = ClientOptions::new().pipe_mode(mode).open(&pipe_name)?; - let _ = ClientOptions::new().open(PIPE_NAME)?; server.connect().await?; + + // this needs a few iterations, presumably Windows waits for a few calls before merging buffers + for _ in 0..10 { + client.write_all(b"hello").await?; + server.write_all(b"world").await?; + } + for _ in 0..10 { + let n = server.read(&mut buf).await?; + if buf[..n] != b"hello"[..] { + assert!(matches!(mode, PipeMode::Byte)); + return Ok(()); + } + let n = client.read(&mut buf).await?; + if buf[..n] != b"world"[..] { + assert!(matches!(mode, PipeMode::Byte)); + return Ok(()); + } + } + // byte mode should have errored before. + assert!(matches!(mode, PipeMode::Message)); Ok(()) } -fn num_instances(pipe_name: impl AsRef) -> io::Result { - use ntapi::ntioapi; - use winapi::shared::ntdef; +// This tests `NamedPipeServer::connect` with various access settings. +#[tokio::test] +async fn test_named_pipe_access() -> io::Result<()> { + const PIPE_NAME: &str = r"\\.\pipe\test-named-pipe-access"; - let mut name = pipe_name.as_ref().encode_utf16().collect::>(); - let mut name = ntdef::UNICODE_STRING { - Length: (name.len() * mem::size_of::()) as u16, - MaximumLength: (name.len() * mem::size_of::()) as u16, - Buffer: name.as_mut_ptr(), - }; - let root = std::fs::File::open(r"\\.\Pipe\")?; - let mut io_status_block = unsafe { mem::zeroed() }; - let mut file_directory_information = [0_u8; 1024]; + for (inb, outb) in [(true, true), (true, false), (false, true)] { + let (tx, rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let s = ServerOptions::new() + .access_inbound(inb) + .access_outbound(outb) + .create(PIPE_NAME)?; + let mut connect_fut = tokio_test::task::spawn(s.connect()); + assert!(connect_fut.poll().is_pending()); + tx.send(()).unwrap(); + connect_fut.await + }); - let status = unsafe { - ntioapi::NtQueryDirectoryFile( - root.as_raw_handle(), - std::ptr::null_mut(), - None, - std::ptr::null_mut(), - &mut io_status_block, - &mut file_directory_information as *mut _ as *mut _, - 1024, - ntioapi::FileDirectoryInformation, - 0, - &mut name, - 0, - ) - }; + // Wait for the server to call connect. + rx.await.unwrap(); + let _ = ClientOptions::new().read(outb).write(inb).open(PIPE_NAME)?; - if status as u32 != winerror::NO_ERROR { - return Err(io::Error::last_os_error()); + server.await??; } - - let info = unsafe { - mem::transmute::<_, &ntioapi::FILE_DIRECTORY_INFORMATION>(&file_directory_information) - }; - let raw_name = unsafe { - std::slice::from_raw_parts( - info.FileName.as_ptr(), - info.FileNameLength as usize / mem::size_of::(), - ) - }; - let name = String::from_utf16(raw_name).unwrap(); - let num_instances = unsafe { *info.EndOfFile.QuadPart() }; - - assert_eq!(name, pipe_name.as_ref()); - - Ok(num_instances as u32) + Ok(()) } diff --git a/third_party/rust/tokio/tests/net_panic.rs b/third_party/rust/tokio/tests/net_panic.rs new file mode 100644 index 000000000000..fc2209ad4b0a --- /dev/null +++ b/third_party/rust/tokio/tests/net_panic.rs @@ -0,0 +1,188 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] + +use std::error::Error; +use tokio::net::{TcpListener, TcpStream}; +use tokio::runtime::{Builder, Runtime}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn udp_socket_from_std_panic_caller() -> Result<(), Box> { + use std::net::SocketAddr; + use tokio::net::UdpSocket; + + let addr = "127.0.0.1:0".parse::().unwrap(); + let std_sock = std::net::UdpSocket::bind(addr).unwrap(); + std_sock.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _sock = UdpSocket::from_std(std_sock); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn tcp_listener_from_std_panic_caller() -> Result<(), Box> { + let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + std_listener.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = TcpListener::from_std(std_listener); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn tcp_stream_from_std_panic_caller() -> Result<(), Box> { + let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + + let std_stream = std::net::TcpStream::connect(std_listener.local_addr().unwrap()).unwrap(); + std_stream.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = TcpStream::from_std(std_stream); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_listener_bind_panic_caller() -> Result<(), Box> { + use tokio::net::UnixListener; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixListener::bind(&sock_path); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_listener_from_std_panic_caller() -> Result<(), Box> { + use tokio::net::UnixListener; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + let std_listener = std::os::unix::net::UnixListener::bind(&sock_path).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixListener::from_std(std_listener); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_stream_from_std_panic_caller() -> Result<(), Box> { + use tokio::net::UnixStream; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + let _std_listener = std::os::unix::net::UnixListener::bind(&sock_path).unwrap(); + let std_stream = std::os::unix::net::UnixStream::connect(&sock_path).unwrap(); + + let panic_location_file = test_panic(|| { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixStream::from_std(std_stream); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(unix)] +fn unix_datagram_from_std_panic_caller() -> Result<(), Box> { + use std::os::unix::net::UnixDatagram as StdUDS; + use tokio::net::UnixDatagram; + + let dir = tempfile::tempdir().unwrap(); + let sock_path = dir.path().join("socket"); + + // Bind the socket to a filesystem path + // /let socket_path = tmp.path().join("socket"); + let std_socket = StdUDS::bind(&sock_path).unwrap(); + std_socket.set_nonblocking(true).unwrap(); + + let panic_location_file = test_panic(move || { + let rt = runtime_without_io(); + rt.block_on(async { + let _ = UnixDatagram::from_std(std_socket); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +#[cfg(windows)] +fn server_options_max_instances_panic_caller() -> Result<(), Box> { + use tokio::net::windows::named_pipe::ServerOptions; + + let panic_location_file = test_panic(move || { + let rt = runtime_without_io(); + rt.block_on(async { + let mut options = ServerOptions::new(); + options.max_instances(255); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +// Runtime without `enable_io` so it has no IO driver set. +fn runtime_without_io() -> Runtime { + Builder::new_current_thread().build().unwrap() +} diff --git a/third_party/rust/tokio/tests/net_unix_pipe.rs b/third_party/rust/tokio/tests/net_unix_pipe.rs new file mode 100644 index 000000000000..c96d6e70fbde --- /dev/null +++ b/third_party/rust/tokio/tests/net_unix_pipe.rs @@ -0,0 +1,429 @@ +#![cfg(feature = "full")] +#![cfg(unix)] + +use tokio::io::{AsyncReadExt, AsyncWriteExt, Interest}; +use tokio::net::unix::pipe; +use tokio_test::task; +use tokio_test::{assert_err, assert_ok, assert_pending, assert_ready_ok}; + +use std::fs::File; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::io::AsRawFd; +use std::path::{Path, PathBuf}; + +/// Helper struct which will clean up temporary files once dropped. +struct TempFifo { + path: PathBuf, + _dir: tempfile::TempDir, +} + +impl TempFifo { + fn new(name: &str) -> io::Result { + let dir = tempfile::Builder::new() + .prefix("tokio-fifo-tests") + .tempdir()?; + let path = dir.path().join(name); + nix::unistd::mkfifo(&path, nix::sys::stat::Mode::S_IRWXU)?; + + Ok(TempFifo { path, _dir: dir }) + } +} + +impl AsRef for TempFifo { + fn as_ref(&self) -> &Path { + self.path.as_ref() + } +} + +#[tokio::test] +async fn fifo_simple_send() -> io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + let fifo = TempFifo::new("simple_send")?; + + // Create a reading task which should wait for data from the pipe. + let mut reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + let mut read_fut = task::spawn(async move { + let mut buf = vec![0; DATA.len()]; + reader.read_exact(&mut buf).await?; + Ok::<_, io::Error>(buf) + }); + assert_pending!(read_fut.poll()); + + let mut writer = pipe::OpenOptions::new().open_sender(&fifo)?; + writer.write_all(DATA).await?; + + // Let the IO driver poll events for the reader. + while !read_fut.is_woken() { + tokio::task::yield_now().await; + } + + // Reading task should be ready now. + let read_data = assert_ready_ok!(read_fut.poll()); + assert_eq!(&read_data, DATA); + + Ok(()) +} + +#[tokio::test] +#[cfg(target_os = "linux")] +async fn fifo_simple_send_sender_first() -> io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + // Create a new fifo file with *no reading ends open*. + let fifo = TempFifo::new("simple_send_sender_first")?; + + // Simple `open_sender` should fail with ENXIO (no such device or address). + let err = assert_err!(pipe::OpenOptions::new().open_sender(&fifo)); + assert_eq!(err.raw_os_error(), Some(libc::ENXIO)); + + // `open_sender` in read-write mode should succeed and the pipe should be ready to write. + let mut writer = pipe::OpenOptions::new() + .read_write(true) + .open_sender(&fifo)?; + writer.write_all(DATA).await?; + + // Read the written data and validate. + let mut reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + let mut read_data = vec![0; DATA.len()]; + reader.read_exact(&mut read_data).await?; + assert_eq!(&read_data, DATA); + + Ok(()) +} + +// Opens a FIFO file, write and *close the writer*. +async fn write_and_close(path: impl AsRef, msg: &[u8]) -> io::Result<()> { + let mut writer = pipe::OpenOptions::new().open_sender(path)?; + writer.write_all(msg).await?; + drop(writer); // Explicit drop. + Ok(()) +} + +/// Checks EOF behavior with single reader and writers sequentially opening +/// and closing a FIFO. +#[tokio::test] +async fn fifo_multiple_writes() -> io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + let fifo = TempFifo::new("fifo_multiple_writes")?; + + let mut reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + + write_and_close(&fifo, DATA).await?; + let ev = reader.ready(Interest::READABLE).await?; + assert!(ev.is_readable()); + let mut read_data = vec![0; DATA.len()]; + assert_ok!(reader.read_exact(&mut read_data).await); + + // Check that reader hits EOF. + let err = assert_err!(reader.read_exact(&mut read_data).await); + assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof); + + // Write more data and read again. + write_and_close(&fifo, DATA).await?; + assert_ok!(reader.read_exact(&mut read_data).await); + + Ok(()) +} + +/// Checks behavior of a resilient reader (Receiver in O_RDWR access mode) +/// with writers sequentially opening and closing a FIFO. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn fifo_resilient_reader() -> io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + let fifo = TempFifo::new("fifo_resilient_reader")?; + + // Open reader in read-write access mode. + let mut reader = pipe::OpenOptions::new() + .read_write(true) + .open_receiver(&fifo)?; + + write_and_close(&fifo, DATA).await?; + let ev = reader.ready(Interest::READABLE).await?; + let mut read_data = vec![0; DATA.len()]; + reader.read_exact(&mut read_data).await?; + + // Check that reader didn't hit EOF. + assert!(!ev.is_read_closed()); + + // Resilient reader can asynchronously wait for the next writer. + let mut second_read_fut = task::spawn(reader.read_exact(&mut read_data)); + assert_pending!(second_read_fut.poll()); + + // Write more data and read again. + write_and_close(&fifo, DATA).await?; + assert_ok!(second_read_fut.await); + + Ok(()) +} + +#[tokio::test] +async fn open_detects_not_a_fifo() -> io::Result<()> { + let dir = tempfile::Builder::new() + .prefix("tokio-fifo-tests") + .tempdir() + .unwrap(); + let path = dir.path().join("not_a_fifo"); + + // Create an ordinary file. + File::create(&path)?; + + // Check if Sender detects invalid file type. + let err = assert_err!(pipe::OpenOptions::new().open_sender(&path)); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + + // Check if Receiver detects invalid file type. + let err = assert_err!(pipe::OpenOptions::new().open_sender(&path)); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + + Ok(()) +} + +#[tokio::test] +async fn from_file() -> io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + let fifo = TempFifo::new("from_file")?; + + // Construct a Receiver from a File. + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo)?; + let mut reader = pipe::Receiver::from_file(file)?; + + // Construct a Sender from a File. + let file = std::fs::OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo)?; + let mut writer = pipe::Sender::from_file(file)?; + + // Write and read some data to test async. + let mut read_fut = task::spawn(async move { + let mut buf = vec![0; DATA.len()]; + reader.read_exact(&mut buf).await?; + Ok::<_, io::Error>(buf) + }); + assert_pending!(read_fut.poll()); + + writer.write_all(DATA).await?; + + let read_data = assert_ok!(read_fut.await); + assert_eq!(&read_data, DATA); + + Ok(()) +} + +#[tokio::test] +async fn from_file_detects_not_a_fifo() -> io::Result<()> { + let dir = tempfile::Builder::new() + .prefix("tokio-fifo-tests") + .tempdir() + .unwrap(); + let path = dir.path().join("not_a_fifo"); + + // Create an ordinary file. + File::create(&path)?; + + // Check if Sender detects invalid file type. + let file = std::fs::OpenOptions::new().write(true).open(&path)?; + let err = assert_err!(pipe::Sender::from_file(file)); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + + // Check if Receiver detects invalid file type. + let file = std::fs::OpenOptions::new().read(true).open(&path)?; + let err = assert_err!(pipe::Receiver::from_file(file)); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + + Ok(()) +} + +#[tokio::test] +async fn from_file_detects_wrong_access_mode() -> io::Result<()> { + let fifo = TempFifo::new("wrong_access_mode")?; + + // Open a read end to open the fifo for writing. + let _reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + + // Check if Receiver detects write-only access mode. + let wronly = std::fs::OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo)?; + let err = assert_err!(pipe::Receiver::from_file(wronly)); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + + // Check if Sender detects read-only access mode. + let rdonly = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo)?; + let err = assert_err!(pipe::Sender::from_file(rdonly)); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + + Ok(()) +} + +fn is_nonblocking(fd: &T) -> io::Result { + let flags = nix::fcntl::fcntl(fd.as_raw_fd(), nix::fcntl::F_GETFL)?; + Ok((flags & libc::O_NONBLOCK) != 0) +} + +#[tokio::test] +async fn from_file_sets_nonblock() -> io::Result<()> { + let fifo = TempFifo::new("sets_nonblock")?; + + // Open read and write ends to let blocking files open. + let _reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + let _writer = pipe::OpenOptions::new().open_sender(&fifo)?; + + // Check if Receiver sets the pipe in non-blocking mode. + let rdonly = std::fs::OpenOptions::new().read(true).open(&fifo)?; + assert!(!is_nonblocking(&rdonly)?); + let reader = pipe::Receiver::from_file(rdonly)?; + assert!(is_nonblocking(&reader)?); + + // Check if Sender sets the pipe in non-blocking mode. + let wronly = std::fs::OpenOptions::new().write(true).open(&fifo)?; + assert!(!is_nonblocking(&wronly)?); + let writer = pipe::Sender::from_file(wronly)?; + assert!(is_nonblocking(&writer)?); + + Ok(()) +} + +fn writable_by_poll(writer: &pipe::Sender) -> bool { + task::spawn(writer.writable()).poll().is_ready() +} + +#[tokio::test] +async fn try_read_write() -> io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + // Create a pipe pair over a fifo file. + let fifo = TempFifo::new("try_read_write")?; + let reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + let writer = pipe::OpenOptions::new().open_sender(&fifo)?; + + // Fill the pipe buffer with `try_write`. + let mut write_data = Vec::new(); + while writable_by_poll(&writer) { + match writer.try_write(DATA) { + Ok(n) => write_data.extend(&DATA[..n]), + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::WouldBlock); + break; + } + } + } + + // Drain the pipe buffer with `try_read`. + let mut read_data = vec![0; write_data.len()]; + let mut i = 0; + while i < write_data.len() { + reader.readable().await?; + match reader.try_read(&mut read_data[i..]) { + Ok(n) => i += n, + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::WouldBlock); + continue; + } + } + } + + assert_eq!(read_data, write_data); + + Ok(()) +} + +#[tokio::test] +async fn try_read_write_vectored() -> io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + // Create a pipe pair over a fifo file. + let fifo = TempFifo::new("try_read_write_vectored")?; + let reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + let writer = pipe::OpenOptions::new().open_sender(&fifo)?; + + let write_bufs: Vec<_> = DATA.chunks(3).map(io::IoSlice::new).collect(); + + // Fill the pipe buffer with `try_write_vectored`. + let mut write_data = Vec::new(); + while writable_by_poll(&writer) { + match writer.try_write_vectored(&write_bufs) { + Ok(n) => write_data.extend(&DATA[..n]), + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::WouldBlock); + break; + } + } + } + + // Drain the pipe buffer with `try_read_vectored`. + let mut read_data = vec![0; write_data.len()]; + let mut i = 0; + while i < write_data.len() { + reader.readable().await?; + + let mut read_bufs: Vec<_> = read_data[i..] + .chunks_mut(0x10000) + .map(io::IoSliceMut::new) + .collect(); + match reader.try_read_vectored(&mut read_bufs) { + Ok(n) => i += n, + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::WouldBlock); + continue; + } + } + } + + assert_eq!(read_data, write_data); + + Ok(()) +} + +#[tokio::test] +async fn try_read_buf() -> std::io::Result<()> { + const DATA: &[u8] = b"this is some data to write to the fifo"; + + // Create a pipe pair over a fifo file. + let fifo = TempFifo::new("try_read_write_vectored")?; + let reader = pipe::OpenOptions::new().open_receiver(&fifo)?; + let writer = pipe::OpenOptions::new().open_sender(&fifo)?; + + // Fill the pipe buffer with `try_write`. + let mut write_data = Vec::new(); + while writable_by_poll(&writer) { + match writer.try_write(DATA) { + Ok(n) => write_data.extend(&DATA[..n]), + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::WouldBlock); + break; + } + } + } + + // Drain the pipe buffer with `try_read_buf`. + let mut read_data = vec![0; write_data.len()]; + let mut i = 0; + while i < write_data.len() { + reader.readable().await?; + match reader.try_read_buf(&mut read_data) { + Ok(n) => i += n, + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::WouldBlock); + continue; + } + } + } + + assert_eq!(read_data, write_data); + + Ok(()) +} diff --git a/third_party/rust/tokio/tests/no_rt.rs b/third_party/rust/tokio/tests/no_rt.rs index 64e56f4d43bf..747fab6af7dc 100644 --- a/third_party/rust/tokio/tests/no_rt.rs +++ b/third_party/rust/tokio/tests/no_rt.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery use tokio::net::TcpStream; use tokio::sync::oneshot; diff --git a/third_party/rust/tokio/tests/process_kill_on_drop.rs b/third_party/rust/tokio/tests/process_kill_on_drop.rs index 658e4addd610..d919b1a27fec 100644 --- a/third_party/rust/tokio/tests/process_kill_on_drop.rs +++ b/third_party/rust/tokio/tests/process_kill_on_drop.rs @@ -12,7 +12,7 @@ use tokio_test::assert_ok; #[tokio::test] async fn kill_on_drop() { let mut cmd = Command::new("bash"); - cmd.args(&[ + cmd.args([ "-c", " # Fork another child that won't get killed diff --git a/third_party/rust/tokio/tests/process_raw_handle.rs b/third_party/rust/tokio/tests/process_raw_handle.rs index 727e66d65e12..fbe22b07c880 100644 --- a/third_party/rust/tokio/tests/process_raw_handle.rs +++ b/third_party/rust/tokio/tests/process_raw_handle.rs @@ -3,7 +3,7 @@ #![cfg(windows)] use tokio::process::Command; -use winapi::um::processthreadsapi::GetProcessId; +use windows_sys::Win32::System::Threading::GetProcessId; #[tokio::test] async fn obtain_raw_handle() { diff --git a/third_party/rust/tokio/tests/process_smoke.rs b/third_party/rust/tokio/tests/process_smoke.rs index fae5793fab7a..81d2020a079f 100644 --- a/third_party/rust/tokio/tests/process_smoke.rs +++ b/third_party/rust/tokio/tests/process_smoke.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi cannot run system commands use tokio::process::Command; use tokio_test::assert_ok; diff --git a/third_party/rust/tokio/tests/rt_basic.rs b/third_party/rust/tokio/tests/rt_basic.rs index cc6ac677280a..6caf0a44bf8e 100644 --- a/third_party/rust/tokio/tests/rt_basic.rs +++ b/third_party/rust/tokio/tests/rt_basic.rs @@ -178,6 +178,7 @@ fn drop_tasks_in_context() { } #[test] +#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] #[should_panic(expected = "boom")] fn wake_in_drop_after_panic() { let (tx, rx) = oneshot::channel::<()>(); @@ -238,6 +239,7 @@ fn spawn_two() { } } +#[cfg_attr(tokio_wasi, ignore = "WASI: std::thread::spawn not supported")] #[test] fn spawn_remote() { let rt = rt(); @@ -274,6 +276,7 @@ fn spawn_remote() { } #[test] +#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] #[should_panic( expected = "A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers." )] @@ -288,6 +291,155 @@ fn timeout_panics_when_no_time_handle() { }); } +#[cfg(tokio_unstable)] +mod unstable { + use tokio::runtime::{Builder, RngSeed, UnhandledPanic}; + + #[test] + #[should_panic( + expected = "a spawned task panicked and the runtime is configured to shut down on unhandled panic" + )] + fn shutdown_on_panic() { + let rt = Builder::new_current_thread() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .build() + .unwrap(); + + rt.block_on(async { + tokio::spawn(async { + panic!("boom"); + }); + + futures::future::pending::<()>().await; + }) + } + + #[test] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] + fn spawns_do_nothing() { + use std::sync::Arc; + + let rt = Builder::new_current_thread() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .build() + .unwrap(); + + let rt1 = Arc::new(rt); + let rt2 = rt1.clone(); + + let _ = std::thread::spawn(move || { + rt2.block_on(async { + tokio::spawn(async { + panic!("boom"); + }); + + futures::future::pending::<()>().await; + }) + }) + .join(); + + let task = rt1.spawn(async {}); + let res = futures::executor::block_on(task); + assert!(res.is_err()); + } + + #[test] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] + fn shutdown_all_concurrent_block_on() { + const N: usize = 2; + use std::sync::{mpsc, Arc}; + + let rt = Builder::new_current_thread() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .build() + .unwrap(); + + let rt = Arc::new(rt); + let mut ths = vec![]; + let (tx, rx) = mpsc::channel(); + + for _ in 0..N { + let rt = rt.clone(); + let tx = tx.clone(); + ths.push(std::thread::spawn(move || { + rt.block_on(async { + tx.send(()).unwrap(); + futures::future::pending::<()>().await; + }); + })); + } + + for _ in 0..N { + rx.recv().unwrap(); + } + + rt.spawn(async { + panic!("boom"); + }); + + for th in ths { + assert!(th.join().is_err()); + } + } + + #[test] + fn rng_seed() { + let seed = b"bytes used to generate seed"; + let rt1 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values = rt1.block_on(async { + let rand_1 = tokio::macros::support::thread_rng_n(100); + let rand_2 = tokio::macros::support::thread_rng_n(100); + + (rand_1, rand_2) + }); + + let rt2 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values = rt2.block_on(async { + let rand_1 = tokio::macros::support::thread_rng_n(100); + let rand_2 = tokio::macros::support::thread_rng_n(100); + + (rand_1, rand_2) + }); + + assert_eq!(rt1_values, rt2_values); + } + + #[test] + fn rng_seed_multi_enter() { + let seed = b"bytes used to generate seed"; + + fn two_rand_values() -> (u32, u32) { + let rand_1 = tokio::macros::support::thread_rng_n(100); + let rand_2 = tokio::macros::support::thread_rng_n(100); + + (rand_1, rand_2) + } + + let rt1 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt1_values_1 = rt1.block_on(async { two_rand_values() }); + let rt1_values_2 = rt1.block_on(async { two_rand_values() }); + + let rt2 = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(seed)) + .build() + .unwrap(); + let rt2_values_1 = rt2.block_on(async { two_rand_values() }); + let rt2_values_2 = rt2.block_on(async { two_rand_values() }); + + assert_eq!(rt1_values_1, rt2_values_1); + assert_eq!(rt1_values_2, rt2_values_2); + } +} + fn rt() -> Runtime { tokio::runtime::Builder::new_current_thread() .enable_all() diff --git a/third_party/rust/tokio/tests/rt_common.rs b/third_party/rust/tokio/tests/rt_common.rs index cb1d0f661522..9c6add047a72 100644 --- a/third_party/rust/tokio/tests/rt_common.rs +++ b/third_party/rust/tokio/tests/rt_common.rs @@ -2,13 +2,16 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "full")] -// Tests to run on both current-thread & thread-pool runtime variants. +// Tests to run on both current-thread & multi-thread runtime variants. macro_rules! rt_test { ($($t:tt)*) => { mod current_thread_scheduler { $($t)* + #[cfg(not(target_os="wasi"))] + const NUM_WORKERS: usize = 1; + fn rt() -> Arc { tokio::runtime::Builder::new_current_thread() .enable_all() @@ -18,9 +21,12 @@ macro_rules! rt_test { } } + #[cfg(not(tokio_wasi))] // Wasi doesn't support threads mod threaded_scheduler_4_threads { $($t)* + const NUM_WORKERS: usize = 4; + fn rt() -> Arc { tokio::runtime::Builder::new_multi_thread() .worker_threads(4) @@ -31,9 +37,12 @@ macro_rules! rt_test { } } + #[cfg(not(tokio_wasi))] // Wasi doesn't support threads mod threaded_scheduler_1_thread { $($t)* + const NUM_WORKERS: usize = 1; + fn rt() -> Arc { tokio::runtime::Builder::new_multi_thread() .worker_threads(1) @@ -55,18 +64,30 @@ fn send_sync_bound() { } rt_test! { - use tokio::net::{TcpListener, TcpStream, UdpSocket}; + #[cfg(not(target_os="wasi"))] + use tokio::net::{TcpListener, TcpStream}; + #[cfg(not(target_os="wasi"))] use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::runtime::Runtime; use tokio::sync::oneshot; use tokio::{task, time}; - use tokio_test::{assert_err, assert_ok}; + + #[cfg(not(target_os="wasi"))] + use tokio_test::assert_err; + use tokio_test::assert_ok; use futures::future::poll_fn; use std::future::Future; use std::pin::Pin; - use std::sync::{mpsc, Arc}; + + #[cfg(not(target_os="wasi"))] + use std::sync::mpsc; + + use std::sync::Arc; use std::task::{Context, Poll}; + + #[cfg(not(target_os="wasi"))] use std::thread; use std::time::{Duration, Instant}; @@ -83,6 +104,7 @@ rt_test! { } + #[cfg(not(target_os="wasi"))] #[test] fn block_on_async() { let rt = rt(); @@ -164,6 +186,7 @@ rt_test! { assert_eq!(out, "ZOMG"); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_many_from_block_on() { use tokio::sync::mpsc; @@ -214,6 +237,7 @@ rt_test! { } } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_many_from_task() { use tokio::sync::mpsc; @@ -226,14 +250,6 @@ rt_test! { tokio::spawn(async move { let (done_tx, mut done_rx) = mpsc::unbounded_channel(); - /* - for _ in 0..100 { - tokio::spawn(async move { }); - } - - tokio::task::yield_now().await; - */ - let mut txs = (0..ITER) .map(|i| { let (tx, rx) = oneshot::channel(); @@ -274,6 +290,31 @@ rt_test! { } } + #[test] + fn spawn_one_from_block_on_called_on_handle() { + let rt = rt(); + let (tx, rx) = oneshot::channel(); + + #[allow(clippy::async_yields_async)] + let handle = rt.handle().block_on(async { + tokio::spawn(async move { + tx.send("ZOMG").unwrap(); + "DONE" + }) + }); + + let out = rt.block_on(async { + let msg = assert_ok!(rx.await); + + let out = assert_ok!(handle.await); + assert_eq!(out, "DONE"); + + msg + }); + + assert_eq!(out, "ZOMG"); + } + #[test] fn spawn_await_chain() { let rt = rt(); @@ -329,6 +370,7 @@ rt_test! { assert_eq!(out, "ZOMG"); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn complete_block_on_under_load() { let rt = rt(); @@ -352,6 +394,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn complete_task_under_load() { let rt = rt(); @@ -381,6 +424,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_from_other_thread_idle() { let rt = rt(); @@ -401,6 +445,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_from_other_thread_under_load() { let rt = rt(); @@ -461,6 +506,7 @@ rt_test! { assert!(now.elapsed() >= dur); } + #[cfg(not(target_os="wasi"))] // Wasi does not support bind #[test] fn block_on_socket() { let rt = rt(); @@ -481,6 +527,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_from_blocking() { let rt = rt(); @@ -496,6 +543,7 @@ rt_test! { assert_eq!(out, "hello") } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn spawn_blocking_from_blocking() { let rt = rt(); @@ -511,6 +559,7 @@ rt_test! { assert_eq!(out, "hello") } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn sleep_from_blocking() { let rt = rt(); @@ -531,6 +580,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support bind #[test] fn socket_from_blocking() { let rt = rt(); @@ -554,6 +604,7 @@ rt_test! { }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn always_active_parker() { // This test it to show that we will always have @@ -600,6 +651,7 @@ rt_test! { // concern. There also isn't a great/obvious solution to take. For now, the // test is disabled. #[cfg(not(windows))] + #[cfg(not(target_os="wasi"))] // Wasi does not support bind or threads fn io_driver_called_when_under_load() { let rt = rt(); @@ -607,7 +659,12 @@ rt_test! { for _ in 0..100 { rt.spawn(async { loop { - tokio::task::yield_now().await; + // Don't use Tokio's `yield_now()` to avoid special defer + // logic. + futures::future::poll_fn::<(), _>(|cx| { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + }).await; } }); } @@ -635,6 +692,113 @@ rt_test! { }); } + /// Tests that yielded tasks are not scheduled until **after** resource + /// drivers are polled. + /// + /// The OS does not guarantee when I/O events are delivered, so there may be + /// more yields than anticipated. This makes the test slightly flaky. To + /// help avoid flakiness, we run the test 10 times and only fail it after + /// 10 failures in a row. + /// + /// Note that if the test fails by panicking rather than by returning false, + /// then we fail it immediately. That kind of failure should not happen + /// spuriously. + #[test] + #[cfg(not(target_os="wasi"))] + fn yield_defers_until_park() { + for _ in 0..10 { + if yield_defers_until_park_inner() { + // test passed + return; + } + + // Wait a bit and run the test again. + std::thread::sleep(std::time::Duration::from_secs(2)); + } + + panic!("yield_defers_until_park is failing consistently"); + } + + /// Implementation of `yield_defers_until_park` test. Returns `true` if the + /// test passed. + #[cfg(not(target_os="wasi"))] + fn yield_defers_until_park_inner() -> bool { + use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; + use std::sync::Barrier; + + let rt = rt(); + + let flag = Arc::new(AtomicBool::new(false)); + let barrier = Arc::new(Barrier::new(NUM_WORKERS)); + + rt.block_on(async { + // Make sure other workers cannot steal tasks + #[allow(clippy::reversed_empty_ranges)] + for _ in 0..(NUM_WORKERS-1) { + let flag = flag.clone(); + let barrier = barrier.clone(); + + tokio::spawn(async move { + barrier.wait(); + + while !flag.load(SeqCst) { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + }); + } + + barrier.wait(); + + let (fail_test, fail_test_recv) = oneshot::channel::<()>(); + + let jh = tokio::spawn(async move { + // Create a TCP litener + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + tokio::join!( + async { + // Done in a blocking manner intentionally. + let _socket = std::net::TcpStream::connect(addr).unwrap(); + + // Yield until connected + let mut cnt = 0; + while !flag.load(SeqCst){ + tokio::task::yield_now().await; + cnt += 1; + + if cnt >= 10 { + // yielded too many times; report failure and + // sleep forever so that the `fail_test` branch + // of the `select!` below triggers. + let _ = fail_test.send(()); + futures::future::pending::<()>().await; + break; + } + } + }, + async { + let _ = listener.accept().await.unwrap(); + flag.store(true, SeqCst); + } + ); + }); + + // Wait until the spawned task completes or fails. If no message is + // sent on `fail_test`, then the test succeeds. Otherwise, it fails. + let success = fail_test_recv.await.is_err(); + + if success { + // Check for panics in spawned task. + jh.abort(); + jh.await.unwrap(); + } + + success + }) + } + + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn client_server_block_on() { let rt = rt(); @@ -646,6 +810,7 @@ rt_test! { assert_err!(rx.try_recv()); } + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support threads or panic recovery")] #[test] fn panic_in_task() { let rt = rt(); @@ -674,11 +839,13 @@ rt_test! { #[test] #[should_panic] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] fn panic_in_block_on() { let rt = rt(); rt.block_on(async { panic!() }); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads async fn yield_once() { let mut yielded = false; poll_fn(|cx| { @@ -748,7 +915,11 @@ rt_test! { #[test] fn wake_while_rt_is_dropping() { - use tokio::task; + use tokio::sync::Barrier; + use core::sync::atomic::{AtomicBool, Ordering}; + + let drop_triggered = Arc::new(AtomicBool::new(false)); + let set_drop_triggered = drop_triggered.clone(); struct OnDrop(F); @@ -762,17 +933,21 @@ rt_test! { let (tx2, rx2) = oneshot::channel(); let (tx3, rx3) = oneshot::channel(); - let rt = rt(); + let barrier = Arc::new(Barrier::new(4)); + let barrier1 = barrier.clone(); + let barrier2 = barrier.clone(); + let barrier3 = barrier.clone(); - let h1 = rt.clone(); + let rt = rt(); rt.spawn(async move { // Ensure a waker gets stored in oneshot 1. - let _ = rx1.await; + let _ = tokio::join!(rx1, barrier1.wait()); tx3.send(()).unwrap(); }); rt.spawn(async move { + let h1 = tokio::runtime::Handle::current(); // When this task is dropped, we'll be "closing remotes". // We spawn a new task that owns the `tx1`, to move its Drop // out of here. @@ -785,36 +960,40 @@ rt_test! { h1.spawn(async move { tx1.send(()).unwrap(); }); + // Just a sanity check that this entire thing actually happened + set_drop_triggered.store(true, Ordering::Relaxed); }); - let _ = rx2.await; + let _ = tokio::join!(rx2, barrier2.wait()); }); rt.spawn(async move { - let _ = rx3.await; + let _ = tokio::join!(rx3, barrier3.wait()); // We'll never get here, but once task 3 drops, this will // force task 2 to re-schedule since it's waiting on oneshot 2. tx2.send(()).unwrap(); }); - // Tick the loop - rt.block_on(async { - task::yield_now().await; - }); + // Wait until every oneshot channel has been polled. + rt.block_on(barrier.wait()); // Drop the rt drop(rt); + + // Make sure that the spawn actually happened + assert!(drop_triggered.load(Ordering::Relaxed)); } + #[cfg(not(target_os="wasi"))] // Wasi doesn't support UDP or bind() #[test] fn io_notify_while_shutting_down() { - use std::net::Ipv6Addr; + use tokio::net::UdpSocket; use std::sync::Arc; for _ in 1..10 { let runtime = rt(); runtime.block_on(async { - let socket = UdpSocket::bind((Ipv6Addr::LOCALHOST, 0)).await.unwrap(); + let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); let addr = socket.local_addr().unwrap(); let send_half = Arc::new(socket); let recv_half = send_half.clone(); @@ -840,6 +1019,7 @@ rt_test! { } } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn shutdown_timeout() { let (tx, rx) = oneshot::channel(); @@ -857,6 +1037,7 @@ rt_test! { Arc::try_unwrap(runtime).unwrap().shutdown_timeout(Duration::from_millis(100)); } + #[cfg(not(target_os="wasi"))] // Wasi does not support threads #[test] fn shutdown_timeout_0() { let runtime = rt(); @@ -888,6 +1069,7 @@ rt_test! { // See https://github.com/rust-lang/rust/issues/74875 #[test] #[cfg(not(windows))] + #[cfg_attr(tokio_wasi, ignore = "Wasi does not support threads")] fn runtime_in_thread_local() { use std::cell::RefCell; use std::thread; @@ -907,6 +1089,7 @@ rt_test! { }).join().unwrap(); } + #[cfg(not(target_os="wasi"))] // Wasi does not support bind async fn client_server(tx: mpsc::Sender<()>) { let server = assert_ok!(TcpListener::bind("127.0.0.1:0").await); @@ -931,6 +1114,7 @@ rt_test! { tx.send(()).unwrap(); } + #[cfg(not(tokio_wasi))] // Wasi does not support bind #[test] fn local_set_block_on_socket() { let rt = rt(); @@ -952,6 +1136,7 @@ rt_test! { }); } + #[cfg(not(tokio_wasi))] // Wasi does not support bind #[test] fn local_set_client_server_block_on() { let rt = rt(); @@ -965,6 +1150,7 @@ rt_test! { assert_err!(rx.try_recv()); } + #[cfg(not(tokio_wasi))] // Wasi does not support bind async fn client_server_local(tx: mpsc::Sender<()>) { let server = assert_ok!(TcpListener::bind("127.0.0.1:0").await); @@ -992,22 +1178,22 @@ rt_test! { #[test] fn coop() { use std::task::Poll::Ready; + use tokio::sync::mpsc; let rt = rt(); rt.block_on(async { - // Create a bunch of tasks - let mut tasks = (0..1_000).map(|_| { - tokio::spawn(async { }) - }).collect::>(); + let (send, mut recv) = mpsc::unbounded_channel(); - // Hope that all the tasks complete... - time::sleep(Duration::from_millis(100)).await; + // Send a bunch of messages. + for _ in 0..1_000 { + send.send(()).unwrap(); + } poll_fn(|cx| { - // At least one task should not be ready - for task in &mut tasks { - if Pin::new(task).poll(cx).is_pending() { + // At least one response should return pending. + for _ in 0..1_000 { + if recv.poll_recv(cx).is_pending() { return Ready(()); } } @@ -1020,22 +1206,22 @@ rt_test! { #[test] fn coop_unconstrained() { use std::task::Poll::Ready; + use tokio::sync::mpsc; let rt = rt(); rt.block_on(async { - // Create a bunch of tasks - let mut tasks = (0..1_000).map(|_| { - tokio::spawn(async { }) - }).collect::>(); + let (send, mut recv) = mpsc::unbounded_channel(); - // Hope that all the tasks complete... - time::sleep(Duration::from_millis(100)).await; + // Send a bunch of messages. + for _ in 0..1_000 { + send.send(()).unwrap(); + } tokio::task::unconstrained(poll_fn(|cx| { - // All the tasks should be ready - for task in &mut tasks { - assert!(Pin::new(task).poll(cx).is_ready()); + // All the responses should be ready. + for _ in 0..1_000 { + assert_eq!(recv.poll_recv(cx), Poll::Ready(Some(()))); } Ready(()) @@ -1043,6 +1229,31 @@ rt_test! { }); } + #[cfg(tokio_unstable)] + #[test] + fn coop_consume_budget() { + let rt = rt(); + + rt.block_on(async { + poll_fn(|cx| { + let counter = Arc::new(std::sync::Mutex::new(0)); + let counter_clone = Arc::clone(&counter); + let mut worker = Box::pin(async move { + // Consume the budget until a yield happens + for _ in 0..1000 { + *counter.lock().unwrap() += 1; + task::consume_budget().await + } + }); + // Assert that the worker was yielded and it didn't manage + // to finish the whole work (assuming the total budget of 128) + assert!(Pin::new(&mut worker).poll(cx).is_pending()); + assert!(*counter_clone.lock().unwrap() < 1000); + std::task::Poll::Ready(()) + }).await; + }); + } + // Tests that the "next task" scheduler optimization is not able to starve // other tasks. #[test] @@ -1060,7 +1271,7 @@ rt_test! { let (spawned_tx, mut spawned_rx) = mpsc::unbounded_channel(); let mut tasks = vec![]; - // Spawn a bunch of tasks that ping ping between each other to + // Spawn a bunch of tasks that ping-pong between each other to // saturate the runtime. for _ in 0..NUM { let (tx1, mut rx1) = mpsc::unbounded_channel(); @@ -1106,4 +1317,39 @@ rt_test! { } }); } + + #[test] + #[cfg(not(target_os="wasi"))] + fn shutdown_concurrent_spawn() { + const NUM_TASKS: usize = 10_000; + for _ in 0..5 { + let (tx, rx) = std::sync::mpsc::channel(); + let rt = rt(); + + let mut txs = vec![]; + + for _ in 0..NUM_TASKS { + let (tx, rx) = tokio::sync::oneshot::channel(); + txs.push(tx); + rt.spawn(async move { + rx.await.unwrap(); + }); + } + + // Prime the tasks + rt.block_on(async { tokio::task::yield_now().await }); + + let th = std::thread::spawn(move || { + tx.send(()).unwrap(); + for tx in txs.drain(..) { + let _ = tx.send(()); + } + }); + + rx.recv().unwrap(); + drop(rt); + + th.join().unwrap(); + } + } } diff --git a/third_party/rust/tokio/tests/rt_handle.rs b/third_party/rust/tokio/tests/rt_handle.rs new file mode 100644 index 000000000000..34c99cdaead5 --- /dev/null +++ b/third_party/rust/tokio/tests/rt_handle.rs @@ -0,0 +1,67 @@ +#![warn(rust_2018_idioms)] +#![cfg(feature = "full")] + +use tokio::runtime::Runtime; + +#[test] +fn basic_enter() { + let rt1 = rt(); + let rt2 = rt(); + + let enter1 = rt1.enter(); + let enter2 = rt2.enter(); + + drop(enter2); + drop(enter1); +} + +#[test] +#[should_panic] +fn interleave_enter_different_rt() { + let rt1 = rt(); + let rt2 = rt(); + + let enter1 = rt1.enter(); + let enter2 = rt2.enter(); + + drop(enter1); + drop(enter2); +} + +#[test] +#[should_panic] +fn interleave_enter_same_rt() { + let rt1 = rt(); + + let _enter1 = rt1.enter(); + let enter2 = rt1.enter(); + let enter3 = rt1.enter(); + + drop(enter2); + drop(enter3); +} + +#[test] +#[cfg(not(tokio_wasi))] +fn interleave_then_enter() { + let _ = std::panic::catch_unwind(|| { + let rt1 = rt(); + let rt2 = rt(); + + let enter1 = rt1.enter(); + let enter2 = rt2.enter(); + + drop(enter1); + drop(enter2); + }); + + // Can still enter + let rt3 = rt(); + let _enter = rt3.enter(); +} + +fn rt() -> Runtime { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() +} diff --git a/third_party/rust/tokio/tests/rt_handle_block_on.rs b/third_party/rust/tokio/tests/rt_handle_block_on.rs index 5c1d533a0104..5ec783e5588e 100644 --- a/third_party/rust/tokio/tests/rt_handle_block_on.rs +++ b/third_party/rust/tokio/tests/rt_handle_block_on.rs @@ -10,9 +10,10 @@ use std::time::Duration; use tokio::runtime::{Handle, Runtime}; use tokio::sync::mpsc; -use tokio::task::spawn_blocking; -use tokio::{fs, net, time}; +#[cfg(not(tokio_wasi))] +use tokio::{net, time}; +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads macro_rules! multi_threaded_rt_test { ($($t:tt)*) => { mod threaded_scheduler_4_threads_only { @@ -45,6 +46,7 @@ macro_rules! multi_threaded_rt_test { } } +#[cfg(not(tokio_wasi))] macro_rules! rt_test { ($($t:tt)*) => { mod current_thread_scheduler { @@ -124,7 +126,9 @@ fn unbounded_mpsc_channel() { }) } +#[cfg(not(tokio_wasi))] // Wasi doesn't support file operations or bind rt_test! { + use tokio::fs; // ==== spawn blocking futures ====== #[test] @@ -156,6 +160,7 @@ rt_test! { #[test] fn basic_spawn_blocking() { + use tokio::task::spawn_blocking; let rt = rt(); let _enter = rt.enter(); @@ -171,6 +176,7 @@ rt_test! { #[test] fn spawn_blocking_after_shutdown_fails() { + use tokio::task::spawn_blocking; let rt = rt(); let _enter = rt.enter(); rt.shutdown_timeout(Duration::from_secs(1000)); @@ -187,6 +193,7 @@ rt_test! { #[test] fn spawn_blocking_started_before_shutdown_continues() { + use tokio::task::spawn_blocking; let rt = rt(); let _enter = rt.enter(); @@ -412,6 +419,7 @@ rt_test! { } } +#[cfg(not(tokio_wasi))] multi_threaded_rt_test! { #[cfg(unix)] #[test] @@ -474,6 +482,7 @@ multi_threaded_rt_test! { // ==== utils ====== /// Create a new multi threaded runtime +#[cfg(not(tokio_wasi))] fn new_multi_thread(n: usize) -> Runtime { tokio::runtime::Builder::new_multi_thread() .worker_threads(n) @@ -496,37 +505,30 @@ where F: Fn(), { { - println!("current thread runtime"); - let rt = new_current_thread(); let _enter = rt.enter(); f(); - println!("current thread runtime after shutdown"); rt.shutdown_timeout(Duration::from_secs(1000)); f(); } + #[cfg(not(tokio_wasi))] { - println!("multi thread (1 thread) runtime"); - let rt = new_multi_thread(1); let _enter = rt.enter(); f(); - println!("multi thread runtime after shutdown"); rt.shutdown_timeout(Duration::from_secs(1000)); f(); } + #[cfg(not(tokio_wasi))] { - println!("multi thread (4 threads) runtime"); - let rt = new_multi_thread(4); let _enter = rt.enter(); f(); - println!("multi thread runtime after shutdown"); rt.shutdown_timeout(Duration::from_secs(1000)); f(); } diff --git a/third_party/rust/tokio/tests/rt_metrics.rs b/third_party/rust/tokio/tests/rt_metrics.rs index 0a26b80285dc..0fe839a2f975 100644 --- a/third_party/rust/tokio/tests/rt_metrics.rs +++ b/third_party/rust/tokio/tests/rt_metrics.rs @@ -1,23 +1,108 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", tokio_unstable))] +#![cfg(all(feature = "full", tokio_unstable, not(tokio_wasi)))] + +use std::future::Future; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use tokio::macros::support::poll_fn; use tokio::runtime::Runtime; +use tokio::task::consume_budget; use tokio::time::{self, Duration}; #[test] fn num_workers() { - let rt = basic(); + let rt = current_thread(); assert_eq!(1, rt.metrics().num_workers()); let rt = threaded(); assert_eq!(2, rt.metrics().num_workers()); } +#[test] +fn num_blocking_threads() { + let rt = current_thread(); + assert_eq!(0, rt.metrics().num_blocking_threads()); + let _ = rt.block_on(rt.spawn_blocking(move || {})); + assert_eq!(1, rt.metrics().num_blocking_threads()); +} + +#[test] +fn num_idle_blocking_threads() { + let rt = current_thread(); + assert_eq!(0, rt.metrics().num_idle_blocking_threads()); + let _ = rt.block_on(rt.spawn_blocking(move || {})); + rt.block_on(async { + time::sleep(Duration::from_millis(5)).await; + }); + + // We need to wait until the blocking thread has become idle. Usually 5ms is + // enough for this to happen, but not always. When it isn't enough, sleep + // for another second. We don't always wait for a whole second since we want + // the test suite to finish quickly. + // + // Note that the timeout for idle threads to be killed is 10 seconds. + if 0 == rt.metrics().num_idle_blocking_threads() { + rt.block_on(async { + time::sleep(Duration::from_secs(1)).await; + }); + } + + assert_eq!(1, rt.metrics().num_idle_blocking_threads()); +} + +#[test] +fn blocking_queue_depth() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + + assert_eq!(0, rt.metrics().blocking_queue_depth()); + + let ready = Arc::new(Mutex::new(())); + let guard = ready.lock().unwrap(); + + let ready_cloned = ready.clone(); + let wait_until_ready = move || { + let _unused = ready_cloned.lock().unwrap(); + }; + + let h1 = rt.spawn_blocking(wait_until_ready.clone()); + let h2 = rt.spawn_blocking(wait_until_ready); + assert!(rt.metrics().blocking_queue_depth() > 0); + + drop(guard); + + let _ = rt.block_on(h1); + let _ = rt.block_on(h2); + + assert_eq!(0, rt.metrics().blocking_queue_depth()); +} + +#[test] +fn active_tasks_count() { + let rt = current_thread(); + let metrics = rt.metrics(); + assert_eq!(0, metrics.active_tasks_count()); + rt.spawn(async move { + assert_eq!(1, metrics.active_tasks_count()); + }); + + let rt = threaded(); + let metrics = rt.metrics(); + assert_eq!(0, metrics.active_tasks_count()); + rt.spawn(async move { + assert_eq!(1, metrics.active_tasks_count()); + }); +} + #[test] fn remote_schedule_count() { use std::thread; - let rt = basic(); + let rt = current_thread(); let handle = rt.handle().clone(); let task = thread::spawn(move || { handle.spawn(async { @@ -48,13 +133,13 @@ fn remote_schedule_count() { #[test] fn worker_park_count() { - let rt = basic(); + let rt = current_thread(); let metrics = rt.metrics(); rt.block_on(async { time::sleep(Duration::from_millis(1)).await; }); drop(rt); - assert!(2 <= metrics.worker_park_count(0)); + assert!(1 <= metrics.worker_park_count(0)); let rt = threaded(); let metrics = rt.metrics(); @@ -71,13 +156,13 @@ fn worker_noop_count() { // There isn't really a great way to generate no-op parks as they happen as // false-positive events under concurrency. - let rt = basic(); + let rt = current_thread(); let metrics = rt.metrics(); rt.block_on(async { time::sleep(Duration::from_millis(1)).await; }); drop(rt); - assert!(2 <= metrics.worker_noop_count(0)); + assert!(0 < metrics.worker_noop_count(0)); let rt = threaded(); let metrics = rt.metrics(); @@ -85,8 +170,8 @@ fn worker_noop_count() { time::sleep(Duration::from_millis(1)).await; }); drop(rt); - assert!(1 <= metrics.worker_noop_count(0)); - assert!(1 <= metrics.worker_noop_count(1)); + assert!(0 < metrics.worker_noop_count(0)); + assert!(0 < metrics.worker_noop_count(1)); } #[test] @@ -113,7 +198,7 @@ fn worker_steal_count() { // scheduled" slot. tokio::spawn(async {}); - // Blocking receive on the channe. + // Blocking receive on the channel. rx.recv().unwrap(); }) .await @@ -133,7 +218,7 @@ fn worker_steal_count() { fn worker_poll_count() { const N: u64 = 5; - let rt = basic(); + let rt = current_thread(); let metrics = rt.metrics(); rt.block_on(async { for _ in 0..N { @@ -143,6 +228,12 @@ fn worker_poll_count() { drop(rt); assert_eq!(N, metrics.worker_poll_count(0)); + // Does not populate the histogram + assert!(!metrics.poll_count_histogram_enabled()); + for i in 0..10 { + assert_eq!(0, metrics.poll_count_histogram_bucket_count(0, i)); + } + let rt = threaded(); let metrics = rt.metrics(); rt.block_on(async { @@ -157,6 +248,126 @@ fn worker_poll_count() { .sum(); assert_eq!(N, n); + + // Does not populate the histogram + assert!(!metrics.poll_count_histogram_enabled()); + for n in 0..metrics.num_workers() { + for i in 0..10 { + assert_eq!(0, metrics.poll_count_histogram_bucket_count(n, i)); + } + } +} + +#[test] +fn worker_poll_count_histogram() { + const N: u64 = 5; + + let rts = [ + tokio::runtime::Builder::new_current_thread() + .enable_all() + .enable_metrics_poll_count_histogram() + .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear) + .metrics_poll_count_histogram_buckets(3) + .metrics_poll_count_histogram_resolution(Duration::from_millis(50)) + .build() + .unwrap(), + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .enable_metrics_poll_count_histogram() + .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear) + .metrics_poll_count_histogram_buckets(3) + .metrics_poll_count_histogram_resolution(Duration::from_millis(50)) + .build() + .unwrap(), + ]; + + for rt in rts { + let metrics = rt.metrics(); + rt.block_on(async { + for _ in 0..N { + tokio::spawn(async {}).await.unwrap(); + } + }); + drop(rt); + + let num_workers = metrics.num_workers(); + let num_buckets = metrics.poll_count_histogram_num_buckets(); + + assert!(metrics.poll_count_histogram_enabled()); + assert_eq!(num_buckets, 3); + + let n = (0..num_workers) + .flat_map(|i| (0..num_buckets).map(move |j| (i, j))) + .map(|(worker, bucket)| metrics.poll_count_histogram_bucket_count(worker, bucket)) + .sum(); + assert_eq!(N, n); + } +} + +#[test] +fn worker_poll_count_histogram_range() { + let max = Duration::from_nanos(u64::MAX); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .enable_metrics_poll_count_histogram() + .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear) + .metrics_poll_count_histogram_buckets(3) + .metrics_poll_count_histogram_resolution(us(50)) + .build() + .unwrap(); + let metrics = rt.metrics(); + + assert_eq!(metrics.poll_count_histogram_bucket_range(0), us(0)..us(50)); + assert_eq!( + metrics.poll_count_histogram_bucket_range(1), + us(50)..us(100) + ); + assert_eq!(metrics.poll_count_histogram_bucket_range(2), us(100)..max); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .enable_metrics_poll_count_histogram() + .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Log) + .metrics_poll_count_histogram_buckets(3) + .metrics_poll_count_histogram_resolution(us(50)) + .build() + .unwrap(); + let metrics = rt.metrics(); + + let a = Duration::from_nanos(50000_u64.next_power_of_two()); + let b = a * 2; + + assert_eq!(metrics.poll_count_histogram_bucket_range(0), us(0)..a); + assert_eq!(metrics.poll_count_histogram_bucket_range(1), a..b); + assert_eq!(metrics.poll_count_histogram_bucket_range(2), b..max); +} + +#[test] +fn worker_poll_count_histogram_disabled_without_explicit_enable() { + let rts = [ + tokio::runtime::Builder::new_current_thread() + .enable_all() + .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear) + .metrics_poll_count_histogram_buckets(3) + .metrics_poll_count_histogram_resolution(Duration::from_millis(50)) + .build() + .unwrap(), + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .metrics_poll_count_histogram_scale(tokio::runtime::HistogramScale::Linear) + .metrics_poll_count_histogram_buckets(3) + .metrics_poll_count_histogram_resolution(Duration::from_millis(50)) + .build() + .unwrap(), + ]; + + for rt in rts { + let metrics = rt.metrics(); + assert!(!metrics.poll_count_histogram_enabled()); + } } #[test] @@ -165,7 +376,7 @@ fn worker_total_busy_duration() { let zero = Duration::from_millis(0); - let rt = basic(); + let rt = current_thread(); let metrics = rt.metrics(); rt.block_on(async { @@ -204,7 +415,7 @@ fn worker_total_busy_duration() { #[test] fn worker_local_schedule_count() { - let rt = basic(); + let rt = current_thread(); let metrics = rt.metrics(); rt.block_on(async { tokio::spawn(async {}).await.unwrap(); @@ -280,7 +491,7 @@ fn worker_overflow_count() { fn injection_queue_depth() { use std::thread; - let rt = basic(); + let rt = current_thread(); let handle = rt.handle().clone(); let metrics = rt.metrics(); @@ -299,10 +510,20 @@ fn injection_queue_depth() { // First we need to block the runtime workers let (tx1, rx1) = std::sync::mpsc::channel(); let (tx2, rx2) = std::sync::mpsc::channel(); + let (tx3, rx3) = std::sync::mpsc::channel(); + let rx3 = Arc::new(Mutex::new(rx3)); rt.spawn(async move { rx1.recv().unwrap() }); rt.spawn(async move { rx2.recv().unwrap() }); + // Spawn some more to make sure there are items + for _ in 0..10 { + let rx = rx3.clone(); + rt.spawn(async move { + rx.lock().unwrap().recv().unwrap(); + }); + } + thread::spawn(move || { handle.spawn(async {}); }) @@ -311,7 +532,11 @@ fn injection_queue_depth() { let n = metrics.injection_queue_depth(); assert!(1 <= n, "{}", n); - assert!(3 >= n, "{}", n); + assert!(15 >= n, "{}", n); + + for _ in 0..10 { + tx3.send(()).unwrap(); + } tx1.send(()).unwrap(); tx2.send(()).unwrap(); @@ -321,7 +546,7 @@ fn injection_queue_depth() { fn worker_local_queue_depth() { const N: usize = 100; - let rt = basic(); + let rt = current_thread(); let metrics = rt.metrics(); rt.block_on(async { for _ in 0..N { @@ -369,7 +594,111 @@ fn worker_local_queue_depth() { }); } -fn basic() -> Runtime { +#[test] +fn budget_exhaustion_yield() { + let rt = current_thread(); + let metrics = rt.metrics(); + + assert_eq!(0, metrics.budget_forced_yield_count()); + + let mut did_yield = false; + + // block on a task which consumes budget until it yields + rt.block_on(poll_fn(|cx| loop { + if did_yield { + return Poll::Ready(()); + } + + let fut = consume_budget(); + tokio::pin!(fut); + + if fut.poll(cx).is_pending() { + did_yield = true; + return Poll::Pending; + } + })); + + assert_eq!(1, rt.metrics().budget_forced_yield_count()); +} + +#[test] +fn budget_exhaustion_yield_with_joins() { + let rt = current_thread(); + let metrics = rt.metrics(); + + assert_eq!(0, metrics.budget_forced_yield_count()); + + let mut did_yield_1 = false; + let mut did_yield_2 = false; + + // block on a task which consumes budget until it yields + rt.block_on(async { + tokio::join!( + poll_fn(|cx| loop { + if did_yield_1 { + return Poll::Ready(()); + } + + let fut = consume_budget(); + tokio::pin!(fut); + + if fut.poll(cx).is_pending() { + did_yield_1 = true; + return Poll::Pending; + } + }), + poll_fn(|cx| loop { + if did_yield_2 { + return Poll::Ready(()); + } + + let fut = consume_budget(); + tokio::pin!(fut); + + if fut.poll(cx).is_pending() { + did_yield_2 = true; + return Poll::Pending; + } + }) + ) + }); + + assert_eq!(1, rt.metrics().budget_forced_yield_count()); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn io_driver_fd_count() { + let rt = current_thread(); + let metrics = rt.metrics(); + + assert_eq!(metrics.io_driver_fd_registered_count(), 0); + + let stream = tokio::net::TcpStream::connect("google.com:80"); + let stream = rt.block_on(async move { stream.await.unwrap() }); + + assert_eq!(metrics.io_driver_fd_registered_count(), 1); + assert_eq!(metrics.io_driver_fd_deregistered_count(), 0); + + drop(stream); + + assert_eq!(metrics.io_driver_fd_deregistered_count(), 1); + assert_eq!(metrics.io_driver_fd_registered_count(), 1); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn io_driver_ready_count() { + let rt = current_thread(); + let metrics = rt.metrics(); + + let stream = tokio::net::TcpStream::connect("google.com:80"); + let _stream = rt.block_on(async move { stream.await.unwrap() }); + + assert_eq!(metrics.io_driver_ready_count(), 1); +} + +fn current_thread() -> Runtime { tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -383,3 +712,7 @@ fn threaded() -> Runtime { .build() .unwrap() } + +fn us(n: u64) -> Duration { + Duration::from_micros(n) +} diff --git a/third_party/rust/tokio/tests/rt_panic.rs b/third_party/rust/tokio/tests/rt_panic.rs new file mode 100644 index 000000000000..f9a684fdadae --- /dev/null +++ b/third_party/rust/tokio/tests/rt_panic.rs @@ -0,0 +1,77 @@ +#![warn(rust_2018_idioms)] +#![cfg(feature = "full")] +#![cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery + +use futures::future; +use std::error::Error; +use tokio::runtime::{Builder, Handle, Runtime}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn current_handle_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _ = Handle::current(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn into_panic_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(move || { + let rt = current_thread(); + rt.block_on(async { + let handle = tokio::spawn(future::pending::<()>()); + + handle.abort(); + + let err = handle.await.unwrap_err(); + assert!(!&err.is_panic()); + + let _ = err.into_panic(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn builder_worker_threads_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _ = Builder::new_multi_thread().worker_threads(0).build(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn builder_max_blocking_threads_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _ = Builder::new_multi_thread().max_blocking_threads(0).build(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +fn current_thread() -> Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} diff --git a/third_party/rust/tokio/tests/rt_threaded.rs b/third_party/rust/tokio/tests/rt_threaded.rs index b2f84fd33ff9..69b186947bd8 100644 --- a/third_party/rust/tokio/tests/rt_threaded.rs +++ b/third_party/rust/tokio/tests/rt_threaded.rs @@ -1,9 +1,9 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use tokio::runtime::{self, Runtime}; +use tokio::runtime; use tokio::sync::oneshot; use tokio_test::{assert_err, assert_ok}; @@ -30,7 +30,8 @@ fn single_thread() { let _ = runtime::Builder::new_multi_thread() .enable_all() .worker_threads(1) - .build(); + .build() + .unwrap(); } #[test] @@ -160,6 +161,32 @@ fn many_multishot_futures() { } } +#[test] +fn lifo_slot_budget() { + async fn my_fn() { + spawn_another(); + } + + fn spawn_another() { + tokio::spawn(my_fn()); + } + + let rt = runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(1) + .build() + .unwrap(); + + let (send, recv) = oneshot::channel(); + + rt.spawn(async move { + tokio::spawn(my_fn()); + let _ = send.send(()); + }); + + let _ = rt.block_on(recv); +} + #[test] fn spawn_shutdown() { let rt = rt(); @@ -415,6 +442,32 @@ fn coop_and_block_in_place() { }); } +#[test] +fn yield_after_block_in_place() { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + + rt.block_on(async { + tokio::spawn(async move { + // Block in place then enter a new runtime + tokio::task::block_in_place(|| { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(async {}); + }); + + // Yield, then complete + tokio::task::yield_now().await; + }) + .await + .unwrap() + }); +} + // Testing this does not panic #[test] fn max_blocking_threads() { @@ -480,9 +533,7 @@ fn wake_during_shutdown() { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { let me = Pin::into_inner(self); let mut lock = me.shared.lock().unwrap(); - println!("poll {}", me.put_waker); if me.put_waker { - println!("putting"); lock.waker = Some(cx.waker().clone()); } Poll::Pending @@ -491,13 +542,11 @@ fn wake_during_shutdown() { impl Drop for MyFuture { fn drop(&mut self) { - println!("drop {} start", self.put_waker); let mut lock = self.shared.lock().unwrap(); if !self.put_waker { lock.waker.take().unwrap().wake(); } drop(lock); - println!("drop {} stop", self.put_waker); } } @@ -539,6 +588,178 @@ async fn test_block_in_place4() { tokio::task::block_in_place(|| {}); } -fn rt() -> Runtime { - Runtime::new().unwrap() +// Repro for tokio-rs/tokio#5239 +#[test] +fn test_nested_block_in_place_with_block_on_between() { + let rt = runtime::Builder::new_multi_thread() + .worker_threads(1) + // Needs to be more than 0 + .max_blocking_threads(1) + .build() + .unwrap(); + + // Triggered by a race condition, so run a few times to make sure it is OK. + for _ in 0..100 { + let h = rt.handle().clone(); + + rt.block_on(async move { + tokio::spawn(async move { + tokio::task::block_in_place(|| { + h.block_on(async { + tokio::task::block_in_place(|| {}); + }); + }) + }) + .await + .unwrap() + }); + } +} + +// Testing the tuning logic is tricky as it is inherently timing based, and more +// of a heuristic than an exact behavior. This test checks that the interval +// changes over time based on load factors. There are no assertions, completion +// is sufficient. If there is a regression, this test will hang. In theory, we +// could add limits, but that would be likely to fail on CI. +#[test] +#[cfg(not(tokio_no_tuning_tests))] +fn test_tuning() { + use std::sync::atomic::AtomicBool; + use std::time::Duration; + + let rt = runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + + fn iter(flag: Arc, counter: Arc, stall: bool) { + if flag.load(Relaxed) { + if stall { + std::thread::sleep(Duration::from_micros(5)); + } + + counter.fetch_add(1, Relaxed); + tokio::spawn(async move { iter(flag, counter, stall) }); + } + } + + let flag = Arc::new(AtomicBool::new(true)); + let counter = Arc::new(AtomicUsize::new(61)); + let interval = Arc::new(AtomicUsize::new(61)); + + { + let flag = flag.clone(); + let counter = counter.clone(); + rt.spawn(async move { iter(flag, counter, true) }); + } + + // Now, hammer the injection queue until the interval drops. + let mut n = 0; + loop { + let curr = interval.load(Relaxed); + + if curr <= 8 { + n += 1; + } else { + n = 0; + } + + // Make sure we get a few good rounds. Jitter in the tuning could result + // in one "good" value without being representative of reaching a good + // state. + if n == 3 { + break; + } + + if Arc::strong_count(&interval) < 5_000 { + let counter = counter.clone(); + let interval = interval.clone(); + + rt.spawn(async move { + let prev = counter.swap(0, Relaxed); + interval.store(prev, Relaxed); + }); + + std::thread::yield_now(); + } + } + + flag.store(false, Relaxed); + + let w = Arc::downgrade(&interval); + drop(interval); + + while w.strong_count() > 0 { + std::thread::sleep(Duration::from_micros(500)); + } + + // Now, run it again with a faster task + let flag = Arc::new(AtomicBool::new(true)); + // Set it high, we know it shouldn't ever really be this high + let counter = Arc::new(AtomicUsize::new(10_000)); + let interval = Arc::new(AtomicUsize::new(10_000)); + + { + let flag = flag.clone(); + let counter = counter.clone(); + rt.spawn(async move { iter(flag, counter, false) }); + } + + // Now, hammer the injection queue until the interval reaches the expected range. + let mut n = 0; + loop { + let curr = interval.load(Relaxed); + + if curr <= 1_000 && curr > 32 { + n += 1; + } else { + n = 0; + } + + if n == 3 { + break; + } + + if Arc::strong_count(&interval) <= 5_000 { + let counter = counter.clone(); + let interval = interval.clone(); + + rt.spawn(async move { + let prev = counter.swap(0, Relaxed); + interval.store(prev, Relaxed); + }); + } + + std::thread::yield_now(); + } + + flag.store(false, Relaxed); +} + +fn rt() -> runtime::Runtime { + runtime::Runtime::new().unwrap() +} + +#[cfg(tokio_unstable)] +mod unstable { + use super::*; + + #[test] + fn test_disable_lifo_slot() { + let rt = runtime::Builder::new_multi_thread() + .disable_lifo_slot() + .worker_threads(2) + .build() + .unwrap(); + + rt.block_on(async { + tokio::spawn(async { + // Spawn another task and block the thread until completion. If the LIFO slot + // is used then the test doesn't complete. + futures::executor::block_on(tokio::spawn(async {})).unwrap(); + }) + .await + .unwrap(); + }) + } } diff --git a/third_party/rust/tokio/tests/rt_time_start_paused.rs b/third_party/rust/tokio/tests/rt_time_start_paused.rs new file mode 100644 index 000000000000..283f4748a8a8 --- /dev/null +++ b/third_party/rust/tokio/tests/rt_time_start_paused.rs @@ -0,0 +1,14 @@ +#![cfg(all(feature = "full"))] + +use tokio::time::{Duration, Instant}; + +#[tokio::test(start_paused = true)] +async fn test_start_paused() { + let now = Instant::now(); + + // Pause a few times w/ std sleep and ensure `now` stays the same + for _ in 0..5 { + std::thread::sleep(Duration::from_millis(1)); + assert_eq!(now, Instant::now()); + } +} diff --git a/third_party/rust/tokio/tests/signal_no_rt.rs b/third_party/rust/tokio/tests/signal_no_rt.rs index b0f32b2d10f5..ffcc6651c4d3 100644 --- a/third_party/rust/tokio/tests/signal_no_rt.rs +++ b/third_party/rust/tokio/tests/signal_no_rt.rs @@ -4,6 +4,7 @@ use tokio::signal::unix::{signal, SignalKind}; +#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")] #[test] #[should_panic] fn no_runtime_panics_creating_signals() { diff --git a/third_party/rust/tokio/tests/signal_panic.rs b/third_party/rust/tokio/tests/signal_panic.rs new file mode 100644 index 000000000000..ce1ec3e4a73d --- /dev/null +++ b/third_party/rust/tokio/tests/signal_panic.rs @@ -0,0 +1,29 @@ +#![warn(rust_2018_idioms)] +#![cfg(feature = "full")] +#![cfg(unix)] + +use std::error::Error; +use tokio::runtime::Builder; +use tokio::signal::unix::{signal, SignalKind}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn signal_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = Builder::new_current_thread().build().unwrap(); + + rt.block_on(async { + let kind = SignalKind::from_raw(-1); + let _ = signal(kind); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/third_party/rust/tokio/tests/support/leaked_buffers.rs b/third_party/rust/tokio/tests/support/leaked_buffers.rs new file mode 100644 index 000000000000..a6079fb707a7 --- /dev/null +++ b/third_party/rust/tokio/tests/support/leaked_buffers.rs @@ -0,0 +1,26 @@ +/// Can create buffers of arbitrary lifetime. +/// Frees created buffers when dropped. +/// +/// This struct is of course unsafe and the fact that +/// it must outlive the created slices has to be ensured by +/// the programmer. +/// +/// Used at certain test scenarios as a safer version of +/// Vec::leak, to satisfy the address sanitizer. +pub struct LeakedBuffers { + leaked_vecs: Vec>, +} + +impl LeakedBuffers { + pub fn new() -> Self { + Self { + leaked_vecs: vec![], + } + } + pub unsafe fn create<'a>(&mut self, size: usize) -> &'a mut [u8] { + let new_mem = vec![0u8; size].into_boxed_slice(); + self.leaked_vecs.push(new_mem); + let new_mem = self.leaked_vecs.last_mut().unwrap(); + std::slice::from_raw_parts_mut(new_mem.as_mut_ptr(), new_mem.len()) + } +} diff --git a/third_party/rust/tokio/tests/support/panic.rs b/third_party/rust/tokio/tests/support/panic.rs new file mode 100644 index 000000000000..df2f59d30ffb --- /dev/null +++ b/third_party/rust/tokio/tests/support/panic.rs @@ -0,0 +1,34 @@ +use std::panic; +use std::sync::{Arc, Mutex}; + +pub fn test_panic(func: Func) -> Option { + static PANIC_MUTEX: Mutex<()> = Mutex::new(()); + + { + let _guard = PANIC_MUTEX.lock(); + let panic_file: Arc>> = Arc::new(Mutex::new(None)); + + let prev_hook = panic::take_hook(); + { + let panic_file = panic_file.clone(); + panic::set_hook(Box::new(move |panic_info| { + let panic_location = panic_info.location().unwrap(); + panic_file + .lock() + .unwrap() + .clone_from(&Some(panic_location.file().to_string())); + })); + } + + let result = panic::catch_unwind(func); + // Return to the previously set panic hook (maybe default) so that we get nice error + // messages in the tests. + panic::set_hook(prev_hook); + + if result.is_err() { + panic_file.lock().unwrap().clone() + } else { + None + } + } +} diff --git a/third_party/rust/tokio/tests/sync_barrier.rs b/third_party/rust/tokio/tests/sync_barrier.rs index 5fe7ba98c8be..2001c3598cbb 100644 --- a/third_party/rust/tokio/tests/sync_barrier.rs +++ b/third_party/rust/tokio/tests/sync_barrier.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::Barrier; diff --git a/third_party/rust/tokio/tests/sync_broadcast.rs b/third_party/rust/tokio/tests/sync_broadcast.rs index 1b68eb7edbdc..feed03148aff 100644 --- a/third_party/rust/tokio/tests/sync_broadcast.rs +++ b/third_party/rust/tokio/tests/sync_broadcast.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::broadcast; @@ -47,7 +47,7 @@ macro_rules! assert_closed { ($e:expr) => { match assert_err!($e) { broadcast::error::TryRecvError::Closed => {} - _ => panic!("did not lag"), + _ => panic!("is not closed"), } }; } @@ -276,14 +276,14 @@ fn send_no_rx() { #[test] #[should_panic] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn zero_capacity() { broadcast::channel::<()>(0); } #[test] #[should_panic] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn capacity_too_big() { use std::usize; @@ -291,7 +291,8 @@ fn capacity_too_big() { } #[test] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(panic = "unwind")] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn panic_in_clone() { use std::panic::{self, AssertUnwindSafe}; @@ -457,6 +458,186 @@ fn lagging_receiver_recovers_after_wrap_open() { assert_empty!(rx); } +#[test] +fn receiver_len_with_lagged() { + let (tx, mut rx) = broadcast::channel(3); + + tx.send(10).unwrap(); + tx.send(20).unwrap(); + tx.send(30).unwrap(); + tx.send(40).unwrap(); + + assert_eq!(rx.len(), 4); + assert_eq!(assert_recv!(rx), 10); + + tx.send(50).unwrap(); + tx.send(60).unwrap(); + + assert_eq!(rx.len(), 5); + assert_lagged!(rx.try_recv(), 1); +} + fn is_closed(err: broadcast::error::RecvError) -> bool { matches!(err, broadcast::error::RecvError::Closed) } + +#[test] +fn resubscribe_points_to_tail() { + let (tx, mut rx) = broadcast::channel(3); + tx.send(1).unwrap(); + + let mut rx_resub = rx.resubscribe(); + + // verify we're one behind at the start + assert_empty!(rx_resub); + assert_eq!(assert_recv!(rx), 1); + + // verify we do not affect rx + tx.send(2).unwrap(); + assert_eq!(assert_recv!(rx_resub), 2); + tx.send(3).unwrap(); + assert_eq!(assert_recv!(rx), 2); + assert_eq!(assert_recv!(rx), 3); + assert_empty!(rx); + + assert_eq!(assert_recv!(rx_resub), 3); + assert_empty!(rx_resub); +} + +#[test] +fn resubscribe_lagged() { + let (tx, mut rx) = broadcast::channel(1); + tx.send(1).unwrap(); + tx.send(2).unwrap(); + + let mut rx_resub = rx.resubscribe(); + assert_lagged!(rx.try_recv(), 1); + assert_empty!(rx_resub); + + assert_eq!(assert_recv!(rx), 2); + assert_empty!(rx); + assert_empty!(rx_resub); +} + +#[test] +fn resubscribe_to_closed_channel() { + let (tx, rx) = tokio::sync::broadcast::channel::(2); + drop(tx); + + let mut rx_resub = rx.resubscribe(); + assert_closed!(rx_resub.try_recv()); +} + +#[test] +fn sender_len() { + let (tx, mut rx1) = broadcast::channel(4); + let mut rx2 = tx.subscribe(); + + assert_eq!(tx.len(), 0); + assert!(tx.is_empty()); + + tx.send(1).unwrap(); + tx.send(2).unwrap(); + tx.send(3).unwrap(); + + assert_eq!(tx.len(), 3); + assert!(!tx.is_empty()); + + assert_recv!(rx1); + assert_recv!(rx1); + + assert_eq!(tx.len(), 3); + assert!(!tx.is_empty()); + + assert_recv!(rx2); + + assert_eq!(tx.len(), 2); + assert!(!tx.is_empty()); + + tx.send(4).unwrap(); + tx.send(5).unwrap(); + tx.send(6).unwrap(); + + assert_eq!(tx.len(), 4); + assert!(!tx.is_empty()); +} + +#[test] +#[cfg(not(tokio_wasm_not_wasi))] +fn sender_len_random() { + use rand::Rng; + + let (tx, mut rx1) = broadcast::channel(16); + let mut rx2 = tx.subscribe(); + + for _ in 0..1000 { + match rand::thread_rng().gen_range(0..4) { + 0 => { + let _ = rx1.try_recv(); + } + 1 => { + let _ = rx2.try_recv(); + } + _ => { + tx.send(0).unwrap(); + } + } + + let expected_len = usize::min(usize::max(rx1.len(), rx2.len()), 16); + assert_eq!(tx.len(), expected_len); + } +} + +#[test] +fn send_in_waker_drop() { + use futures::task::ArcWake; + use std::future::Future; + use std::task::Context; + + struct SendOnDrop(broadcast::Sender<()>); + + impl Drop for SendOnDrop { + fn drop(&mut self) { + let _ = self.0.send(()); + } + } + + impl ArcWake for SendOnDrop { + fn wake_by_ref(_arc_self: &Arc) {} + } + + // Test if there is no deadlock when replacing the old waker. + + let (tx, mut rx) = broadcast::channel(16); + + let mut fut = Box::pin(async { + let _ = rx.recv().await; + }); + + // Store our special waker in the receiving future. + let waker = futures::task::waker(Arc::new(SendOnDrop(tx))); + let mut cx = Context::from_waker(&waker); + assert!(fut.as_mut().poll(&mut cx).is_pending()); + drop(waker); + + // Second poll shouldn't deadlock. + let mut cx = Context::from_waker(futures::task::noop_waker_ref()); + let _ = fut.as_mut().poll(&mut cx); + + // Test if there is no deadlock when calling waker.wake(). + + let (tx, mut rx) = broadcast::channel(16); + + let mut fut = Box::pin(async { + let _ = rx.recv().await; + }); + + // Store our special waker in the receiving future. + let waker = futures::task::waker(Arc::new(SendOnDrop(tx.clone()))); + let mut cx = Context::from_waker(&waker); + assert!(fut.as_mut().poll(&mut cx).is_pending()); + drop(waker); + + // Shouldn't deadlock. + let _ = tx.send(()); +} diff --git a/third_party/rust/tokio/tests/sync_errors.rs b/third_party/rust/tokio/tests/sync_errors.rs index 2eac585d40ce..bf0a6cf7374b 100644 --- a/third_party/rust/tokio/tests/sync_errors.rs +++ b/third_party/rust/tokio/tests/sync_errors.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; fn is_error() {} diff --git a/third_party/rust/tokio/tests/sync_mpsc.rs b/third_party/rust/tokio/tests/sync_mpsc.rs index abbfa9d7f46a..6e870964113f 100644 --- a/third_party/rust/tokio/tests/sync_mpsc.rs +++ b/third_party/rust/tokio/tests/sync_mpsc.rs @@ -2,21 +2,20 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] -use tokio::test as maybe_tokio_test; - +use std::fmt; +use std::sync::Arc; use tokio::sync::mpsc; use tokio::sync::mpsc::error::{TryRecvError, TrySendError}; +#[cfg(not(tokio_wasm_not_wasi))] +use tokio::test as maybe_tokio_test; use tokio_test::*; -use std::sync::Arc; - -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm))] mod support { pub(crate) mod mpsc_stream; } @@ -88,7 +87,7 @@ async fn reserve_disarm() { } #[tokio::test] -#[cfg(feature = "full")] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads async fn send_recv_stream_with_buffer() { use tokio_stream::StreamExt; @@ -155,7 +154,7 @@ async fn start_send_past_cap() { #[test] #[should_panic] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn buffer_gteq_one() { mpsc::channel::(0); } @@ -192,7 +191,7 @@ async fn async_send_recv_unbounded() { } #[tokio::test] -#[cfg(feature = "full")] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads async fn send_recv_stream_unbounded() { use tokio_stream::StreamExt; @@ -217,9 +216,9 @@ async fn no_t_bounds_buffer() { let (tx, mut rx) = mpsc::channel(100); // sender should be Debug even though T isn't Debug - println!("{:?}", tx); + is_debug(&tx); // same with Receiver - println!("{:?}", rx); + is_debug(&rx); // and sender should be Clone even though T isn't Clone assert!(tx.clone().try_send(NoImpls).is_ok()); @@ -233,9 +232,9 @@ async fn no_t_bounds_unbounded() { let (tx, mut rx) = mpsc::unbounded_channel(); // sender should be Debug even though T isn't Debug - println!("{:?}", tx); + is_debug(&tx); // same with Receiver - println!("{:?}", rx); + is_debug(&rx); // and sender should be Clone even though T isn't Clone assert!(tx.clone().send(NoImpls).is_ok()); @@ -453,7 +452,7 @@ fn unconsumed_messages_are_dropped() { } #[test] -#[cfg(feature = "full")] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads fn blocking_recv() { let (tx, mut rx) = mpsc::channel::(1); @@ -471,14 +470,14 @@ fn blocking_recv() { #[tokio::test] #[should_panic] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding async fn blocking_recv_async() { let (_tx, mut rx) = mpsc::channel::<()>(1); let _ = rx.blocking_recv(); } #[test] -#[cfg(feature = "full")] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads fn blocking_send() { let (tx, mut rx) = mpsc::channel::(1); @@ -496,7 +495,7 @@ fn blocking_send() { #[tokio::test] #[should_panic] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding async fn blocking_send_async() { let (tx, _rx) = mpsc::channel::<()>(1); let _ = tx.blocking_send(()); @@ -649,7 +648,7 @@ async fn recv_timeout() { #[test] #[should_panic = "there is no reactor running, must be called from the context of a Tokio 1.x runtime"] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn recv_timeout_panic() { use futures::future::FutureExt; use tokio::time::Duration; @@ -657,3 +656,24 @@ fn recv_timeout_panic() { let (tx, _rx) = mpsc::channel(5); tx.send_timeout(10, Duration::from_secs(1)).now_or_never(); } + +// Tests that channel `capacity` changes and `max_capacity` stays the same +#[tokio::test] +async fn test_tx_capacity() { + let (tx, _rx) = mpsc::channel::<()>(10); + // both capacities are same before + assert_eq!(tx.capacity(), 10); + assert_eq!(tx.max_capacity(), 10); + + let _permit = tx.reserve().await.unwrap(); + // after reserve, only capacity should drop by one + assert_eq!(tx.capacity(), 9); + assert_eq!(tx.max_capacity(), 10); + + tx.send(()).await.unwrap(); + // after send, capacity should drop by one again + assert_eq!(tx.capacity(), 8); + assert_eq!(tx.max_capacity(), 10); +} + +fn is_debug(_: &T) {} diff --git a/third_party/rust/tokio/tests/sync_mpsc_weak.rs b/third_party/rust/tokio/tests/sync_mpsc_weak.rs new file mode 100644 index 000000000000..0fdfc00708bb --- /dev/null +++ b/third_party/rust/tokio/tests/sync_mpsc_weak.rs @@ -0,0 +1,513 @@ +#![allow(clippy::redundant_clone)] +#![warn(rust_2018_idioms)] +#![cfg(feature = "sync")] + +#[cfg(tokio_wasm_not_wasi)] +use wasm_bindgen_test::wasm_bindgen_test as test; + +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::{Acquire, Release}; +use tokio::sync::mpsc::{self, channel, unbounded_channel}; +use tokio::sync::oneshot; + +#[tokio::test] +async fn weak_sender() { + let (tx, mut rx) = channel(11); + + let tx_weak = tokio::spawn(async move { + let tx_weak = tx.clone().downgrade(); + + for i in 0..10 { + if tx.send(i).await.is_err() { + return None; + } + } + + let tx2 = tx_weak + .upgrade() + .expect("expected to be able to upgrade tx_weak"); + let _ = tx2.send(20).await; + let tx_weak = tx2.downgrade(); + + Some(tx_weak) + }) + .await + .unwrap(); + + for i in 0..12 { + let recvd = rx.recv().await; + + match recvd { + Some(msg) => { + if i == 10 { + assert_eq!(msg, 20); + } + } + None => { + assert_eq!(i, 11); + break; + } + } + } + + let tx_weak = tx_weak.unwrap(); + let upgraded = tx_weak.upgrade(); + assert!(upgraded.is_none()); +} + +#[tokio::test] +async fn actor_weak_sender() { + pub struct MyActor { + receiver: mpsc::Receiver, + sender: mpsc::WeakSender, + next_id: u32, + pub received_self_msg: bool, + } + + enum ActorMessage { + GetUniqueId { respond_to: oneshot::Sender }, + SelfMessage {}, + } + + impl MyActor { + fn new( + receiver: mpsc::Receiver, + sender: mpsc::WeakSender, + ) -> Self { + MyActor { + receiver, + sender, + next_id: 0, + received_self_msg: false, + } + } + + fn handle_message(&mut self, msg: ActorMessage) { + match msg { + ActorMessage::GetUniqueId { respond_to } => { + self.next_id += 1; + + // The `let _ =` ignores any errors when sending. + // + // This can happen if the `select!` macro is used + // to cancel waiting for the response. + let _ = respond_to.send(self.next_id); + } + ActorMessage::SelfMessage { .. } => { + self.received_self_msg = true; + } + } + } + + async fn send_message_to_self(&mut self) { + let msg = ActorMessage::SelfMessage {}; + + let sender = self.sender.clone(); + + // cannot move self.sender here + if let Some(sender) = sender.upgrade() { + let _ = sender.send(msg).await; + self.sender = sender.downgrade(); + } + } + + async fn run(&mut self) { + let mut i = 0; + while let Some(msg) = self.receiver.recv().await { + self.handle_message(msg); + + if i == 0 { + self.send_message_to_self().await; + } + + i += 1 + } + + assert!(self.received_self_msg); + } + } + + #[derive(Clone)] + pub struct MyActorHandle { + sender: mpsc::Sender, + } + + impl MyActorHandle { + pub fn new() -> (Self, MyActor) { + let (sender, receiver) = mpsc::channel(8); + let actor = MyActor::new(receiver, sender.clone().downgrade()); + + (Self { sender }, actor) + } + + pub async fn get_unique_id(&self) -> u32 { + let (send, recv) = oneshot::channel(); + let msg = ActorMessage::GetUniqueId { respond_to: send }; + + // Ignore send errors. If this send fails, so does the + // recv.await below. There's no reason to check the + // failure twice. + let _ = self.sender.send(msg).await; + recv.await.expect("Actor task has been killed") + } + } + + let (handle, mut actor) = MyActorHandle::new(); + + let actor_handle = tokio::spawn(async move { actor.run().await }); + + let _ = tokio::spawn(async move { + let _ = handle.get_unique_id().await; + drop(handle); + }) + .await; + + let _ = actor_handle.await; +} + +static NUM_DROPPED: AtomicUsize = AtomicUsize::new(0); + +#[derive(Debug)] +struct Msg; + +impl Drop for Msg { + fn drop(&mut self) { + NUM_DROPPED.fetch_add(1, Release); + } +} + +// Tests that no pending messages are put onto the channel after `Rx` was +// dropped. +// +// Note: After the introduction of `WeakSender`, which internally +// used `Arc` and doesn't call a drop of the channel after the last strong +// `Sender` was dropped while more than one `WeakSender` remains, we want to +// ensure that no messages are kept in the channel, which were sent after +// the receiver was dropped. +#[tokio::test] +async fn test_msgs_dropped_on_rx_drop() { + let (tx, mut rx) = mpsc::channel(3); + + tx.send(Msg {}).await.unwrap(); + tx.send(Msg {}).await.unwrap(); + + // This msg will be pending and should be dropped when `rx` is dropped + let sent_fut = tx.send(Msg {}); + + let _ = rx.recv().await.unwrap(); + let _ = rx.recv().await.unwrap(); + + sent_fut.await.unwrap(); + + drop(rx); + + assert_eq!(NUM_DROPPED.load(Acquire), 3); + + // This msg will not be put onto `Tx` list anymore, since `Rx` is closed. + assert!(tx.send(Msg {}).await.is_err()); + + assert_eq!(NUM_DROPPED.load(Acquire), 4); +} + +// Tests that a `WeakSender` is upgradeable when other `Sender`s exist. +#[test] +fn downgrade_upgrade_sender_success() { + let (tx, _rx) = mpsc::channel::(1); + let weak_tx = tx.downgrade(); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that a `WeakSender` fails to upgrade when no other `Sender` exists. +#[test] +fn downgrade_upgrade_sender_failure() { + let (tx, _rx) = mpsc::channel::(1); + let weak_tx = tx.downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that a `WeakSender` cannot be upgraded after a `Sender` was dropped, +// which existed at the time of the `downgrade` call. +#[test] +fn downgrade_drop_upgrade() { + let (tx, _rx) = mpsc::channel::(1); + + // the cloned `Tx` is dropped right away + let weak_tx = tx.clone().downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that we can upgrade a weak sender with an outstanding permit +// but no other strong senders. +#[tokio::test] +async fn downgrade_get_permit_upgrade_no_senders() { + let (tx, _rx) = mpsc::channel::(1); + let weak_tx = tx.downgrade(); + let _permit = tx.reserve_owned().await.unwrap(); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that you can downgrade and upgrade a sender with an outstanding permit +// but no other senders left. +#[tokio::test] +async fn downgrade_upgrade_get_permit_no_senders() { + let (tx, _rx) = mpsc::channel::(1); + let tx2 = tx.clone(); + let _permit = tx.reserve_owned().await.unwrap(); + let weak_tx = tx2.downgrade(); + drop(tx2); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that `downgrade` does not change the `tx_count` of the channel. +#[test] +fn test_tx_count_weak_sender() { + let (tx, _rx) = mpsc::channel::(1); + let tx_weak = tx.downgrade(); + let tx_weak2 = tx.downgrade(); + drop(tx); + + assert!(tx_weak.upgrade().is_none() && tx_weak2.upgrade().is_none()); +} + +#[tokio::test] +async fn weak_unbounded_sender() { + let (tx, mut rx) = unbounded_channel(); + + let tx_weak = tokio::spawn(async move { + let tx_weak = tx.clone().downgrade(); + + for i in 0..10 { + if tx.send(i).is_err() { + return None; + } + } + + let tx2 = tx_weak + .upgrade() + .expect("expected to be able to upgrade tx_weak"); + let _ = tx2.send(20); + let tx_weak = tx2.downgrade(); + + Some(tx_weak) + }) + .await + .unwrap(); + + for i in 0..12 { + let recvd = rx.recv().await; + + match recvd { + Some(msg) => { + if i == 10 { + assert_eq!(msg, 20); + } + } + None => { + assert_eq!(i, 11); + break; + } + } + } + + let tx_weak = tx_weak.unwrap(); + let upgraded = tx_weak.upgrade(); + assert!(upgraded.is_none()); +} + +#[tokio::test] +async fn actor_weak_unbounded_sender() { + pub struct MyActor { + receiver: mpsc::UnboundedReceiver, + sender: mpsc::WeakUnboundedSender, + next_id: u32, + pub received_self_msg: bool, + } + + enum ActorMessage { + GetUniqueId { respond_to: oneshot::Sender }, + SelfMessage {}, + } + + impl MyActor { + fn new( + receiver: mpsc::UnboundedReceiver, + sender: mpsc::WeakUnboundedSender, + ) -> Self { + MyActor { + receiver, + sender, + next_id: 0, + received_self_msg: false, + } + } + + fn handle_message(&mut self, msg: ActorMessage) { + match msg { + ActorMessage::GetUniqueId { respond_to } => { + self.next_id += 1; + + // The `let _ =` ignores any errors when sending. + // + // This can happen if the `select!` macro is used + // to cancel waiting for the response. + let _ = respond_to.send(self.next_id); + } + ActorMessage::SelfMessage { .. } => { + self.received_self_msg = true; + } + } + } + + async fn send_message_to_self(&mut self) { + let msg = ActorMessage::SelfMessage {}; + + let sender = self.sender.clone(); + + // cannot move self.sender here + if let Some(sender) = sender.upgrade() { + let _ = sender.send(msg); + self.sender = sender.downgrade(); + } + } + + async fn run(&mut self) { + let mut i = 0; + while let Some(msg) = self.receiver.recv().await { + self.handle_message(msg); + + if i == 0 { + self.send_message_to_self().await; + } + + i += 1 + } + + assert!(self.received_self_msg); + } + } + + #[derive(Clone)] + pub struct MyActorHandle { + sender: mpsc::UnboundedSender, + } + + impl MyActorHandle { + pub fn new() -> (Self, MyActor) { + let (sender, receiver) = mpsc::unbounded_channel(); + let actor = MyActor::new(receiver, sender.clone().downgrade()); + + (Self { sender }, actor) + } + + pub async fn get_unique_id(&self) -> u32 { + let (send, recv) = oneshot::channel(); + let msg = ActorMessage::GetUniqueId { respond_to: send }; + + // Ignore send errors. If this send fails, so does the + // recv.await below. There's no reason to check the + // failure twice. + let _ = self.sender.send(msg); + recv.await.expect("Actor task has been killed") + } + } + + let (handle, mut actor) = MyActorHandle::new(); + + let actor_handle = tokio::spawn(async move { actor.run().await }); + + let _ = tokio::spawn(async move { + let _ = handle.get_unique_id().await; + drop(handle); + }) + .await; + + let _ = actor_handle.await; +} + +static NUM_DROPPED_UNBOUNDED: AtomicUsize = AtomicUsize::new(0); + +#[derive(Debug)] +struct MsgUnbounded; + +impl Drop for MsgUnbounded { + fn drop(&mut self) { + NUM_DROPPED_UNBOUNDED.fetch_add(1, Release); + } +} + +// Tests that no pending messages are put onto the channel after `Rx` was +// dropped. +// +// Note: After the introduction of `UnboundedWeakSender`, which internally +// used `Arc` and doesn't call a drop of the channel after the last strong +// `UnboundedSender` was dropped while more than one `UnboundedWeakSender` +// remains, we want to ensure that no messages are kept in the channel, which +// were sent after the receiver was dropped. +#[tokio::test] +async fn test_msgs_dropped_on_unbounded_rx_drop() { + let (tx, mut rx) = mpsc::unbounded_channel(); + + tx.send(MsgUnbounded {}).unwrap(); + tx.send(MsgUnbounded {}).unwrap(); + + // This msg will be pending and should be dropped when `rx` is dropped + let sent = tx.send(MsgUnbounded {}); + + let _ = rx.recv().await.unwrap(); + let _ = rx.recv().await.unwrap(); + + sent.unwrap(); + + drop(rx); + + assert_eq!(NUM_DROPPED_UNBOUNDED.load(Acquire), 3); + + // This msg will not be put onto `Tx` list anymore, since `Rx` is closed. + assert!(tx.send(MsgUnbounded {}).is_err()); + + assert_eq!(NUM_DROPPED_UNBOUNDED.load(Acquire), 4); +} + +// Tests that an `WeakUnboundedSender` is upgradeable when other +// `UnboundedSender`s exist. +#[test] +fn downgrade_upgrade_unbounded_sender_success() { + let (tx, _rx) = mpsc::unbounded_channel::(); + let weak_tx = tx.downgrade(); + assert!(weak_tx.upgrade().is_some()); +} + +// Tests that a `WeakUnboundedSender` fails to upgrade when no other +// `UnboundedSender` exists. +#[test] +fn downgrade_upgrade_unbounded_sender_failure() { + let (tx, _rx) = mpsc::unbounded_channel::(); + let weak_tx = tx.downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that an `WeakUnboundedSender` cannot be upgraded after an +// `UnboundedSender` was dropped, which existed at the time of the `downgrade` call. +#[test] +fn downgrade_drop_upgrade_unbounded() { + let (tx, _rx) = mpsc::unbounded_channel::(); + + // the cloned `Tx` is dropped right away + let weak_tx = tx.clone().downgrade(); + drop(tx); + assert!(weak_tx.upgrade().is_none()); +} + +// Tests that `downgrade` does not change the `tx_count` of the channel. +#[test] +fn test_tx_count_weak_unbounded_sender() { + let (tx, _rx) = mpsc::unbounded_channel::(); + let tx_weak = tx.downgrade(); + let tx_weak2 = tx.downgrade(); + drop(tx); + + assert!(tx_weak.upgrade().is_none() && tx_weak2.upgrade().is_none()); +} diff --git a/third_party/rust/tokio/tests/sync_mutex.rs b/third_party/rust/tokio/tests/sync_mutex.rs index 51dbe03dc73c..1e35a558c1db 100644 --- a/third_party/rust/tokio/tests/sync_mutex.rs +++ b/third_party/rust/tokio/tests/sync_mutex.rs @@ -1,12 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; use tokio::sync::Mutex; @@ -53,7 +53,7 @@ fn readiness() { // But once g unlocks, we can acquire it drop(g); assert!(t2.is_woken()); - assert_ready!(t2.poll()); + let _t2 = assert_ready!(t2.poll()); } /* @@ -103,7 +103,7 @@ async fn aborted_future_1() { timeout(Duration::from_millis(1u64), async move { let iv = interval(Duration::from_millis(1000)); tokio::pin!(iv); - m2.lock().await; + let _g = m2.lock().await; iv.as_mut().tick().await; iv.as_mut().tick().await; }) @@ -112,7 +112,7 @@ async fn aborted_future_1() { } // This should succeed as there is no lock left for the mutex. timeout(Duration::from_millis(1u64), async move { - m1.lock().await; + let _g = m1.lock().await; }) .await .expect("Mutex is locked"); @@ -134,7 +134,7 @@ async fn aborted_future_2() { let m2 = m1.clone(); // Try to lock mutex in a future that is aborted prematurely timeout(Duration::from_millis(1u64), async move { - m2.lock().await; + let _g = m2.lock().await; }) .await .unwrap_err(); @@ -142,7 +142,7 @@ async fn aborted_future_2() { } // This should succeed as there is no lock left for the mutex. timeout(Duration::from_millis(1u64), async move { - m1.lock().await; + let _g = m1.lock().await; }) .await .expect("Mutex is locked"); @@ -155,7 +155,7 @@ fn try_lock() { let g1 = m.try_lock(); assert!(g1.is_ok()); let g2 = m.try_lock(); - assert!(!g2.is_ok()); + assert!(g2.is_err()); } let g3 = m.try_lock(); assert!(g3.is_ok()); diff --git a/third_party/rust/tokio/tests/sync_mutex_owned.rs b/third_party/rust/tokio/tests/sync_mutex_owned.rs index 2ce15de5b9d9..ba472fee78e3 100644 --- a/third_party/rust/tokio/tests/sync_mutex_owned.rs +++ b/third_party/rust/tokio/tests/sync_mutex_owned.rs @@ -1,12 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; use tokio::sync::Mutex; @@ -122,7 +122,7 @@ fn try_lock_owned() { let g1 = m.clone().try_lock_owned(); assert!(g1.is_ok()); let g2 = m.clone().try_lock_owned(); - assert!(!g2.is_ok()); + assert!(g2.is_err()); } let g3 = m.try_lock_owned(); assert!(g3.is_ok()); diff --git a/third_party/rust/tokio/tests/sync_notify.rs b/third_party/rust/tokio/tests/sync_notify.rs index 5318d131cf80..aa58039fac13 100644 --- a/third_party/rust/tokio/tests/sync_notify.rs +++ b/third_party/rust/tokio/tests/sync_notify.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::Notify; @@ -154,3 +154,74 @@ fn notify_one_after_dropped_all() { assert_ready!(notified2.poll()); } + +#[test] +fn test_notify_one_not_enabled() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + notify.notify_one(); + assert_ready!(future.poll()); +} + +#[test] +fn test_notify_one_after_enable() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + future.enter(|_, fut| assert!(!fut.enable())); + + notify.notify_one(); + assert_ready!(future.poll()); + future.enter(|_, fut| assert!(fut.enable())); +} + +#[test] +fn test_poll_after_enable() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + future.enter(|_, fut| assert!(!fut.enable())); + assert_pending!(future.poll()); +} + +#[test] +fn test_enable_after_poll() { + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + assert_pending!(future.poll()); + future.enter(|_, fut| assert!(!fut.enable())); +} + +#[test] +fn test_enable_consumes_permit() { + let notify = Notify::new(); + + // Add a permit. + notify.notify_one(); + + let mut future1 = spawn(notify.notified()); + future1.enter(|_, fut| assert!(fut.enable())); + + let mut future2 = spawn(notify.notified()); + future2.enter(|_, fut| assert!(!fut.enable())); +} + +#[test] +fn test_waker_update() { + use futures::task::noop_waker; + use std::future::Future; + use std::task::Context; + + let notify = Notify::new(); + let mut future = spawn(notify.notified()); + + let noop = noop_waker(); + future.enter(|_, fut| assert_pending!(fut.poll(&mut Context::from_waker(&noop)))); + + assert_pending!(future.poll()); + notify.notify_one(); + + assert!(future.is_woken()); +} diff --git a/third_party/rust/tokio/tests/sync_once_cell.rs b/third_party/rust/tokio/tests/sync_once_cell.rs index 18eaf9382bd3..38dfa7ca0b66 100644 --- a/third_party/rust/tokio/tests/sync_once_cell.rs +++ b/third_party/rust/tokio/tests/sync_once_cell.rs @@ -4,178 +4,7 @@ use std::mem; use std::ops::Drop; use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::Duration; -use tokio::runtime; -use tokio::sync::{OnceCell, SetError}; -use tokio::time; - -async fn func1() -> u32 { - 5 -} - -async fn func2() -> u32 { - time::sleep(Duration::from_millis(1)).await; - 10 -} - -async fn func_err() -> Result { - Err(()) -} - -async fn func_ok() -> Result { - Ok(10) -} - -async fn func_panic() -> u32 { - time::sleep(Duration::from_millis(1)).await; - panic!(); -} - -async fn sleep_and_set() -> u32 { - // Simulate sleep by pausing time and waiting for another thread to - // resume clock when calling `set`, then finding the cell being initialized - // by this call - time::sleep(Duration::from_millis(2)).await; - 5 -} - -async fn advance_time_and_set(cell: &'static OnceCell, v: u32) -> Result<(), SetError> { - time::advance(Duration::from_millis(1)).await; - cell.set(v) -} - -#[test] -fn get_or_init() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .start_paused(true) - .build() - .unwrap(); - - static ONCE: OnceCell = OnceCell::const_new(); - - rt.block_on(async { - let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); - let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await }); - - time::advance(Duration::from_millis(1)).await; - time::resume(); - - let result1 = handle1.await.unwrap(); - let result2 = handle2.await.unwrap(); - - assert_eq!(*result1, 5); - assert_eq!(*result2, 5); - }); -} - -#[test] -fn get_or_init_panic() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - static ONCE: OnceCell = OnceCell::const_new(); - - rt.block_on(async { - time::pause(); - - let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); - let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await }); - - time::advance(Duration::from_millis(1)).await; - - let result1 = handle1.await.unwrap(); - let result2 = handle2.await.unwrap(); - - assert_eq!(*result1, 5); - assert_eq!(*result2, 5); - }); -} - -#[test] -fn set_and_get() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - static ONCE: OnceCell = OnceCell::const_new(); - - rt.block_on(async { - let _ = rt.spawn(async { ONCE.set(5) }).await; - let value = ONCE.get().unwrap(); - assert_eq!(*value, 5); - }); -} - -#[test] -fn get_uninit() { - static ONCE: OnceCell = OnceCell::const_new(); - let uninit = ONCE.get(); - assert!(uninit.is_none()); -} - -#[test] -fn set_twice() { - static ONCE: OnceCell = OnceCell::const_new(); - - let first = ONCE.set(5); - assert_eq!(first, Ok(())); - let second = ONCE.set(6); - assert!(second.err().unwrap().is_already_init_err()); -} - -#[test] -fn set_while_initializing() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - static ONCE: OnceCell = OnceCell::const_new(); - - rt.block_on(async { - time::pause(); - - let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await }); - let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await }); - - time::advance(Duration::from_millis(2)).await; - - let result1 = handle1.await.unwrap(); - let result2 = handle2.await.unwrap(); - - assert_eq!(*result1, 5); - assert!(result2.err().unwrap().is_initializing_err()); - }); -} - -#[test] -fn get_or_try_init() { - let rt = runtime::Builder::new_current_thread() - .enable_time() - .start_paused(true) - .build() - .unwrap(); - - static ONCE: OnceCell = OnceCell::const_new(); - - rt.block_on(async { - let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await }); - let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await }); - - time::advance(Duration::from_millis(1)).await; - time::resume(); - - let result1 = handle1.await.unwrap(); - assert!(result1.is_err()); - - let result2 = handle2.await.unwrap(); - assert_eq!(*result2.unwrap(), 10); - }); -} +use tokio::sync::OnceCell; #[test] fn drop_cell() { @@ -272,3 +101,185 @@ fn from() { let cell = OnceCell::from(2); assert_eq!(*cell.get().unwrap(), 2); } + +#[cfg(feature = "parking_lot")] +mod parking_lot { + use super::*; + + use tokio::runtime; + use tokio::sync::SetError; + use tokio::time; + + use std::time::Duration; + + async fn func1() -> u32 { + 5 + } + + async fn func2() -> u32 { + time::sleep(Duration::from_millis(1)).await; + 10 + } + + async fn func_err() -> Result { + Err(()) + } + + async fn func_ok() -> Result { + Ok(10) + } + + async fn func_panic() -> u32 { + time::sleep(Duration::from_millis(1)).await; + panic!(); + } + + async fn sleep_and_set() -> u32 { + // Simulate sleep by pausing time and waiting for another thread to + // resume clock when calling `set`, then finding the cell being initialized + // by this call + time::sleep(Duration::from_millis(2)).await; + 5 + } + + async fn advance_time_and_set( + cell: &'static OnceCell, + v: u32, + ) -> Result<(), SetError> { + time::advance(Duration::from_millis(1)).await; + cell.set(v) + } + + #[test] + fn get_or_init() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .unwrap(); + + static ONCE: OnceCell = OnceCell::const_new(); + + rt.block_on(async { + let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); + let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await }); + + time::advance(Duration::from_millis(1)).await; + time::resume(); + + let result1 = handle1.await.unwrap(); + let result2 = handle2.await.unwrap(); + + assert_eq!(*result1, 5); + assert_eq!(*result2, 5); + }); + } + + #[test] + fn get_or_init_panic() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + static ONCE: OnceCell = OnceCell::const_new(); + + rt.block_on(async { + time::pause(); + + let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await }); + let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await }); + + time::advance(Duration::from_millis(1)).await; + + let result1 = handle1.await.unwrap(); + let result2 = handle2.await.unwrap(); + + assert_eq!(*result1, 5); + assert_eq!(*result2, 5); + }); + } + + #[test] + fn set_and_get() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + static ONCE: OnceCell = OnceCell::const_new(); + + rt.block_on(async { + let _ = rt.spawn(async { ONCE.set(5) }).await; + let value = ONCE.get().unwrap(); + assert_eq!(*value, 5); + }); + } + + #[test] + fn get_uninit() { + static ONCE: OnceCell = OnceCell::const_new(); + let uninit = ONCE.get(); + assert!(uninit.is_none()); + } + + #[test] + fn set_twice() { + static ONCE: OnceCell = OnceCell::const_new(); + + let first = ONCE.set(5); + assert_eq!(first, Ok(())); + let second = ONCE.set(6); + assert!(second.err().unwrap().is_already_init_err()); + } + + #[test] + fn set_while_initializing() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + static ONCE: OnceCell = OnceCell::const_new(); + + rt.block_on(async { + time::pause(); + + let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await }); + let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await }); + + time::advance(Duration::from_millis(2)).await; + + let result1 = handle1.await.unwrap(); + let result2 = handle2.await.unwrap(); + + assert_eq!(*result1, 5); + assert!(result2.err().unwrap().is_initializing_err()); + }); + } + + #[test] + fn get_or_try_init() { + let rt = runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .unwrap(); + + static ONCE: OnceCell = OnceCell::const_new(); + + rt.block_on(async { + let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await }); + let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await }); + + time::advance(Duration::from_millis(1)).await; + time::resume(); + + let result1 = handle1.await.unwrap(); + assert!(result1.is_err()); + + let result2 = handle2.await.unwrap(); + assert_eq!(*result2.unwrap(), 10); + }); + } +} diff --git a/third_party/rust/tokio/tests/sync_oneshot.rs b/third_party/rust/tokio/tests/sync_oneshot.rs index 44d09e8adeb3..15b6923701f6 100644 --- a/third_party/rust/tokio/tests/sync_oneshot.rs +++ b/third_party/rust/tokio/tests/sync_oneshot.rs @@ -1,12 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; use tokio::sync::oneshot; @@ -179,7 +179,7 @@ fn explicit_close_try_recv() { #[test] #[should_panic] -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding fn close_try_recv_poll() { let (_tx, rx) = oneshot::channel::(); let mut rx = task::spawn(rx); @@ -212,6 +212,18 @@ fn try_recv_after_completion() { rx.close(); } +#[test] +fn try_recv_after_completion_await() { + let (tx, rx) = oneshot::channel::(); + let mut rx = task::spawn(rx); + + tx.send(17).unwrap(); + + assert_eq!(Ok(17), assert_ready!(rx.poll())); + assert_eq!(Err(TryRecvError::Closed), rx.try_recv()); + rx.close(); +} + #[test] fn drops_tasks() { let (mut tx, mut rx) = oneshot::channel::(); diff --git a/third_party/rust/tokio/tests/sync_panic.rs b/third_party/rust/tokio/tests/sync_panic.rs new file mode 100644 index 000000000000..6c23664998fc --- /dev/null +++ b/third_party/rust/tokio/tests/sync_panic.rs @@ -0,0 +1,197 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] + +use std::{error::Error, sync::Arc}; +use tokio::{ + runtime::{Builder, Runtime}, + sync::{broadcast, mpsc, oneshot, Mutex, RwLock, Semaphore}, +}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn broadcast_channel_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let (_, _) = broadcast::channel::(0); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mutex_blocking_lock_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let mutex = Mutex::new(5_u32); + let _g = mutex.blocking_lock(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn oneshot_blocking_recv_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let (_tx, rx) = oneshot::channel::(); + let _ = rx.blocking_recv(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn rwlock_with_max_readers_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _ = RwLock::::with_max_readers(0, (u32::MAX >> 3) + 1); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn rwlock_blocking_read_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let lock = RwLock::::new(0); + let _ = lock.blocking_read(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn rwlock_blocking_write_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + rt.block_on(async { + let lock = RwLock::::new(0); + let _ = lock.blocking_write(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_bounded_channel_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let (_, _) = mpsc::channel::(0); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_bounded_receiver_blocking_recv_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + let (_tx, mut rx) = mpsc::channel::(1); + rt.block_on(async { + let _ = rx.blocking_recv(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_bounded_sender_blocking_send_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + let (tx, _rx) = mpsc::channel::(1); + rt.block_on(async { + let _ = tx.blocking_send(3); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn mpsc_unbounded_receiver_blocking_recv_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + let (_tx, mut rx) = mpsc::unbounded_channel::(); + rt.block_on(async { + let _ = rx.blocking_recv(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn semaphore_merge_unrelated_owned_permits() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let sem1 = Arc::new(Semaphore::new(42)); + let sem2 = Arc::new(Semaphore::new(42)); + let mut p1 = sem1.try_acquire_owned().unwrap(); + let p2 = sem2.try_acquire_owned().unwrap(); + p1.merge(p2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn semaphore_merge_unrelated_permits() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let sem1 = Semaphore::new(42); + let sem2 = Semaphore::new(42); + let mut p1 = sem1.try_acquire().unwrap(); + let p2 = sem2.try_acquire().unwrap(); + p1.merge(p2); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +fn current_thread() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} diff --git a/third_party/rust/tokio/tests/sync_rwlock.rs b/third_party/rust/tokio/tests/sync_rwlock.rs index 01bbdbb5a58c..667f621721f8 100644 --- a/third_party/rust/tokio/tests/sync_rwlock.rs +++ b/third_party/rust/tokio/tests/sync_rwlock.rs @@ -1,19 +1,19 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(tokio_wasm_not_wasi))] use tokio::test as maybe_tokio_test; use std::task::Poll; use futures::future::FutureExt; -use tokio::sync::RwLock; +use tokio::sync::{RwLock, RwLockWriteGuard}; use tokio_test::task::spawn; use tokio_test::{assert_pending, assert_ready}; @@ -31,7 +31,7 @@ fn read_shared() { let mut t1 = spawn(rwlock.read()); let _g1 = assert_ready!(t1.poll()); let mut t2 = spawn(rwlock.read()); - assert_ready!(t2.poll()); + let _g2 = assert_ready!(t2.poll()); } // When there is an active shared owner, exclusive access should not be possible @@ -75,7 +75,7 @@ fn exhaust_reading() { let g2 = reads.pop().unwrap(); drop(g2); assert!(t1.is_woken()); - assert_ready!(t1.poll()); + let _g1 = assert_ready!(t1.poll()); } // When there is an active exclusive owner, subsequent exclusive access should not be possible @@ -100,7 +100,7 @@ fn write_shared_drop() { assert_pending!(t2.poll()); drop(g1); assert!(t2.is_woken()); - assert_ready!(t2.poll()); + let _g2 = assert_ready!(t2.poll()); } // when there is an active shared owner, and exclusive access is triggered, @@ -112,7 +112,7 @@ fn write_read_shared_pending() { let _g1 = assert_ready!(t1.poll()); let mut t2 = spawn(rwlock.read()); - assert_ready!(t2.poll()); + let _g2 = assert_ready!(t2.poll()); let mut t3 = spawn(rwlock.write()); assert_pending!(t3.poll()); @@ -137,7 +137,7 @@ fn write_read_shared_drop_pending() { drop(t2); assert!(t3.is_woken()); - assert_ready!(t3.poll()); + let _t3 = assert_ready!(t3.poll()); } // Acquire an RwLock nonexclusively by a single task @@ -172,7 +172,7 @@ async fn write_order() { } // A single RwLock is contested by tasks in multiple threads -#[cfg(feature = "full")] +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn multithreaded() { use futures::stream::{self, StreamExt}; @@ -279,3 +279,53 @@ fn try_read_try_write() { assert_eq!(*lock.try_read().unwrap(), 1515); } + +#[maybe_tokio_test] +async fn downgrade_map() { + let lock = RwLock::new(0); + let write_guard = lock.write().await; + let mut read_t = spawn(lock.read()); + + // We can't create a read when a write exists + assert_pending!(read_t.poll()); + + // During the call to `f`, `read_t` doesn't have access yet. + let read_guard1 = RwLockWriteGuard::downgrade_map(write_guard, |v| { + assert_pending!(read_t.poll()); + v + }); + + // After the downgrade, `read_t` got the lock + let read_guard2 = assert_ready!(read_t.poll()); + + // Ensure they're equal, as we return the original value + assert_eq!(&*read_guard1 as *const _, &*read_guard2 as *const _); +} + +#[maybe_tokio_test] +async fn try_downgrade_map() { + let lock = RwLock::new(0); + let write_guard = lock.write().await; + let mut read_t = spawn(lock.read()); + + // We can't create a read when a write exists + assert_pending!(read_t.poll()); + + // During the call to `f`, `read_t` doesn't have access yet. + let write_guard = RwLockWriteGuard::try_downgrade_map(write_guard, |_| { + assert_pending!(read_t.poll()); + None::<&()> + }) + .expect_err("downgrade didn't fail"); + + // After `f` returns `None`, `read_t` doesn't have access + assert_pending!(read_t.poll()); + + // After `f` returns `Some`, `read_t` does have access + let read_guard1 = RwLockWriteGuard::try_downgrade_map(write_guard, |v| Some(v)) + .expect("downgrade didn't succeed"); + let read_guard2 = assert_ready!(read_t.poll()); + + // Ensure they're equal, as we return the original value + assert_eq!(&*read_guard1 as *const _, &*read_guard2 as *const _); +} diff --git a/third_party/rust/tokio/tests/sync_semaphore.rs b/third_party/rust/tokio/tests/sync_semaphore.rs index d9266650085c..3d47ed0ed732 100644 --- a/third_party/rust/tokio/tests/sync_semaphore.rs +++ b/third_party/rust/tokio/tests/sync_semaphore.rs @@ -1,6 +1,6 @@ #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::Arc; @@ -63,9 +63,34 @@ fn forget() { assert!(sem.try_acquire().is_err()); } +#[test] +fn merge() { + let sem = Arc::new(Semaphore::new(3)); + { + let mut p1 = sem.try_acquire().unwrap(); + assert_eq!(sem.available_permits(), 2); + let p2 = sem.try_acquire_many(2).unwrap(); + assert_eq!(sem.available_permits(), 0); + p1.merge(p2); + assert_eq!(sem.available_permits(), 0); + } + assert_eq!(sem.available_permits(), 3); +} + +#[test] +#[cfg(not(tokio_wasm))] // No stack unwinding on wasm targets +#[should_panic] +fn merge_unrelated_permits() { + let sem1 = Arc::new(Semaphore::new(3)); + let sem2 = Arc::new(Semaphore::new(3)); + let mut p1 = sem1.try_acquire().unwrap(); + let p2 = sem2.try_acquire().unwrap(); + p1.merge(p2); +} + #[tokio::test] #[cfg(feature = "full")] -async fn stresstest() { +async fn stress_test() { let sem = Arc::new(Semaphore::new(5)); let mut join_handles = Vec::new(); for _ in 0..1000 { @@ -89,14 +114,37 @@ async fn stresstest() { #[test] fn add_max_amount_permits() { let s = tokio::sync::Semaphore::new(0); - s.add_permits(usize::MAX >> 3); - assert_eq!(s.available_permits(), usize::MAX >> 3); + s.add_permits(tokio::sync::Semaphore::MAX_PERMITS); + assert_eq!(s.available_permits(), tokio::sync::Semaphore::MAX_PERMITS); } -#[cfg(not(target_arch = "wasm32"))] // wasm currently doesn't support unwinding +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding #[test] #[should_panic] -fn add_more_than_max_amount_permits() { +fn add_more_than_max_amount_permits1() { let s = tokio::sync::Semaphore::new(1); - s.add_permits(usize::MAX >> 3); + s.add_permits(tokio::sync::Semaphore::MAX_PERMITS); +} + +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +#[test] +#[should_panic] +fn add_more_than_max_amount_permits2() { + let s = Semaphore::new(Semaphore::MAX_PERMITS - 1); + s.add_permits(1); + s.add_permits(1); +} + +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +#[test] +#[should_panic] +fn panic_when_exceeds_maxpermits() { + let _ = Semaphore::new(Semaphore::MAX_PERMITS + 1); +} + +#[test] +fn no_panic_at_maxpermits() { + let _ = Semaphore::new(Semaphore::MAX_PERMITS); + let s = Semaphore::new(Semaphore::MAX_PERMITS - 1); + s.add_permits(1); } diff --git a/third_party/rust/tokio/tests/sync_semaphore_owned.rs b/third_party/rust/tokio/tests/sync_semaphore_owned.rs index 98c20d7b0353..f69457647862 100644 --- a/third_party/rust/tokio/tests/sync_semaphore_owned.rs +++ b/third_party/rust/tokio/tests/sync_semaphore_owned.rs @@ -1,6 +1,6 @@ #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::Arc; @@ -89,9 +89,34 @@ fn forget() { assert!(sem.try_acquire_owned().is_err()); } +#[test] +fn merge() { + let sem = Arc::new(Semaphore::new(3)); + { + let mut p1 = sem.clone().try_acquire_owned().unwrap(); + assert_eq!(sem.available_permits(), 2); + let p2 = sem.clone().try_acquire_many_owned(2).unwrap(); + assert_eq!(sem.available_permits(), 0); + p1.merge(p2); + assert_eq!(sem.available_permits(), 0); + } + assert_eq!(sem.available_permits(), 3); +} + +#[test] +#[cfg(not(tokio_wasm))] // No stack unwinding on wasm targets +#[should_panic] +fn merge_unrelated_permits() { + let sem1 = Arc::new(Semaphore::new(3)); + let sem2 = Arc::new(Semaphore::new(3)); + let mut p1 = sem1.try_acquire_owned().unwrap(); + let p2 = sem2.try_acquire_owned().unwrap(); + p1.merge(p2) +} + #[tokio::test] #[cfg(feature = "full")] -async fn stresstest() { +async fn stress_test() { let sem = Arc::new(Semaphore::new(5)); let mut join_handles = Vec::new(); for _ in 0..1000 { diff --git a/third_party/rust/tokio/tests/sync_watch.rs b/third_party/rust/tokio/tests/sync_watch.rs index 8b9ea81bb890..d4f8ce87d958 100644 --- a/third_party/rust/tokio/tests/sync_watch.rs +++ b/third_party/rust/tokio/tests/sync_watch.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(target_arch = "wasm32")] +#[cfg(tokio_wasm_not_wasi)] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::watch; @@ -211,3 +211,33 @@ fn reopened_after_subscribe() { drop(rx); assert!(tx.is_closed()); } + +#[test] +#[cfg(panic = "unwind")] +#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding +fn send_modify_panic() { + let (tx, mut rx) = watch::channel("one"); + + tx.send_modify(|old| *old = "two"); + assert_eq!(*rx.borrow_and_update(), "two"); + + let mut rx2 = rx.clone(); + assert_eq!(*rx2.borrow_and_update(), "two"); + + let mut task = spawn(rx2.changed()); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + tx.send_modify(|old| { + *old = "panicked"; + panic!(); + }) + })); + assert!(result.is_err()); + + assert_pending!(task.poll()); + assert_eq!(*rx.borrow(), "panicked"); + + tx.send_modify(|old| *old = "three"); + assert_ready_ok!(task.poll()); + assert_eq!(*rx.borrow_and_update(), "three"); +} diff --git a/third_party/rust/tokio/tests/task_abort.rs b/third_party/rust/tokio/tests/task_abort.rs index fe6b50cd46e1..492f8b551a34 100644 --- a/third_party/rust/tokio/tests/task_abort.rs +++ b/third_party/rust/tokio/tests/task_abort.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery use std::sync::Arc; use std::thread::sleep; @@ -26,10 +26,7 @@ fn test_abort_without_panic_3157() { .unwrap(); rt.block_on(async move { - let handle = tokio::spawn(async move { - println!("task started"); - tokio::time::sleep(Duration::new(100, 0)).await - }); + let handle = tokio::spawn(async move { tokio::time::sleep(Duration::new(100, 0)).await }); // wait for task to sleep. tokio::time::sleep(Duration::from_millis(10)).await; @@ -89,7 +86,7 @@ fn test_abort_without_panic_3662() { // Note: We do the following to trigger a deferred task cleanup. // // The relevant piece of code you want to look at is in: - // `Inner::block_on` of `basic_scheduler.rs`. + // `Inner::block_on` of `scheduler/current_thread.rs`. // // We cause the cleanup to happen by having a poll return Pending once // so that the scheduler can go into the "auxiliary tasks" mode, at @@ -159,7 +156,6 @@ fn test_abort_wakes_task_3964() { let handle = tokio::spawn(async move { // Make sure the Arc is moved into the task let _notify_dropped = notify_dropped; - println!("task started"); tokio::time::sleep(Duration::new(100, 0)).await }); @@ -187,7 +183,6 @@ fn test_abort_task_that_panics_on_drop_contained() { let handle = tokio::spawn(async move { // Make sure the Arc is moved into the task let _panic_dropped = PanicOnDrop; - println!("task started"); tokio::time::sleep(Duration::new(100, 0)).await }); @@ -211,7 +206,6 @@ fn test_abort_task_that_panics_on_drop_returned() { let handle = tokio::spawn(async move { // Make sure the Arc is moved into the task let _panic_dropped = PanicOnDrop; - println!("task started"); tokio::time::sleep(Duration::new(100, 0)).await }); diff --git a/third_party/rust/tokio/tests/task_blocking.rs b/third_party/rust/tokio/tests/task_blocking.rs index ee7e78ad4823..2999758ff36a 100644 --- a/third_party/rust/tokio/tests/task_blocking.rs +++ b/third_party/rust/tokio/tests/task_blocking.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads -use tokio::{runtime, task}; +use tokio::{runtime, task, time}; use tokio_test::assert_ok; use std::thread; @@ -77,7 +77,7 @@ async fn block_in_block() { #[tokio::test(flavor = "current_thread")] #[should_panic] -async fn no_block_in_basic_scheduler() { +async fn no_block_in_current_thread_scheduler() { task::block_in_place(|| {}); } @@ -91,7 +91,7 @@ fn yes_block_in_threaded_block_on() { #[test] #[should_panic] -fn no_block_in_basic_block_on() { +fn no_block_in_current_thread_block_on() { let rt = runtime::Builder::new_current_thread().build().unwrap(); rt.block_on(async { task::block_in_place(|| {}); @@ -99,7 +99,7 @@ fn no_block_in_basic_block_on() { } #[test] -fn can_enter_basic_rt_from_within_block_in_place() { +fn can_enter_current_thread_rt_from_within_block_in_place() { let outer = tokio::runtime::Runtime::new().unwrap(); outer.block_on(async { @@ -226,3 +226,84 @@ fn coop_disabled_in_block_in_place_in_block_on() { done_rx.recv().unwrap().unwrap(); } + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn blocking_when_paused() { + // Do not auto-advance time when we have started a blocking task that has + // not yet finished. + time::timeout( + Duration::from_secs(3), + task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))), + ) + .await + .expect("timeout should not trigger") + .expect("blocking task should finish"); + + // Really: Do not auto-advance time, even if the timeout is short and the + // blocking task runs for longer than that. It doesn't matter: Tokio time + // is paused; system time is not. + time::timeout( + Duration::from_millis(1), + task::spawn_blocking(|| thread::sleep(Duration::from_millis(50))), + ) + .await + .expect("timeout should not trigger") + .expect("blocking task should finish"); +} + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn blocking_task_wakes_paused_runtime() { + let t0 = std::time::Instant::now(); + time::timeout( + Duration::from_secs(15), + task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))), + ) + .await + .expect("timeout should not trigger") + .expect("blocking task should finish"); + assert!( + t0.elapsed() < Duration::from_secs(10), + "completing a spawn_blocking should wake the scheduler if it's parked while time is paused" + ); +} + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn unawaited_blocking_task_wakes_paused_runtime() { + let t0 = std::time::Instant::now(); + + // When this task finishes, time should auto-advance, even though the + // JoinHandle has not been awaited yet. + let a = task::spawn_blocking(|| { + thread::sleep(Duration::from_millis(1)); + }); + + crate::time::sleep(Duration::from_secs(15)).await; + a.await.expect("blocking task should finish"); + assert!( + t0.elapsed() < Duration::from_secs(10), + "completing a spawn_blocking should wake the scheduler if it's parked while time is paused" + ); +} + +#[cfg(feature = "test-util")] +#[tokio::test(start_paused = true)] +async fn panicking_blocking_task_wakes_paused_runtime() { + let t0 = std::time::Instant::now(); + let result = time::timeout( + Duration::from_secs(15), + task::spawn_blocking(|| { + thread::sleep(Duration::from_millis(1)); + panic!("blocking task panicked"); + }), + ) + .await + .expect("timeout should not trigger"); + assert!(result.is_err(), "blocking task should have panicked"); + assert!( + t0.elapsed() < Duration::from_secs(10), + "completing a spawn_blocking should wake the scheduler if it's parked while time is paused" + ); +} diff --git a/third_party/rust/tokio/tests/task_builder.rs b/third_party/rust/tokio/tests/task_builder.rs index 1499abf19e4f..78329ff26a4b 100644 --- a/third_party/rust/tokio/tests/task_builder.rs +++ b/third_party/rust/tokio/tests/task_builder.rs @@ -11,6 +11,7 @@ mod tests { let result = Builder::new() .name("name") .spawn(async { "task executed" }) + .unwrap() .await; assert_eq!(result.unwrap(), "task executed"); @@ -21,6 +22,7 @@ mod tests { let result = Builder::new() .name("name") .spawn_blocking(|| "task executed") + .unwrap() .await; assert_eq!(result.unwrap(), "task executed"); @@ -34,6 +36,7 @@ mod tests { Builder::new() .name("name") .spawn_local(async move { unsend_data }) + .unwrap() .await }) .await; @@ -43,14 +46,20 @@ mod tests { #[test] async fn spawn_without_name() { - let result = Builder::new().spawn(async { "task executed" }).await; + let result = Builder::new() + .spawn(async { "task executed" }) + .unwrap() + .await; assert_eq!(result.unwrap(), "task executed"); } #[test] async fn spawn_blocking_without_name() { - let result = Builder::new().spawn_blocking(|| "task executed").await; + let result = Builder::new() + .spawn_blocking(|| "task executed") + .unwrap() + .await; assert_eq!(result.unwrap(), "task executed"); } @@ -59,7 +68,12 @@ mod tests { async fn spawn_local_without_name() { let unsend_data = Rc::new("task executed"); let result = LocalSet::new() - .run_until(async move { Builder::new().spawn_local(async move { unsend_data }).await }) + .run_until(async move { + Builder::new() + .spawn_local(async move { unsend_data }) + .unwrap() + .await + }) .await; assert_eq!(*result.unwrap(), "task executed"); diff --git a/third_party/rust/tokio/tests/task_id.rs b/third_party/rust/tokio/tests/task_id.rs new file mode 100644 index 000000000000..d7b7c0cd812a --- /dev/null +++ b/third_party/rust/tokio/tests/task_id.rs @@ -0,0 +1,303 @@ +#![warn(rust_2018_idioms)] +#![allow(clippy::declare_interior_mutable_const)] +#![cfg(all(feature = "full", tokio_unstable))] + +#[cfg(not(tokio_wasi))] +use std::error::Error; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +#[cfg(not(tokio_wasi))] +use tokio::runtime::{Builder, Runtime}; +use tokio::sync::oneshot; +use tokio::task::{self, Id, LocalSet}; + +#[cfg(not(tokio_wasi))] +mod support { + pub mod panic; +} +#[cfg(not(tokio_wasi))] +use support::panic::test_panic; + +#[tokio::test(flavor = "current_thread")] +async fn task_id_spawn() { + tokio::spawn(async { println!("task id: {}", task::id()) }) + .await + .unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "current_thread")] +async fn task_id_spawn_blocking() { + task::spawn_blocking(|| println!("task id: {}", task::id())) + .await + .unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_collision_current_thread() { + let handle1 = tokio::spawn(async { task::id() }); + let handle2 = tokio::spawn(async { task::id() }); + + let (id1, id2) = tokio::join!(handle1, handle2); + assert_ne!(id1.unwrap(), id2.unwrap()); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_collision_multi_thread() { + let handle1 = tokio::spawn(async { task::id() }); + let handle2 = tokio::spawn(async { task::id() }); + + let (id1, id2) = tokio::join!(handle1, handle2); + assert_ne!(id1.unwrap(), id2.unwrap()); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_ids_match_current_thread() { + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(async { + let id = rx.await.unwrap(); + assert_eq!(id, task::id()); + }); + tx.send(handle.id()).unwrap(); + handle.await.unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_ids_match_multi_thread() { + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(async { + let id = rx.await.unwrap(); + assert_eq!(id, task::id()); + }); + tx.send(handle.id()).unwrap(); + handle.await.unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_future_destructor_completion() { + struct MyFuture { + tx: Option>, + } + + impl Future for MyFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + Poll::Ready(()) + } + } + + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { tx: Some(tx) }); + let id = handle.id(); + handle.await.unwrap(); + assert_eq!(rx.await.unwrap(), id); +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_future_destructor_abort() { + struct MyFuture { + tx: Option>, + } + + impl Future for MyFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + Poll::Pending + } + } + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + let (tx, rx) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { tx: Some(tx) }); + let id = handle.id(); + handle.abort(); + assert!(handle.await.unwrap_err().is_cancelled()); + assert_eq!(rx.await.unwrap(), id); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_output_destructor_handle_dropped_before_completion() { + struct MyOutput { + tx: Option>, + } + + impl Drop for MyOutput { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + struct MyFuture { + tx: Option>, + } + + impl Future for MyFuture { + type Output = MyOutput; + + fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { + Poll::Ready(MyOutput { tx: self.tx.take() }) + } + } + + let (tx, mut rx) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { tx: Some(tx) }); + let id = handle.id(); + drop(handle); + assert!(rx.try_recv().is_err()); + assert_eq!(rx.await.unwrap(), id); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_output_destructor_handle_dropped_after_completion() { + struct MyOutput { + tx: Option>, + } + + impl Drop for MyOutput { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(task::id()); + } + } + + struct MyFuture { + tx_output: Option>, + tx_future: Option>, + } + + impl Future for MyFuture { + type Output = MyOutput; + + fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { + let _ = self.tx_future.take().unwrap().send(()); + Poll::Ready(MyOutput { + tx: self.tx_output.take(), + }) + } + } + + let (tx_output, mut rx_output) = oneshot::channel(); + let (tx_future, rx_future) = oneshot::channel(); + let handle = tokio::spawn(MyFuture { + tx_output: Some(tx_output), + tx_future: Some(tx_future), + }); + let id = handle.id(); + rx_future.await.unwrap(); + assert!(rx_output.try_recv().is_err()); + drop(handle); + assert_eq!(rx_output.await.unwrap(), id); +} + +#[test] +fn task_try_id_outside_task() { + assert_eq!(None, task::try_id()); +} + +#[cfg(not(tokio_wasi))] +#[test] +fn task_try_id_inside_block_on() { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + assert_eq!(None, task::try_id()); + }); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_spawn_local() { + LocalSet::new() + .run_until(async { + task::spawn_local(async { println!("task id: {}", task::id()) }) + .await + .unwrap(); + }) + .await +} + +#[tokio::test(flavor = "current_thread")] +async fn task_id_nested_spawn_local() { + LocalSet::new() + .run_until(async { + task::spawn_local(async { + let parent_id = task::id(); + LocalSet::new() + .run_until(async { + task::spawn_local(async move { + assert_ne!(parent_id, task::id()); + }) + .await + .unwrap(); + }) + .await; + assert_eq!(parent_id, task::id()); + }) + .await + .unwrap(); + }) + .await; +} + +#[cfg(not(tokio_wasi))] +#[tokio::test(flavor = "multi_thread")] +async fn task_id_block_in_place_block_on_spawn() { + task::spawn(async { + let parent_id = task::id(); + + task::block_in_place(move || { + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(rt.spawn(async move { + assert_ne!(parent_id, task::id()); + })) + .unwrap(); + }); + + assert_eq!(parent_id, task::id()); + }) + .await + .unwrap(); +} + +#[cfg(not(tokio_wasi))] +#[test] +fn task_id_outside_task_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _ = task::id(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[cfg(not(tokio_wasi))] +#[test] +fn task_id_inside_block_on_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + task::id(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/third_party/rust/tokio/tests/task_join_set.rs b/third_party/rust/tokio/tests/task_join_set.rs index 66a2fbb021d0..b1b6cf9665ff 100644 --- a/third_party/rust/tokio/tests/task_join_set.rs +++ b/third_party/rust/tokio/tests/task_join_set.rs @@ -1,12 +1,10 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", tokio_unstable))] +#![cfg(all(feature = "full"))] use tokio::sync::oneshot; use tokio::task::JoinSet; use tokio::time::Duration; -use futures::future::FutureExt; - fn rt() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_current_thread() .build() @@ -24,7 +22,7 @@ async fn test_with_sleep() { set.detach_all(); assert_eq!(set.len(), 0); - assert!(matches!(set.join_one().await, Ok(None))); + assert!(matches!(set.join_next().await, None)); for i in 0..10 { set.spawn(async move { @@ -35,14 +33,14 @@ async fn test_with_sleep() { } let mut seen = [false; 10]; - while let Some(res) = set.join_one().await.unwrap() { + while let Some(res) = set.join_next().await.transpose().unwrap() { seen[res] = true; } for was_seen in &seen { assert!(was_seen); } - assert!(matches!(set.join_one().await, Ok(None))); + assert!(matches!(set.join_next().await, None)); // Do it again. for i in 0..10 { @@ -53,14 +51,14 @@ async fn test_with_sleep() { } let mut seen = [false; 10]; - while let Some(res) = set.join_one().await.unwrap() { + while let Some(res) = set.join_next().await.transpose().unwrap() { seen[res] = true; } for was_seen in &seen { assert!(was_seen); } - assert!(matches!(set.join_one().await, Ok(None))); + assert!(matches!(set.join_next().await, None)); } #[tokio::test] @@ -99,13 +97,47 @@ async fn alternating() { assert_eq!(set.len(), 2); for _ in 0..16 { - let () = set.join_one().await.unwrap().unwrap(); + let () = set.join_next().await.unwrap().unwrap(); assert_eq!(set.len(), 1); set.spawn(async {}); assert_eq!(set.len(), 2); } } +#[tokio::test(start_paused = true)] +async fn abort_tasks() { + let mut set = JoinSet::new(); + let mut num_canceled = 0; + let mut num_completed = 0; + for i in 0..16 { + let abort = set.spawn(async move { + tokio::time::sleep(Duration::from_secs(i as u64)).await; + i + }); + + if i % 2 != 0 { + // abort odd-numbered tasks. + abort.abort(); + } + } + loop { + match set.join_next().await { + Some(Ok(res)) => { + num_completed += 1; + assert_eq!(res % 2, 0); + } + Some(Err(e)) => { + assert!(e.is_cancelled()); + num_canceled += 1; + } + None => break, + } + } + + assert_eq!(num_canceled, 8); + assert_eq!(num_completed, 8); +} + #[test] fn runtime_gone() { let mut set = JoinSet::new(); @@ -115,50 +147,11 @@ fn runtime_gone() { drop(rt); } - assert!(rt().block_on(set.join_one()).unwrap_err().is_cancelled()); -} - -// This ensures that `join_one` works correctly when the coop budget is -// exhausted. -#[tokio::test(flavor = "current_thread")] -async fn join_set_coop() { - // Large enough to trigger coop. - const TASK_NUM: u32 = 1000; - - static SEM: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(0); - - let mut set = JoinSet::new(); - - for _ in 0..TASK_NUM { - set.spawn(async { - SEM.add_permits(1); - }); - } - - // Wait for all tasks to complete. - // - // Since this is a `current_thread` runtime, there's no race condition - // between the last permit being added and the task completing. - let _ = SEM.acquire_many(TASK_NUM).await.unwrap(); - - let mut count = 0; - let mut coop_count = 0; - loop { - match set.join_one().now_or_never() { - Some(Ok(Some(()))) => {} - Some(Err(err)) => panic!("failed: {}", err), - None => { - coop_count += 1; - tokio::task::yield_now().await; - continue; - } - Some(Ok(None)) => break, - } - - count += 1; - } - assert!(coop_count >= 1); - assert_eq!(count, TASK_NUM); + assert!(rt() + .block_on(set.join_next()) + .unwrap() + .unwrap_err() + .is_cancelled()); } #[tokio::test(start_paused = true)] @@ -181,7 +174,7 @@ async fn abort_all() { assert_eq!(set.len(), 10); let mut count = 0; - while let Some(res) = set.join_one().await.transpose() { + while let Some(res) = set.join_next().await { if let Err(err) = res { assert!(err.is_cancelled()); } @@ -190,3 +183,53 @@ async fn abort_all() { assert_eq!(count, 10); assert_eq!(set.len(), 0); } + +#[cfg(feature = "parking_lot")] +mod parking_lot { + use super::*; + + use futures::future::FutureExt; + + // This ensures that `join_next` works correctly when the coop budget is + // exhausted. + #[tokio::test(flavor = "current_thread")] + async fn join_set_coop() { + // Large enough to trigger coop. + const TASK_NUM: u32 = 1000; + + static SEM: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(0); + + let mut set = JoinSet::new(); + + for _ in 0..TASK_NUM { + set.spawn(async { + SEM.add_permits(1); + }); + } + + // Wait for all tasks to complete. + // + // Since this is a `current_thread` runtime, there's no race condition + // between the last permit being added and the task completing. + let _ = SEM.acquire_many(TASK_NUM).await.unwrap(); + + let mut count = 0; + let mut coop_count = 0; + loop { + match set.join_next().now_or_never() { + Some(Some(Ok(()))) => {} + Some(Some(Err(err))) => panic!("failed: {}", err), + None => { + coop_count += 1; + tokio::task::yield_now().await; + continue; + } + Some(None) => break, + } + + count += 1; + } + assert!(coop_count >= 1); + assert_eq!(count, TASK_NUM); + } +} diff --git a/third_party/rust/tokio/tests/task_local.rs b/third_party/rust/tokio/tests/task_local.rs index 811d63ea0f8e..949a40c2a4d1 100644 --- a/third_party/rust/tokio/tests/task_local.rs +++ b/third_party/rust/tokio/tests/task_local.rs @@ -1,12 +1,17 @@ -#![cfg(feature = "full")] - -tokio::task_local! { - static REQ_ID: u32; - pub static FOO: bool; -} +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads +#![allow(clippy::declare_interior_mutable_const)] +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::sync::oneshot; #[tokio::test(flavor = "multi_thread")] async fn local() { + tokio::task_local! { + static REQ_ID: u32; + pub static FOO: bool; + } + let j1 = tokio::spawn(REQ_ID.scope(1, async move { assert_eq!(REQ_ID.get(), 1); assert_eq!(REQ_ID.get(), 1); @@ -31,3 +36,84 @@ async fn local() { j2.await.unwrap(); j3.await.unwrap(); } + +#[tokio::test] +async fn task_local_available_on_abort() { + tokio::task_local! { + static KEY: u32; + } + + struct MyFuture { + tx_poll: Option>, + tx_drop: Option>, + } + impl Future for MyFuture { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + if let Some(tx_poll) = self.tx_poll.take() { + let _ = tx_poll.send(()); + } + Poll::Pending + } + } + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx_drop.take().unwrap().send(KEY.get()); + } + } + + let (tx_drop, rx_drop) = oneshot::channel(); + let (tx_poll, rx_poll) = oneshot::channel(); + + let h = tokio::spawn(KEY.scope( + 42, + MyFuture { + tx_poll: Some(tx_poll), + tx_drop: Some(tx_drop), + }, + )); + + rx_poll.await.unwrap(); + h.abort(); + assert_eq!(rx_drop.await.unwrap(), 42); + + let err = h.await.unwrap_err(); + if !err.is_cancelled() { + if let Ok(panic) = err.try_into_panic() { + std::panic::resume_unwind(panic); + } else { + panic!(); + } + } +} + +#[tokio::test] +async fn task_local_available_on_completion_drop() { + tokio::task_local! { + static KEY: u32; + } + + struct MyFuture { + tx: Option>, + } + impl Future for MyFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + Poll::Ready(()) + } + } + impl Drop for MyFuture { + fn drop(&mut self) { + let _ = self.tx.take().unwrap().send(KEY.get()); + } + } + + let (tx, rx) = oneshot::channel(); + + let h = tokio::spawn(KEY.scope(42, MyFuture { tx: Some(tx) })); + + assert_eq!(rx.await.unwrap(), 42); + h.await.unwrap(); +} diff --git a/third_party/rust/tokio/tests/task_local_set.rs b/third_party/rust/tokio/tests/task_local_set.rs index f8a35d0ede86..2da87f5aed24 100644 --- a/third_party/rust/tokio/tests/task_local_set.rs +++ b/third_party/rust/tokio/tests/task_local_set.rs @@ -6,18 +6,23 @@ use futures::{ FutureExt, }; -use tokio::runtime::{self, Runtime}; +use tokio::runtime; use tokio::sync::{mpsc, oneshot}; use tokio::task::{self, LocalSet}; use tokio::time; +#[cfg(not(tokio_wasi))] use std::cell::Cell; -use std::sync::atomic::Ordering::{self, SeqCst}; -use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::atomic::AtomicBool; +#[cfg(not(tokio_wasi))] +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +#[cfg(not(tokio_wasi))] +use std::sync::atomic::Ordering::SeqCst; use std::time::Duration; #[tokio::test(flavor = "current_thread")] -async fn local_basic_scheduler() { +async fn local_current_thread_scheduler() { LocalSet::new() .run_until(async { task::spawn_local(async {}).await.unwrap(); @@ -25,6 +30,7 @@ async fn local_basic_scheduler() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn local_threadpool() { thread_local! { @@ -45,6 +51,7 @@ async fn local_threadpool() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn localset_future_threadpool() { thread_local! { @@ -60,6 +67,7 @@ async fn localset_future_threadpool() { local.await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn localset_future_timers() { static RAN1: AtomicBool = AtomicBool::new(false); @@ -104,6 +112,7 @@ async fn localset_future_drives_all_local_futs() { assert!(RAN3.load(Ordering::SeqCst)); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn local_threadpool_timer() { // This test ensures that runtime services like the timer are properly @@ -126,7 +135,23 @@ async fn local_threadpool_timer() { }) .await; } +#[test] +fn enter_guard_spawn() { + let local = LocalSet::new(); + let _guard = local.enter(); + // Run the local task set. + let join = task::spawn_local(async { true }); + let rt = runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + local.block_on(&rt, async move { + assert!(join.await.unwrap()); + }); +} + +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[test] // This will panic, since the thread that calls `block_on` cannot use // in-place blocking inside of `block_on`. @@ -153,6 +178,7 @@ fn local_threadpool_blocking_in_place() { }); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn local_threadpool_blocking_run() { thread_local! { @@ -181,6 +207,7 @@ async fn local_threadpool_blocking_run() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn all_spawns_are_local() { use futures::future; @@ -207,6 +234,7 @@ async fn all_spawns_are_local() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread")] async fn nested_spawn_is_local() { thread_local! { @@ -242,6 +270,7 @@ async fn nested_spawn_is_local() { .await; } +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads #[test] fn join_local_future_elsewhere() { thread_local! { @@ -255,14 +284,12 @@ fn join_local_future_elsewhere() { local.block_on(&rt, async move { let (tx, rx) = oneshot::channel(); let join = task::spawn_local(async move { - println!("hello world running..."); assert!( ON_RT_THREAD.with(|cell| cell.get()), "local task must run on local thread, no matter where it is awaited" ); rx.await.unwrap(); - println!("hello world task done"); "hello world" }); let join2 = task::spawn(async move { @@ -272,16 +299,34 @@ fn join_local_future_elsewhere() { ); tx.send(()).expect("task shouldn't have ended yet"); - println!("waking up hello world..."); join.await.expect("task should complete successfully"); - - println!("hello world task joined"); }); join2.await.unwrap() }); } +// Tests for +#[cfg(not(tokio_wasi))] // Wasi doesn't support threads +#[tokio::test(flavor = "multi_thread")] +async fn localset_in_thread_local() { + thread_local! { + static LOCAL_SET: LocalSet = LocalSet::new(); + } + + // holds runtime thread until end of main fn. + let (_tx, rx) = oneshot::channel::<()>(); + let handle = tokio::runtime::Handle::current(); + + std::thread::spawn(move || { + LOCAL_SET.with(|local_set| { + handle.block_on(local_set.run_until(async move { + let _ = rx.await; + })) + }); + }); +} + #[test] fn drop_cancels_tasks() { use std::rc::Rc; @@ -345,9 +390,7 @@ fn with_timeout(timeout: Duration, f: impl FnOnce() + Send + 'static) { ), // Did the test thread panic? We'll find out for sure when we `join` // with it. - Err(RecvTimeoutError::Disconnected) => { - println!("done_rx dropped, did the test thread panic?"); - } + Err(RecvTimeoutError::Disconnected) => {} // Test completed successfully! Ok(()) => {} } @@ -355,6 +398,7 @@ fn with_timeout(timeout: Duration, f: impl FnOnce() + Send + 'static) { thread.join().expect("test thread should not panic!") } +#[cfg_attr(tokio_wasi, ignore = "`unwrap()` in `with_timeout()` panics on Wasi")] #[test] fn drop_cancels_remote_tasks() { // This test reproduces issue #1885. @@ -377,6 +421,10 @@ fn drop_cancels_remote_tasks() { }); } +#[cfg_attr( + tokio_wasi, + ignore = "FIXME: `task::spawn_local().await.unwrap()` panics on Wasi" +)] #[test] fn local_tasks_wake_join_all() { // This test reproduces issue #2460. @@ -398,6 +446,7 @@ fn local_tasks_wake_join_all() { }); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[test] fn local_tasks_are_polled_after_tick() { // This test depends on timing, so we run it up to five times. @@ -414,6 +463,7 @@ fn local_tasks_are_polled_after_tick() { local_tasks_are_polled_after_tick_inner(); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[tokio::main(flavor = "current_thread")] async fn local_tasks_are_polled_after_tick_inner() { // Reproduces issues #1899 and #1900 @@ -450,12 +500,15 @@ async fn local_tasks_are_polled_after_tick_inner() { tx.send(()).unwrap(); } - time::sleep(Duration::from_millis(20)).await; - let rx1 = RX1.load(SeqCst); - let rx2 = RX2.load(SeqCst); - println!("EXPECT = {}; RX1 = {}; RX2 = {}", EXPECTED, rx1, rx2); - assert_eq!(EXPECTED, rx1); - assert_eq!(EXPECTED, rx2); + loop { + time::sleep(Duration::from_millis(20)).await; + let rx1 = RX1.load(SeqCst); + let rx2 = RX2.load(SeqCst); + + if rx1 == EXPECTED && rx2 == EXPECTED { + break; + } + } }); while let Some(oneshot) = rx.recv().await { @@ -517,7 +570,122 @@ async fn spawn_wakes_localset() { } } -fn rt() -> Runtime { +#[test] +fn store_local_set_in_thread_local_with_runtime() { + use tokio::runtime::Runtime; + + thread_local! { + static CURRENT: RtAndLocalSet = RtAndLocalSet::new(); + } + + struct RtAndLocalSet { + rt: Runtime, + local: LocalSet, + } + + impl RtAndLocalSet { + fn new() -> RtAndLocalSet { + RtAndLocalSet { + rt: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(), + local: LocalSet::new(), + } + } + + async fn inner_method(&self) { + self.local + .run_until(async move { + tokio::task::spawn_local(async {}); + }) + .await + } + + fn method(&self) { + self.rt.block_on(self.inner_method()); + } + } + + CURRENT.with(|f| { + f.method(); + }); +} + +#[cfg(tokio_unstable)] +mod unstable { + use tokio::runtime::UnhandledPanic; + use tokio::task::LocalSet; + + #[tokio::test] + #[should_panic( + expected = "a spawned task panicked and the LocalSet is configured to shutdown on unhandled panic" + )] + async fn shutdown_on_panic() { + LocalSet::new() + .unhandled_panic(UnhandledPanic::ShutdownRuntime) + .run_until(async { + tokio::task::spawn_local(async { + panic!("boom"); + }); + + futures::future::pending::<()>().await; + }) + .await; + } + + // This test compares that, when the task driving `run_until` has already + // consumed budget, the `run_until` future has less budget than a "spawned" + // task. + // + // "Budget" is a fuzzy metric as the Tokio runtime is able to change values + // internally. This is why the test uses indirection to test this. + #[tokio::test] + async fn run_until_does_not_get_own_budget() { + // Consume some budget + tokio::task::consume_budget().await; + + LocalSet::new() + .run_until(async { + let spawned = tokio::spawn(async { + let mut spawned_n = 0; + + { + let mut spawned = tokio_test::task::spawn(async { + loop { + spawned_n += 1; + tokio::task::consume_budget().await; + } + }); + // Poll once + assert!(!spawned.poll().is_ready()); + } + + spawned_n + }); + + let mut run_until_n = 0; + { + let mut run_until = tokio_test::task::spawn(async { + loop { + run_until_n += 1; + tokio::task::consume_budget().await; + } + }); + // Poll once + assert!(!run_until.poll().is_ready()); + } + + let spawned_n = spawned.await.unwrap(); + assert_ne!(spawned_n, 0); + assert_ne!(run_until_n, 0); + assert!(spawned_n > run_until_n); + }) + .await + } +} + +fn rt() -> runtime::Runtime { tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/third_party/rust/tokio/tests/task_panic.rs b/third_party/rust/tokio/tests/task_panic.rs new file mode 100644 index 000000000000..e4cedce27984 --- /dev/null +++ b/third_party/rust/tokio/tests/task_panic.rs @@ -0,0 +1,123 @@ +#![warn(rust_2018_idioms)] +#![allow(clippy::declare_interior_mutable_const)] +#![cfg(all(feature = "full", not(tokio_wasi)))] + +use futures::future; +use std::error::Error; +use tokio::runtime::Builder; +use tokio::task::{self, block_in_place}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn block_in_place_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = Builder::new_current_thread().enable_all().build().unwrap(); + rt.block_on(async { + block_in_place(|| {}); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_set_spawn_local_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _local = task::LocalSet::new(); + + let _ = task::spawn_local(async {}); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_set_block_on_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = Builder::new_current_thread().enable_all().build().unwrap(); + let local = task::LocalSet::new(); + + rt.block_on(async { + local.block_on(&rt, future::pending::<()>()); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn spawn_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + tokio::spawn(future::pending::<()>()); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_key_sync_scope_panic_caller() -> Result<(), Box> { + tokio::task_local! { + static NUMBER: u32; + } + + let panic_location_file = test_panic(|| { + NUMBER.sync_scope(1, || { + NUMBER.with(|_| { + NUMBER.sync_scope(1, || {}); + }); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_key_with_panic_caller() -> Result<(), Box> { + tokio::task_local! { + static NUMBER: u32; + } + + let panic_location_file = test_panic(|| { + NUMBER.with(|_| {}); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn local_key_get_panic_caller() -> Result<(), Box> { + tokio::task_local! { + static NUMBER: u32; + } + + let panic_location_file = test_panic(|| { + NUMBER.get(); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} diff --git a/third_party/rust/tokio/tests/task_yield_now.rs b/third_party/rust/tokio/tests/task_yield_now.rs new file mode 100644 index 000000000000..b16bca528192 --- /dev/null +++ b/third_party/rust/tokio/tests/task_yield_now.rs @@ -0,0 +1,16 @@ +#![cfg(all(feature = "full", tokio_unstable))] + +use tokio::task; +use tokio_test::task::spawn; + +// `yield_now` is tested within the runtime in `rt_common`. +#[test] +fn yield_now_outside_of_runtime() { + let mut task = spawn(async { + task::yield_now().await; + }); + + assert!(task.poll().is_pending()); + assert!(task.is_woken()); + assert!(task.poll().is_ready()); +} diff --git a/third_party/rust/tokio/tests/tcp_accept.rs b/third_party/rust/tokio/tests/tcp_accept.rs index 5ffb946f345c..ba4a498d1a52 100644 --- a/third_party/rust/tokio/tests/tcp_accept.rs +++ b/third_party/rust/tokio/tests/tcp_accept.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{mpsc, oneshot}; diff --git a/third_party/rust/tokio/tests/tcp_connect.rs b/third_party/rust/tokio/tests/tcp_connect.rs index cbe68fa2764e..f2384420ed30 100644 --- a/third_party/rust/tokio/tests/tcp_connect.rs +++ b/third_party/rust/tokio/tests/tcp_connect.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::net::{TcpListener, TcpStream}; use tokio::sync::oneshot; @@ -35,6 +35,7 @@ async fn connect_v4() { } #[tokio::test] +#[cfg(not(tokio_no_ipv6))] async fn connect_v6() { let srv = assert_ok!(TcpListener::bind("[::1]:0").await); let addr = assert_ok!(srv.local_addr()); diff --git a/third_party/rust/tokio/tests/tcp_echo.rs b/third_party/rust/tokio/tests/tcp_echo.rs index 5bb7ff0acc22..f47d4ac185a3 100644 --- a/third_party/rust/tokio/tests/tcp_echo.rs +++ b/third_party/rust/tokio/tests/tcp_echo.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::{self, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; diff --git a/third_party/rust/tokio/tests/tcp_into_split.rs b/third_party/rust/tokio/tests/tcp_into_split.rs index 2e06643a0281..010984af8cc2 100644 --- a/third_party/rust/tokio/tests/tcp_into_split.rs +++ b/third_party/rust/tokio/tests/tcp_into_split.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use std::io::{Error, ErrorKind, Result}; use std::io::{Read, Write}; diff --git a/third_party/rust/tokio/tests/tcp_into_std.rs b/third_party/rust/tokio/tests/tcp_into_std.rs index 4bf24c14dd70..320d9946e819 100644 --- a/third_party/rust/tokio/tests/tcp_into_std.rs +++ b/third_party/rust/tokio/tests/tcp_into_std.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use std::io::Read; use std::io::Result; diff --git a/third_party/rust/tokio/tests/tcp_peek.rs b/third_party/rust/tokio/tests/tcp_peek.rs index aecc0ac19cd2..a5fd6ba4fe53 100644 --- a/third_party/rust/tokio/tests/tcp_peek.rs +++ b/third_party/rust/tokio/tests/tcp_peek.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::AsyncReadExt; use tokio::net::TcpStream; @@ -7,7 +7,7 @@ use tokio::net::TcpStream; use tokio_test::assert_ok; use std::thread; -use std::{convert::TryInto, io::Write, net}; +use std::{io::Write, net}; #[tokio::test] async fn peek() { @@ -15,7 +15,7 @@ async fn peek() { let addr = listener.local_addr().unwrap(); let t = thread::spawn(move || assert_ok!(listener.accept()).0); - let left = net::TcpStream::connect(&addr).unwrap(); + let left = net::TcpStream::connect(addr).unwrap(); let mut right = t.join().unwrap(); let _ = right.write(&[1, 2, 3, 4]).unwrap(); diff --git a/third_party/rust/tokio/tests/tcp_shutdown.rs b/third_party/rust/tokio/tests/tcp_shutdown.rs index 536a16130ab7..1d477f37c22f 100644 --- a/third_party/rust/tokio/tests/tcp_shutdown.rs +++ b/third_party/rust/tokio/tests/tcp_shutdown.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::{self, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; diff --git a/third_party/rust/tokio/tests/tcp_socket.rs b/third_party/rust/tokio/tests/tcp_socket.rs index 3030416502a6..66309d30411c 100644 --- a/third_party/rust/tokio/tests/tcp_socket.rs +++ b/third_party/rust/tokio/tests/tcp_socket.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use std::time::Duration; use tokio::net::TcpSocket; @@ -24,6 +24,7 @@ async fn basic_usage_v4() { } #[tokio::test] +#[cfg(not(tokio_no_ipv6))] async fn basic_usage_v6() { // Create server let addr = assert_ok!("[::1]:0".parse()); diff --git a/third_party/rust/tokio/tests/tcp_split.rs b/third_party/rust/tokio/tests/tcp_split.rs index 7171dac4635b..335b21f29409 100644 --- a/third_party/rust/tokio/tests/tcp_split.rs +++ b/third_party/rust/tokio/tests/tcp_split.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use std::io::Result; use std::io::{Read, Write}; @@ -36,7 +36,7 @@ async fn split() -> Result<()> { assert_eq!(peek_len1, read_len); assert_eq!(&read_buf[..read_len], MSG); - write_half.write(MSG).await?; + assert_eq!(write_half.write(MSG).await?, MSG.len()); handle.join().unwrap(); Ok(()) } diff --git a/third_party/rust/tokio/tests/tcp_stream.rs b/third_party/rust/tokio/tests/tcp_stream.rs index 0b5d12ae8e27..31fe3baa2961 100644 --- a/third_party/rust/tokio/tests/tcp_stream.rs +++ b/third_party/rust/tokio/tests/tcp_stream.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind use tokio::io::{AsyncReadExt, AsyncWriteExt, Interest}; use tokio::net::{TcpListener, TcpStream}; @@ -254,30 +254,34 @@ async fn create_pair() -> (TcpStream, TcpStream) { (client, server) } -fn read_until_pending(stream: &mut TcpStream) { +fn read_until_pending(stream: &mut TcpStream) -> usize { let mut buf = vec![0u8; 1024 * 1024]; + let mut total = 0; loop { match stream.try_read(&mut buf) { - Ok(_) => (), + Ok(n) => total += n, Err(err) => { assert_eq!(err.kind(), io::ErrorKind::WouldBlock); break; } } } + total } -fn write_until_pending(stream: &mut TcpStream) { +fn write_until_pending(stream: &mut TcpStream) -> usize { let buf = vec![0u8; 1024 * 1024]; + let mut total = 0; loop { match stream.try_write(&buf) { - Ok(_) => (), + Ok(n) => total += n, Err(err) => { assert_eq!(err.kind(), io::ErrorKind::WouldBlock); break; } } } + total } #[tokio::test] @@ -357,3 +361,40 @@ async fn try_read_buf() { } } } + +// read_closed is a best effort event, so test only for no false positives. +#[tokio::test] +async fn read_closed() { + let (client, mut server) = create_pair().await; + + let mut ready_fut = task::spawn(client.ready(Interest::READABLE)); + assert_pending!(ready_fut.poll()); + + assert_ok!(server.write_all(b"ping").await); + + let ready_event = assert_ok!(ready_fut.await); + + assert!(!ready_event.is_read_closed()); +} + +// write_closed is a best effort event, so test only for no false positives. +#[tokio::test] +async fn write_closed() { + let (mut client, mut server) = create_pair().await; + + // Fill the write buffer. + let write_size = write_until_pending(&mut client); + let mut ready_fut = task::spawn(client.ready(Interest::WRITABLE)); + assert_pending!(ready_fut.poll()); + + // Drain the socket to make client writable. + let mut read_size = 0; + while read_size < write_size { + server.readable().await.unwrap(); + read_size += read_until_pending(&mut server); + } + + let ready_event = assert_ok!(ready_fut.await); + + assert!(!ready_event.is_write_closed()); +} diff --git a/third_party/rust/tokio/tests/time_interval.rs b/third_party/rust/tokio/tests/time_interval.rs index 186582e2e52b..42285364a1a6 100644 --- a/third_party/rust/tokio/tests/time_interval.rs +++ b/third_party/rust/tokio/tests/time_interval.rs @@ -1,10 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "full")] -use tokio::time::{self, Duration, Instant, MissedTickBehavior}; -use tokio_test::{assert_pending, assert_ready_eq, task}; +use std::pin::Pin; +use std::task::{Context, Poll}; -use std::task::Poll; +use futures::{Stream, StreamExt}; +use tokio::time::{self, Duration, Instant, Interval, MissedTickBehavior}; +use tokio_test::{assert_pending, assert_ready_eq, task}; // Takes the `Interval` task, `start` variable, and optional time deltas // For each time delta, it polls the `Interval` and asserts that the result is @@ -209,3 +211,113 @@ fn poll_next(interval: &mut task::Spawn) -> Poll { fn ms(n: u64) -> Duration { Duration::from_millis(n) } + +/// Helper struct to test the [tokio::time::Interval::poll_tick()] method. +/// +/// `poll_tick()` should register the waker in the context only if it returns +/// `Poll::Pending`, not when returning `Poll::Ready`. This struct contains an +/// interval timer and counts up on every tick when used as stream. When the +/// counter is a multiple of four, it yields the current counter value. +/// Depending on the value for `wake_on_pending`, it will reschedule itself when +/// it returns `Poll::Pending` or not. When used with `wake_on_pending=false`, +/// we expect that the stream stalls because the timer will **not** reschedule +/// the next wake-up itself once it returned `Poll::Ready`. +struct IntervalStreamer { + counter: u32, + timer: Interval, + wake_on_pending: bool, +} + +impl Stream for IntervalStreamer { + type Item = u32; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = Pin::into_inner(self); + + if this.counter > 12 { + return Poll::Ready(None); + } + + match this.timer.poll_tick(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(_) => { + this.counter += 1; + if this.counter % 4 == 0 { + Poll::Ready(Some(this.counter)) + } else { + if this.wake_on_pending { + // Schedule this task for wake-up + cx.waker().wake_by_ref(); + } + Poll::Pending + } + } + } + } +} + +#[tokio::test(start_paused = true)] +async fn stream_with_interval_poll_tick_self_waking() { + let stream = IntervalStreamer { + counter: 0, + timer: tokio::time::interval(tokio::time::Duration::from_millis(10)), + wake_on_pending: true, + }; + + let (res_tx, mut res_rx) = tokio::sync::mpsc::channel(12); + + // Wrap task in timeout so that it will finish eventually even if the stream + // stalls. + tokio::spawn(tokio::time::timeout( + tokio::time::Duration::from_millis(150), + async move { + tokio::pin!(stream); + + while let Some(item) = stream.next().await { + res_tx.send(item).await.ok(); + } + }, + )); + + let mut items = Vec::with_capacity(3); + while let Some(result) = res_rx.recv().await { + items.push(result); + } + + // We expect the stream to yield normally and thus three items. + assert_eq!(items, vec![4, 8, 12]); +} + +#[tokio::test(start_paused = true)] +async fn stream_with_interval_poll_tick_no_waking() { + let stream = IntervalStreamer { + counter: 0, + timer: tokio::time::interval(tokio::time::Duration::from_millis(10)), + wake_on_pending: false, + }; + + let (res_tx, mut res_rx) = tokio::sync::mpsc::channel(12); + + // Wrap task in timeout so that it will finish eventually even if the stream + // stalls. + tokio::spawn(tokio::time::timeout( + tokio::time::Duration::from_millis(150), + async move { + tokio::pin!(stream); + + while let Some(item) = stream.next().await { + res_tx.send(item).await.ok(); + } + }, + )); + + let mut items = Vec::with_capacity(0); + while let Some(result) = res_rx.recv().await { + items.push(result); + } + + // We expect the stream to stall because it does not reschedule itself on + // `Poll::Pending` and neither does [tokio::time::Interval] reschedule the + // task when returning `Poll::Ready`. + assert_eq!(items, vec![]); +} diff --git a/third_party/rust/tokio/tests/time_panic.rs b/third_party/rust/tokio/tests/time_panic.rs new file mode 100644 index 000000000000..aaff11bb4f0b --- /dev/null +++ b/third_party/rust/tokio/tests/time_panic.rs @@ -0,0 +1,93 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery + +use futures::future; +use std::error::Error; +use std::time::Duration; +use tokio::runtime::{Builder, Runtime}; +use tokio::time::{self, interval, interval_at, timeout, Instant}; + +mod support { + pub mod panic; +} +use support::panic::test_panic; + +#[test] +fn pause_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + + rt.block_on(async { + time::pause(); + time::pause(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn resume_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let rt = current_thread(); + + rt.block_on(async { + time::resume(); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn interval_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _ = interval(Duration::from_millis(0)); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn interval_at_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + let _ = interval_at(Instant::now(), Duration::from_millis(0)); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +#[test] +fn timeout_panic_caller() -> Result<(), Box> { + let panic_location_file = test_panic(|| { + // Runtime without `enable_time` so it has no current timer set. + let rt = Builder::new_current_thread().build().unwrap(); + rt.block_on(async { + let _ = timeout(Duration::from_millis(5), future::pending::<()>()); + }); + }); + + // The panic location should be in this file + assert_eq!(&panic_location_file.unwrap(), file!()); + + Ok(()) +} + +fn current_thread() -> Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} diff --git a/third_party/rust/tokio/tests/time_pause.rs b/third_party/rust/tokio/tests/time_pause.rs index 02e050a2dc47..c772e387384b 100644 --- a/third_party/rust/tokio/tests/time_pause.rs +++ b/third_party/rust/tokio/tests/time_pause.rs @@ -4,7 +4,10 @@ use rand::SeedableRng; use rand::{rngs::StdRng, Rng}; use tokio::time::{self, Duration, Instant, Sleep}; -use tokio_test::{assert_elapsed, assert_err, assert_pending, assert_ready, assert_ready_eq, task}; +use tokio_test::{assert_elapsed, assert_pending, assert_ready, assert_ready_eq, task}; + +#[cfg(not(tokio_wasi))] +use tokio_test::assert_err; use std::{ future::Future, @@ -26,12 +29,14 @@ async fn pause_time_in_task() { t.await.unwrap(); } +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[should_panic] async fn pause_time_in_main_threads() { tokio::time::pause(); } +#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn pause_time_in_spawn_threads() { let t = tokio::spawn(async { diff --git a/third_party/rust/tokio/tests/time_rt.rs b/third_party/rust/tokio/tests/time_rt.rs index 23367be41881..20f9e181b4b1 100644 --- a/third_party/rust/tokio/tests/time_rt.rs +++ b/third_party/rust/tokio/tests/time_rt.rs @@ -5,6 +5,7 @@ use tokio::time::*; use std::sync::mpsc; +#[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] // Wasi doesn't support threads #[test] fn timer_with_threaded_runtime() { use tokio::runtime::Runtime; @@ -25,7 +26,7 @@ fn timer_with_threaded_runtime() { } #[test] -fn timer_with_basic_scheduler() { +fn timer_with_current_thread_scheduler() { use tokio::runtime::Builder; let rt = Builder::new_current_thread().enable_all().build().unwrap(); diff --git a/third_party/rust/tokio/tests/time_sleep.rs b/third_party/rust/tokio/tests/time_sleep.rs index 97cadecb0a59..94022e3c00cf 100644 --- a/third_party/rust/tokio/tests/time_sleep.rs +++ b/third_party/rust/tokio/tests/time_sleep.rs @@ -168,6 +168,7 @@ async fn reset_sleep_to_past() { assert_ready!(sleep.poll()); } +#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery #[test] #[should_panic] fn creating_sleep_outside_of_context() { @@ -188,10 +189,7 @@ async fn greater_than_max() { #[tokio::test] async fn short_sleeps() { - for i in 0..10000 { - if (i % 10) == 0 { - eprintln!("=== {}", i); - } + for _ in 0..10000 { tokio::time::sleep(std::time::Duration::from_millis(0)).await; } } @@ -269,6 +267,20 @@ async fn exactly_max() { time::sleep(ms(MAX_DURATION)).await; } +#[tokio::test] +async fn issue_5183() { + time::pause(); + + let big = std::time::Duration::from_secs(u64::MAX / 10); + // This is a workaround since awaiting sleep(big) will never finish. + #[rustfmt::skip] + tokio::select! { + biased; + _ = tokio::time::sleep(big) => {} + _ = tokio::time::sleep(std::time::Duration::from_nanos(1)) => {} + } +} + #[tokio::test] async fn no_out_of_bounds_close_to_max() { time::pause(); @@ -317,18 +329,18 @@ async fn drop_from_wake() { tokio::time::pause(); - let mut lock = list.lock().unwrap(); + { + let mut lock = list.lock().unwrap(); - for _ in 0..100 { - let mut timer = Box::pin(tokio::time::sleep(Duration::from_millis(10))); + for _ in 0..100 { + let mut timer = Box::pin(tokio::time::sleep(Duration::from_millis(10))); - let _ = timer.as_mut().poll(&mut Context::from_waker(&arc_wake)); + let _ = timer.as_mut().poll(&mut Context::from_waker(&arc_wake)); - lock.push(timer); + lock.push(timer); + } } - drop(lock); - tokio::time::sleep(Duration::from_millis(11)).await; assert!( diff --git a/third_party/rust/tokio/tests/time_timeout.rs b/third_party/rust/tokio/tests/time_timeout.rs index a1ff51e7d270..be6b8fb44627 100644 --- a/third_party/rust/tokio/tests/time_timeout.rs +++ b/third_party/rust/tokio/tests/time_timeout.rs @@ -17,6 +17,7 @@ async fn simultaneous_deadline_future_completion() { assert_ready_ok!(fut.poll()); } +#[cfg_attr(tokio_wasi, ignore = "FIXME: `fut.poll()` panics on Wasi")] #[tokio::test] async fn completed_future_past_deadline() { // Wrap it with a deadline diff --git a/third_party/rust/tokio/tests/udp.rs b/third_party/rust/tokio/tests/udp.rs index ec2a1e961042..565323b37423 100644 --- a/third_party/rust/tokio/tests/udp.rs +++ b/third_party/rust/tokio/tests/udp.rs @@ -1,5 +1,5 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind or UDP use futures::future::poll_fn; use std::io; @@ -39,7 +39,7 @@ async fn send_recv_poll() -> std::io::Result<()> { let mut recv_buf = [0u8; 32]; let mut read = ReadBuf::new(&mut recv_buf); - let _len = poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; + poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; assert_eq!(read.filled(), MSG); Ok(()) @@ -106,6 +106,45 @@ async fn send_to_peek_from() -> std::io::Result<()> { Ok(()) } +#[tokio::test] +async fn send_to_try_peek_from() -> std::io::Result<()> { + let sender = UdpSocket::bind("127.0.0.1:0").await?; + let receiver = UdpSocket::bind("127.0.0.1:0").await?; + + let receiver_addr = receiver.local_addr()?; + poll_fn(|cx| sender.poll_send_to(cx, MSG, receiver_addr)).await?; + + // peek + let mut recv_buf = [0u8; 32]; + + loop { + match receiver.try_peek_from(&mut recv_buf) { + Ok((n, addr)) => { + assert_eq!(&recv_buf[..n], MSG); + assert_eq!(addr, sender.local_addr()?); + break; + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + receiver.readable().await?; + } + Err(e) => return Err(e), + } + } + + // peek + let mut recv_buf = [0u8; 32]; + let (n, addr) = receiver.peek_from(&mut recv_buf).await?; + assert_eq!(&recv_buf[..n], MSG); + assert_eq!(addr, sender.local_addr()?); + + let mut recv_buf = [0u8; 32]; + let (n, addr) = receiver.recv_from(&mut recv_buf).await?; + assert_eq!(&recv_buf[..n], MSG); + assert_eq!(addr, sender.local_addr()?); + + Ok(()) +} + #[tokio::test] async fn send_to_peek_from_poll() -> std::io::Result<()> { let sender = UdpSocket::bind("127.0.0.1:0").await?; @@ -134,6 +173,92 @@ async fn send_to_peek_from_poll() -> std::io::Result<()> { Ok(()) } +#[tokio::test] +async fn peek_sender() -> std::io::Result<()> { + let sender = UdpSocket::bind("127.0.0.1:0").await?; + let receiver = UdpSocket::bind("127.0.0.1:0").await?; + + let sender_addr = sender.local_addr()?; + let receiver_addr = receiver.local_addr()?; + + let msg = b"Hello, world!"; + sender.send_to(msg, receiver_addr).await?; + + let peeked_sender = receiver.peek_sender().await?; + assert_eq!(peeked_sender, sender_addr); + + // Assert that `peek_sender()` returns the right sender but + // doesn't remove from the receive queue. + let mut recv_buf = [0u8; 32]; + let (read, received_sender) = receiver.recv_from(&mut recv_buf).await?; + + assert_eq!(&recv_buf[..read], msg); + assert_eq!(received_sender, peeked_sender); + + Ok(()) +} + +#[tokio::test] +async fn poll_peek_sender() -> std::io::Result<()> { + let sender = UdpSocket::bind("127.0.0.1:0").await?; + let receiver = UdpSocket::bind("127.0.0.1:0").await?; + + let sender_addr = sender.local_addr()?; + let receiver_addr = receiver.local_addr()?; + + let msg = b"Hello, world!"; + poll_fn(|cx| sender.poll_send_to(cx, msg, receiver_addr)).await?; + + let peeked_sender = poll_fn(|cx| receiver.poll_peek_sender(cx)).await?; + assert_eq!(peeked_sender, sender_addr); + + // Assert that `poll_peek_sender()` returns the right sender but + // doesn't remove from the receive queue. + let mut recv_buf = [0u8; 32]; + let mut read = ReadBuf::new(&mut recv_buf); + let received_sender = poll_fn(|cx| receiver.poll_recv_from(cx, &mut read)).await?; + + assert_eq!(read.filled(), msg); + assert_eq!(received_sender, peeked_sender); + + Ok(()) +} + +#[tokio::test] +async fn try_peek_sender() -> std::io::Result<()> { + let sender = UdpSocket::bind("127.0.0.1:0").await?; + let receiver = UdpSocket::bind("127.0.0.1:0").await?; + + let sender_addr = sender.local_addr()?; + let receiver_addr = receiver.local_addr()?; + + let msg = b"Hello, world!"; + sender.send_to(msg, receiver_addr).await?; + + let peeked_sender = loop { + match receiver.try_peek_sender() { + Ok(peeked_sender) => break peeked_sender, + Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + receiver.readable().await?; + } + Err(e) => return Err(e), + } + }; + + assert_eq!(peeked_sender, sender_addr); + + // Assert that `try_peek_sender()` returns the right sender but + // didn't remove from the receive queue. + let mut recv_buf = [0u8; 32]; + // We already peeked the sender so there must be data in the receive queue. + let (read, received_sender) = receiver.try_recv_from(&mut recv_buf).unwrap(); + + assert_eq!(&recv_buf[..read], msg); + assert_eq!(received_sender, peeked_sender); + + Ok(()) +} + #[tokio::test] async fn split() -> std::io::Result<()> { let socket = UdpSocket::bind("127.0.0.1:0").await?; @@ -399,6 +524,23 @@ async fn try_recv_buf() { } } +#[tokio::test] +async fn recv_buf() -> std::io::Result<()> { + let sender = UdpSocket::bind("127.0.0.1:0").await?; + let receiver = UdpSocket::bind("127.0.0.1:0").await?; + + sender.connect(receiver.local_addr()?).await?; + receiver.connect(sender.local_addr()?).await?; + + sender.send(MSG).await?; + let mut recv_buf = Vec::with_capacity(32); + let len = receiver.recv_buf(&mut recv_buf).await?; + + assert_eq!(len, MSG_LEN); + assert_eq!(&recv_buf[..len], MSG); + Ok(()) +} + #[tokio::test] async fn try_recv_buf_from() { // Create listener @@ -442,6 +584,23 @@ async fn try_recv_buf_from() { } } +#[tokio::test] +async fn recv_buf_from() -> std::io::Result<()> { + let sender = UdpSocket::bind("127.0.0.1:0").await?; + let receiver = UdpSocket::bind("127.0.0.1:0").await?; + + sender.connect(receiver.local_addr()?).await?; + + sender.send(MSG).await?; + let mut recv_buf = Vec::with_capacity(32); + let (len, caddr) = receiver.recv_buf_from(&mut recv_buf).await?; + + assert_eq!(len, MSG_LEN); + assert_eq!(&recv_buf[..len], MSG); + assert_eq!(caddr, sender.local_addr()?); + Ok(()) +} + #[tokio::test] async fn poll_ready() { // Create listener diff --git a/third_party/rust/tokio/tests/uds_datagram.rs b/third_party/rust/tokio/tests/uds_datagram.rs index 5e5486ba3320..ad22a0b99dd0 100644 --- a/third_party/rust/tokio/tests/uds_datagram.rs +++ b/third_party/rust/tokio/tests/uds_datagram.rs @@ -29,9 +29,7 @@ async fn echo() -> io::Result<()> { let server_socket = UnixDatagram::bind(server_path.clone())?; tokio::spawn(async move { - if let Err(e) = echo_server(server_socket).await { - eprintln!("Error in echo server: {}", e); - } + let _ = echo_server(server_socket).await; }); { @@ -55,9 +53,7 @@ async fn echo_from() -> io::Result<()> { let server_socket = UnixDatagram::bind(server_path.clone())?; tokio::spawn(async move { - if let Err(e) = echo_server(server_socket).await { - eprintln!("Error in echo server: {}", e); - } + let _ = echo_server(server_socket).await; }); { @@ -181,7 +177,7 @@ async fn send_recv_poll() -> std::io::Result<()> { let mut recv_buf = [0u8; 32]; let mut read = ReadBuf::new(&mut recv_buf); - let _len = poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; + poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?; assert_eq!(read.filled(), msg); Ok(()) @@ -281,6 +277,28 @@ async fn try_recv_buf_from() -> std::io::Result<()> { Ok(()) } +#[tokio::test] +async fn recv_buf_from() -> std::io::Result<()> { + let tmp = tempfile::tempdir()?; + + // Bind each socket to a filesystem path + let tx_path = tmp.path().join("tx"); + let tx = UnixDatagram::bind(&tx_path)?; + let rx_path = tmp.path().join("rx"); + let rx = UnixDatagram::bind(&rx_path)?; + + let bytes = b"hello world"; + tx.send_to(bytes, &rx_path).await?; + + let mut buf = Vec::with_capacity(24); + let (size, addr) = rx.recv_buf_from(&mut buf).await?; + + let dgram = &buf[..size]; + assert_eq!(dgram, bytes); + assert_eq!(addr.as_pathname().unwrap(), &tx_path); + Ok(()) +} + // Even though we use sync non-blocking io we still need a reactor. #[tokio::test] async fn try_recv_buf_never_block() -> io::Result<()> { @@ -329,6 +347,24 @@ async fn try_recv_buf_never_block() -> io::Result<()> { Ok(()) } +#[tokio::test] +async fn recv_buf() -> std::io::Result<()> { + // Create the pair of sockets + let (sock1, sock2) = UnixDatagram::pair()?; + + // Since the sockets are paired, the paired send/recv + // functions can be used + let bytes = b"hello world"; + sock1.send(bytes).await?; + + let mut buff = Vec::with_capacity(24); + let size = sock2.recv_buf(&mut buff).await?; + + let dgram = &buff[..size]; + assert_eq!(dgram, bytes); + Ok(()) +} + #[tokio::test] async fn poll_ready() -> io::Result<()> { let dir = tempfile::tempdir().unwrap(); diff --git a/third_party/rust/tokio/tests/uds_stream.rs b/third_party/rust/tokio/tests/uds_stream.rs index 5f1b4cffbcc2..b8c4e6a8eed4 100644 --- a/third_party/rust/tokio/tests/uds_stream.rs +++ b/third_party/rust/tokio/tests/uds_stream.rs @@ -25,13 +25,13 @@ async fn accept_read_write() -> std::io::Result<()> { let connect = UnixStream::connect(&sock_path); let ((mut server, _), mut client) = try_join(accept, connect).await?; - // Write to the client. TODO: Switch to write_all. - let write_len = client.write(b"hello").await?; - assert_eq!(write_len, 5); + // Write to the client. + client.write_all(b"hello").await?; drop(client); - // Read from the server. TODO: Switch to read_to_end. - let mut buf = [0u8; 5]; - server.read_exact(&mut buf).await?; + + // Read from the server. + let mut buf = vec![]; + server.read_to_end(&mut buf).await?; assert_eq!(&buf, b"hello"); let len = server.read(&mut buf).await?; assert_eq!(len, 0); diff --git a/third_party/rust/tokio/tests/unwindsafe.rs b/third_party/rust/tokio/tests/unwindsafe.rs index 09fe83945651..3e63820864d8 100644 --- a/third_party/rust/tokio/tests/unwindsafe.rs +++ b/third_party/rust/tokio/tests/unwindsafe.rs @@ -1,8 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery use std::panic::{RefUnwindSafe, UnwindSafe}; +#[test] +fn notify_is_unwind_safe() { + is_unwind_safe::(); +} + #[test] fn join_handle_is_unwind_safe() { is_unwind_safe::>();