From ea48ee988efa24298145fc23996ac394c0feecde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 7 Aug 2024 16:34:23 +0200 Subject: [PATCH 001/193] Upgrade XCode 15.4.0 (#14794) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2999c861fbcb..9118ce51ec2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,7 @@ jobs: test_darwin: macos: - xcode: 13.4.1 + xcode: 15.4.0 environment: <<: *env TRAVIS_OS_NAME: osx From bddb53fb5177c55706ef88cd3e045bebec92af40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 7 Aug 2024 16:35:13 +0200 Subject: [PATCH 002/193] Refactor cancellation of `IOCP::OverlappedOperation` (#14754) When an overlapped operation gets cancelled, we still need to wait for completion of the operation (with status `ERROR_OPERATION_ABORTED`) before it can be freed. Previously we stored a reference to cancelled operations in a linked list and removed them when complete. This allows continuing executing the fiber directly after the cancellation is triggered, but it's also quite a bit of overhead. Also it makes it impossible to allocate operations on the stack. Cancellation is triggered when an operation times out. The change in this patch is that after a timeout the fiber is suspended again, expecting completion via the event loop. Then the operation can be freed. * Removes the `CANCELLED` state. It's no longer necessary, we only need to distinguish whether a fiber was woken up due to timeout or completion. A follow-up will further simplify the state handling. * Replace special timeout event and fiber scheduling logic with generic `sleep` and `suspend` * Drops `@@cancelled` linked list. * Drops the workaround from https://github.com/crystal-lang/crystal/pull/14724#issuecomment-2187401075 --- src/crystal/system/win32/iocp.cr | 88 ++++++++++++------------------ src/crystal/system/win32/socket.cr | 10 +--- 2 files changed, 39 insertions(+), 59 deletions(-) diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index ba0f11eb2af5..add5a29c2814 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -65,15 +65,11 @@ module Crystal::IOCP enum State STARTED DONE - CANCELLED end @overlapped = LibC::OVERLAPPED.new @fiber = Fiber.current @state : State = :started - property next : OverlappedOperation? - property previous : OverlappedOperation? - @@canceled = Thread::LinkedList(OverlappedOperation).new def initialize(@handle : LibC::HANDLE) end @@ -83,12 +79,9 @@ module Crystal::IOCP end def self.run(handle, &) - operation = OverlappedOperation.new(handle) - begin - yield operation - ensure - operation.done - end + operation_storage = uninitialized ReferenceStorage(OverlappedOperation) + operation = OverlappedOperation.unsafe_construct(pointerof(operation_storage), handle) + yield operation end def self.unbox(overlapped : LibC::OVERLAPPED*) @@ -103,8 +96,6 @@ module Crystal::IOCP def wait_for_result(timeout, &) wait_for_completion(timeout) - raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? - result = LibC.GetOverlappedResult(@handle, self, out bytes, 0) if result.zero? error = WinError.value @@ -118,11 +109,7 @@ module Crystal::IOCP def wait_for_wsa_result(timeout, &) wait_for_completion(timeout) - wsa_result { |error| yield error } - end - def wsa_result(&) - raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? flags = 0_u32 result = LibC.WSAGetOverlappedResult(LibC::SOCKET.new(@handle.address), self, out bytes, false, pointerof(flags)) if result.zero? @@ -136,49 +123,48 @@ module Crystal::IOCP end protected def schedule(&) - case @state - when .started? - yield @fiber - done! - when .cancelled? - @@canceled.delete(self) - else - raise Exception.new("Invalid state #{@state}") - end - end - - protected def done - case @state - when .started? - # https://learn.microsoft.com/en-us/windows/win32/api/ioapiset/nf-ioapiset-cancelioex - # > The application must not free or reuse the OVERLAPPED structure - # associated with the canceled I/O operations until they have completed - if LibC.CancelIoEx(@handle, self) != 0 - @state = :cancelled - @@canceled.push(self) # to increase lifetime - end - end + done! + yield @fiber end def done! + @fiber.cancel_timeout @state = :done end + def try_cancel : Bool + # Microsoft documentation: + # The application must not free or reuse the OVERLAPPED structure + # associated with the canceled I/O operations until they have completed + # (this does not apply to asynchronous operations that finished + # synchronously, as nothing would be queued to the IOCP) + ret = LibC.CancelIoEx(@handle, self) + if ret.zero? + case error = WinError.value + when .error_not_found? + # Operation has already completed, do nothing + return false + else + raise RuntimeError.from_os_error("CancelIOEx", os_error: error) + end + end + true + end + def wait_for_completion(timeout) if timeout - timeout_event = Crystal::IOCP::Event.new(Fiber.current) - timeout_event.add(timeout) + sleep timeout else - timeout_event = Crystal::IOCP::Event.new(Fiber.current, Time::Span::MAX) + Fiber.suspend end - # memoize event loop to make sure that we still target the same instance - # after wakeup (guaranteed by current MT model but let's be future proof) - event_loop = Crystal::EventLoop.current - event_loop.enqueue(timeout_event) - - Fiber.suspend - event_loop.dequeue(timeout_event) + unless @state.done? + if try_cancel + # Wait for cancellation to complete. We must not free the operation + # until it's completed. + Fiber.suspend + end + end end end @@ -200,13 +186,12 @@ module Crystal::IOCP raise IO::Error.from_os_error(method, error, target: target) end else - operation.done! return value end operation.wait_for_result(timeout) do |error| case error - when .error_io_incomplete? + when .error_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") when .error_handle_eof? return 0_u32 @@ -230,13 +215,12 @@ module Crystal::IOCP raise IO::Error.from_os_error(method, error, target: target) end else - operation.done! return value end operation.wait_for_wsa_result(timeout) do |error| case error - when .wsa_io_incomplete? + when .wsa_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") when .wsaeconnreset? return 0_u32 unless connreset_is_error diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 6a5d44ab5133..17e4ca875dbb 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -142,7 +142,6 @@ module Crystal::System::Socket return ::Socket::Error.from_os_error("ConnectEx", error) end else - operation.done! return nil end @@ -204,18 +203,15 @@ module Crystal::System::Socket return false end else - operation.done! return true end - unless operation.wait_for_completion(read_timeout) - raise IO::TimeoutError.new("#{method} timed out") - end - - operation.wsa_result do |error| + operation.wait_for_wsa_result(read_timeout) do |error| case error when .wsa_io_incomplete?, .wsaenotsock? return false + when .error_operation_aborted? + raise IO::TimeoutError.new("#{method} timed out") end end From 8f26137180b0f44257b88e71bab8b91ebbdffd45 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 7 Aug 2024 16:35:54 +0200 Subject: [PATCH 003/193] Stop & start the world (undocumented API) (#14729) Add `GC.stop_world` and `GC.start_world` methods to be able to stop and restart the world at will from within Crystal. - gc/boehm: delegates to `GC_stop_world_external` and `GC_start_world_external`; - gc/none: implements its own mechanism (tested on UNIX & Windows). My use case is a [perf-tools](https://github.com/crystal-lang/perf-tools) feature for [RFC 2](https://github.com/crystal-lang/rfcs/pull/2) that must stop the world to print out runtime information of each ExecutionContext with their schedulers and fibers. See https://github.com/crystal-lang/perf-tools/pull/18 --- src/crystal/system/thread.cr | 36 +++++++++ src/crystal/system/unix/pthread.cr | 75 +++++++++++++++++++ src/crystal/system/wasi/thread.cr | 15 ++++ src/crystal/system/win32/thread.cr | 28 +++++++ src/gc/boehm.cr | 13 ++++ src/gc/none.cr | 54 +++++++++++++ src/lib_c/aarch64-android/c/signal.cr | 2 + src/lib_c/aarch64-darwin/c/signal.cr | 2 + src/lib_c/aarch64-linux-gnu/c/signal.cr | 2 + src/lib_c/aarch64-linux-musl/c/signal.cr | 2 + src/lib_c/arm-linux-gnueabihf/c/signal.cr | 2 + src/lib_c/i386-linux-gnu/c/signal.cr | 2 + src/lib_c/i386-linux-musl/c/signal.cr | 2 + src/lib_c/x86_64-darwin/c/signal.cr | 2 + src/lib_c/x86_64-dragonfly/c/signal.cr | 2 + src/lib_c/x86_64-freebsd/c/signal.cr | 54 ++++++------- src/lib_c/x86_64-linux-gnu/c/signal.cr | 2 + src/lib_c/x86_64-linux-musl/c/signal.cr | 2 + src/lib_c/x86_64-netbsd/c/signal.cr | 2 + src/lib_c/x86_64-openbsd/c/signal.cr | 2 + src/lib_c/x86_64-solaris/c/signal.cr | 2 + .../c/processthreadsapi.cr | 4 + 22 files changed, 282 insertions(+), 25 deletions(-) diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index d9dc6acf17dc..431708c5cc11 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -23,6 +23,14 @@ module Crystal::System::Thread # private def stack_address : Void* # private def system_name=(String) : String + + # def self.init_suspend_resume : Nil + + # private def system_suspend : Nil + + # private def system_wait_suspended : Nil + + # private def system_resume : Nil end {% if flag?(:wasi) %} @@ -66,6 +74,14 @@ class Thread @@threads.try(&.unsafe_each { |thread| yield thread }) end + def self.lock : Nil + threads.@mutex.lock + end + + def self.unlock : Nil + threads.@mutex.unlock + end + # Creates and starts a new system thread. def initialize(@name : String? = nil, &@func : Thread ->) @system_handle = uninitialized Crystal::System::Thread::Handle @@ -168,6 +184,26 @@ class Thread # Holds the GC thread handler property gc_thread_handler : Void* = Pointer(Void).null + + def suspend : Nil + system_suspend + end + + def wait_suspended : Nil + system_wait_suspended + end + + def resume : Nil + system_resume + end + + def self.stop_world : Nil + GC.stop_world + end + + def self.start_world : Nil + GC.start_world + end end require "./thread_linked_list" diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index d38e52ee012a..b55839ff2784 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -1,5 +1,6 @@ require "c/pthread" require "c/sched" +require "../panic" module Crystal::System::Thread alias Handle = LibC::PthreadT @@ -153,6 +154,80 @@ module Crystal::System::Thread {% end %} name end + + @suspended = Atomic(Bool).new(false) + + def self.init_suspend_resume : Nil + install_sig_suspend_signal_handler + install_sig_resume_signal_handler + end + + private def self.install_sig_suspend_signal_handler + action = LibC::Sigaction.new + action.sa_flags = LibC::SA_SIGINFO + action.sa_sigaction = LibC::SigactionHandlerT.new do |_, _, _| + # notify that the thread has been interrupted + Thread.current_thread.@suspended.set(true) + + # block all signals but SIG_RESUME + mask = LibC::SigsetT.new + LibC.sigfillset(pointerof(mask)) + LibC.sigdelset(pointerof(mask), SIG_RESUME) + + # suspend the thread until it receives the SIG_RESUME signal + LibC.sigsuspend(pointerof(mask)) + end + LibC.sigemptyset(pointerof(action.@sa_mask)) + LibC.sigaction(SIG_SUSPEND, pointerof(action), nil) + end + + private def self.install_sig_resume_signal_handler + action = LibC::Sigaction.new + action.sa_flags = 0 + action.sa_sigaction = LibC::SigactionHandlerT.new do |_, _, _| + # do nothing (a handler is still required to receive the signal) + end + LibC.sigemptyset(pointerof(action.@sa_mask)) + LibC.sigaction(SIG_RESUME, pointerof(action), nil) + end + + private def system_suspend : Nil + @suspended.set(false) + + if LibC.pthread_kill(@system_handle, SIG_SUSPEND) == -1 + System.panic("pthread_kill()", Errno.value) + end + end + + private def system_wait_suspended : Nil + until @suspended.get + Thread.yield_current + end + end + + private def system_resume : Nil + if LibC.pthread_kill(@system_handle, SIG_RESUME) == -1 + System.panic("pthread_kill()", Errno.value) + end + end + + # the suspend/resume signals follow BDWGC + + private SIG_SUSPEND = + {% if flag?(:linux) %} + LibC::SIGPWR + {% elsif LibC.has_constant?(:SIGRTMIN) %} + LibC::SIGRTMIN + 6 + {% else %} + LibC::SIGXFSZ + {% end %} + + private SIG_RESUME = + {% if LibC.has_constant?(:SIGRTMIN) %} + LibC::SIGRTMIN + 5 + {% else %} + LibC::SIGXCPU + {% end %} end # In musl (alpine) the calls to unwind API segfaults diff --git a/src/crystal/system/wasi/thread.cr b/src/crystal/system/wasi/thread.cr index 6f0c0cbe8260..1e8f6957d526 100644 --- a/src/crystal/system/wasi/thread.cr +++ b/src/crystal/system/wasi/thread.cr @@ -38,4 +38,19 @@ module Crystal::System::Thread # TODO: Implement Pointer(Void).null end + + def self.init_suspend_resume : Nil + end + + private def system_suspend : Nil + raise NotImplementedError.new("Crystal::System::Thread.system_suspend") + end + + private def system_wait_suspended : Nil + raise NotImplementedError.new("Crystal::System::Thread.system_wait_suspended") + end + + private def system_resume : Nil + raise NotImplementedError.new("Crystal::System::Thread.system_resume") + end end diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index ddfe3298b20a..1a4f61a41738 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -1,5 +1,6 @@ require "c/processthreadsapi" require "c/synchapi" +require "../panic" module Crystal::System::Thread alias Handle = LibC::HANDLE @@ -87,4 +88,31 @@ module Crystal::System::Thread {% end %} name end + + def self.init_suspend_resume : Nil + end + + private def system_suspend : Nil + if LibC.SuspendThread(@system_handle) == -1 + Crystal::System.panic("SuspendThread()", WinError.value) + end + end + + private def system_wait_suspended : Nil + # context must be aligned on 16 bytes but we lack a mean to force the + # alignment on the struct, so we overallocate then realign the pointer: + local = uninitialized UInt8[sizeof(Tuple(LibC::CONTEXT, UInt8[15]))] + thread_context = Pointer(LibC::CONTEXT).new(local.to_unsafe.address &+ 15_u64 & ~15_u64) + thread_context.value.contextFlags = LibC::CONTEXT_FULL + + if LibC.GetThreadContext(@system_handle, thread_context) == -1 + Crystal::System.panic("GetThreadContext()", WinError.value) + end + end + + private def system_resume : Nil + if LibC.ResumeThread(@system_handle) == -1 + Crystal::System.panic("ResumeThread()", WinError.value) + end + end end diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 8ccc1bb7b6e8..0ce6a1366b6d 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -161,6 +161,9 @@ lib LibGC alias WarnProc = LibC::Char*, Word -> fun set_warn_proc = GC_set_warn_proc(WarnProc) $warn_proc = GC_current_warn_proc : WarnProc + + fun stop_world_external = GC_stop_world_external + fun start_world_external = GC_start_world_external end module GC @@ -470,4 +473,14 @@ module GC GC.unlock_write end {% end %} + + # :nodoc: + def self.stop_world : Nil + LibGC.stop_world_external + end + + # :nodoc: + def self.start_world : Nil + LibGC.start_world_external + end end diff --git a/src/gc/none.cr b/src/gc/none.cr index 640e6e8f927d..3943bd265ed9 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -5,6 +5,7 @@ require "crystal/tracing" module GC def self.init + Crystal::System::Thread.init_suspend_resume end # :nodoc: @@ -138,4 +139,57 @@ module GC # :nodoc: def self.push_stack(stack_top, stack_bottom) end + + # Stop and start the world. + # + # This isn't a GC-safe stop-the-world implementation (it may allocate objects + # while stopping the world), but the guarantees are enough for the purpose of + # gc_none. It could be GC-safe if Thread::LinkedList(T) became a struct, and + # Thread::Mutex either became a struct or provide low level abstraction + # methods that directly interact with syscalls (without allocating). + # + # Thread safety is guaranteed by the mutex in Thread::LinkedList: either a + # thread is starting and hasn't added itself to the list (it will block until + # it can acquire the lock), or is currently adding itself (the current thread + # will block until it can acquire the lock). + # + # In both cases there can't be a deadlock since we won't suspend another + # thread until it has successfuly added (or removed) itself to (from) the + # linked list and released the lock, and the other thread won't progress until + # it can add (or remove) itself from the list. + # + # Finally, we lock the mutex and keep it locked until we resume the world, so + # any thread waiting on the mutex will only be resumed when the world is + # resumed. + + # :nodoc: + def self.stop_world : Nil + current_thread = Thread.current + + # grab the lock (and keep it until the world is restarted) + Thread.lock + + # tell all threads to stop (async) + Thread.unsafe_each do |thread| + thread.suspend unless thread == current_thread + end + + # wait for all threads to have stopped + Thread.unsafe_each do |thread| + thread.wait_suspended unless thread == current_thread + end + end + + # :nodoc: + def self.start_world : Nil + current_thread = Thread.current + + # tell all threads to resume + Thread.unsafe_each do |thread| + thread.resume unless thread == current_thread + end + + # finally, we can release the lock + Thread.unlock + end end diff --git a/src/lib_c/aarch64-android/c/signal.cr b/src/lib_c/aarch64-android/c/signal.cr index 741c8f0efb65..27676c3f733f 100644 --- a/src/lib_c/aarch64-android/c/signal.cr +++ b/src/lib_c/aarch64-android/c/signal.cr @@ -79,6 +79,7 @@ lib LibC fun kill(__pid : PidT, __signal : Int) : Int fun pthread_sigmask(__how : Int, __new_set : SigsetT*, __old_set : SigsetT*) : Int + fun pthread_kill(__thread : PthreadT, __sig : Int) : Int fun sigaction(__signal : Int, __new_action : Sigaction*, __old_action : Sigaction*) : Int fun sigaltstack(__new_signal_stack : StackT*, __old_signal_stack : StackT*) : Int {% if ANDROID_API >= 21 %} @@ -89,5 +90,6 @@ lib LibC fun sigaddset(__set : SigsetT*, __signal : Int) : Int fun sigdelset(__set : SigsetT*, __signal : Int) : Int fun sigismember(__set : SigsetT*, __signal : Int) : Int + fun sigsuspend(__mask : SigsetT*) : Int {% end %} end diff --git a/src/lib_c/aarch64-darwin/c/signal.cr b/src/lib_c/aarch64-darwin/c/signal.cr index e58adc30289f..0034eef42834 100644 --- a/src/lib_c/aarch64-darwin/c/signal.cr +++ b/src/lib_c/aarch64-darwin/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/aarch64-linux-gnu/c/signal.cr b/src/lib_c/aarch64-linux-gnu/c/signal.cr index 1f7d82eb2145..7ff9fcda1b07 100644 --- a/src/lib_c/aarch64-linux-gnu/c/signal.cr +++ b/src/lib_c/aarch64-linux-gnu/c/signal.cr @@ -78,6 +78,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -86,4 +87,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/aarch64-linux-musl/c/signal.cr b/src/lib_c/aarch64-linux-musl/c/signal.cr index 5bfa187b14ec..c65fbb0ff653 100644 --- a/src/lib_c/aarch64-linux-musl/c/signal.cr +++ b/src/lib_c/aarch64-linux-musl/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/arm-linux-gnueabihf/c/signal.cr b/src/lib_c/arm-linux-gnueabihf/c/signal.cr index d94d657e1ca8..0113c045341c 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/signal.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/i386-linux-gnu/c/signal.cr b/src/lib_c/i386-linux-gnu/c/signal.cr index 11aab8bfe6bb..1a5260073c2d 100644 --- a/src/lib_c/i386-linux-gnu/c/signal.cr +++ b/src/lib_c/i386-linux-gnu/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/i386-linux-musl/c/signal.cr b/src/lib_c/i386-linux-musl/c/signal.cr index f2e554942b69..ac374b684c76 100644 --- a/src/lib_c/i386-linux-musl/c/signal.cr +++ b/src/lib_c/i386-linux-musl/c/signal.cr @@ -76,6 +76,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -84,4 +85,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-darwin/c/signal.cr b/src/lib_c/x86_64-darwin/c/signal.cr index e58adc30289f..0034eef42834 100644 --- a/src/lib_c/x86_64-darwin/c/signal.cr +++ b/src/lib_c/x86_64-darwin/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-dragonfly/c/signal.cr b/src/lib_c/x86_64-dragonfly/c/signal.cr index 1751eeed3176..e362ef1fa218 100644 --- a/src/lib_c/x86_64-dragonfly/c/signal.cr +++ b/src/lib_c/x86_64-dragonfly/c/signal.cr @@ -90,6 +90,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -98,4 +99,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-freebsd/c/signal.cr b/src/lib_c/x86_64-freebsd/c/signal.cr index fd8d07cfd4cc..c79d0630511b 100644 --- a/src/lib_c/x86_64-freebsd/c/signal.cr +++ b/src/lib_c/x86_64-freebsd/c/signal.cr @@ -8,31 +8,33 @@ lib LibC SIGILL = 4 SIGTRAP = 5 SIGIOT = LibC::SIGABRT - SIGABRT = 6 - SIGFPE = 8 - SIGKILL = 9 - SIGBUS = 10 - SIGSEGV = 11 - SIGSYS = 12 - SIGPIPE = 13 - SIGALRM = 14 - SIGTERM = 15 - SIGURG = 16 - SIGSTOP = 17 - SIGTSTP = 18 - SIGCONT = 19 - SIGCHLD = 20 - SIGTTIN = 21 - SIGTTOU = 22 - SIGIO = 23 - SIGXCPU = 24 - SIGXFSZ = 25 - SIGVTALRM = 26 - SIGUSR1 = 30 - SIGUSR2 = 31 - SIGEMT = 7 - SIGINFO = 29 - SIGWINCH = 28 + SIGABRT = 6 + SIGFPE = 8 + SIGKILL = 9 + SIGBUS = 10 + SIGSEGV = 11 + SIGSYS = 12 + SIGPIPE = 13 + SIGALRM = 14 + SIGTERM = 15 + SIGURG = 16 + SIGSTOP = 17 + SIGTSTP = 18 + SIGCONT = 19 + SIGCHLD = 20 + SIGTTIN = 21 + SIGTTOU = 22 + SIGIO = 23 + SIGXCPU = 24 + SIGXFSZ = 25 + SIGVTALRM = 26 + SIGUSR1 = 30 + SIGUSR2 = 31 + SIGEMT = 7 + SIGINFO = 29 + SIGWINCH = 28 + SIGRTMIN = 65 + SIGRTMAX = 126 SIGSTKSZ = 2048 + 32768 # MINSIGSTKSZ + 32768 SIG_SETMASK = 3 @@ -85,6 +87,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -93,4 +96,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-linux-gnu/c/signal.cr b/src/lib_c/x86_64-linux-gnu/c/signal.cr index 07d8e0fe1ae6..b5ed2f8c8fb3 100644 --- a/src/lib_c/x86_64-linux-gnu/c/signal.cr +++ b/src/lib_c/x86_64-linux-gnu/c/signal.cr @@ -78,6 +78,7 @@ lib LibC fun kill(pid : PidT, sig : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(sig : Int, handler : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -86,4 +87,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-linux-musl/c/signal.cr b/src/lib_c/x86_64-linux-musl/c/signal.cr index bba7e0c7c21a..42c2aead3e0f 100644 --- a/src/lib_c/x86_64-linux-musl/c/signal.cr +++ b/src/lib_c/x86_64-linux-musl/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-netbsd/c/signal.cr b/src/lib_c/x86_64-netbsd/c/signal.cr index 93d42e38b093..0b21c5c3f839 100644 --- a/src/lib_c/x86_64-netbsd/c/signal.cr +++ b/src/lib_c/x86_64-netbsd/c/signal.cr @@ -77,6 +77,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction = __sigaction14(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack = __sigaltstack14(x0 : StackT*, x1 : StackT*) : Int @@ -85,4 +86,5 @@ lib LibC fun sigaddset = __sigaddset14(SigsetT*, Int) : Int fun sigdelset = __sigdelset14(SigsetT*, Int) : Int fun sigismember = __sigismember14(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-openbsd/c/signal.cr b/src/lib_c/x86_64-openbsd/c/signal.cr index 04aa27000219..1c9b86137e4a 100644 --- a/src/lib_c/x86_64-openbsd/c/signal.cr +++ b/src/lib_c/x86_64-openbsd/c/signal.cr @@ -76,6 +76,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -84,4 +85,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-solaris/c/signal.cr b/src/lib_c/x86_64-solaris/c/signal.cr index 9bde30946054..ee502aa621e4 100644 --- a/src/lib_c/x86_64-solaris/c/signal.cr +++ b/src/lib_c/x86_64-solaris/c/signal.cr @@ -90,6 +90,7 @@ lib LibC fun kill(x0 : PidT, x1 : Int) : Int fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun pthread_kill(PthreadT, Int) : Int fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void fun sigaction(x0 : Int, x1 : Sigaction*, x2 : Sigaction*) : Int fun sigaltstack(x0 : StackT*, x1 : StackT*) : Int @@ -98,4 +99,5 @@ lib LibC fun sigaddset(SigsetT*, Int) : Int fun sigdelset(SigsetT*, Int) : Int fun sigismember(SigsetT*, Int) : Int + fun sigsuspend(SigsetT*) : Int end diff --git a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr index d1e13eced324..1fcaee65a01c 100644 --- a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr @@ -59,5 +59,9 @@ lib LibC fun SwitchToThread : BOOL fun QueueUserAPC(pfnAPC : PAPCFUNC, hThread : HANDLE, dwData : ULONG_PTR) : DWORD + fun GetThreadContext(hThread : HANDLE, lpContext : CONTEXT*) : DWORD + fun ResumeThread(hThread : HANDLE) : DWORD + fun SuspendThread(hThread : HANDLE) : DWORD + PROCESS_QUERY_INFORMATION = 0x0400 end From ac2ecb058a962878665bd48a2d197ea9008df8ad Mon Sep 17 00:00:00 2001 From: "WukongRework.exe BROKE" Date: Wed, 7 Aug 2024 17:14:15 -0400 Subject: [PATCH 004/193] Expose LLVM instruction builder for `neg` and `fneg` (#14774) --- src/llvm/builder.cr | 10 ++++++---- src/llvm/lib_llvm/core.cr | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/llvm/builder.cr b/src/llvm/builder.cr index 741f9ee8eb5c..3f2060b32084 100644 --- a/src/llvm/builder.cr +++ b/src/llvm/builder.cr @@ -239,11 +239,13 @@ class LLVM::Builder end {% end %} - def not(value, name = "") - # check_value(value) + {% for name in %w(not neg fneg) %} + def {{name.id}}(value, name = "") + # check_value(value) - Value.new LibLLVM.build_not(self, value, name) - end + Value.new LibLLVM.build_{{name.id}}(self, value, name) + end + {% end %} def unreachable Value.new LibLLVM.build_unreachable(self) diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index de6f04010cfa..ff3327a3f78d 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -239,6 +239,8 @@ lib LibLLVM fun build_or = LLVMBuildOr(BuilderRef, lhs : ValueRef, rhs : ValueRef, name : Char*) : ValueRef fun build_xor = LLVMBuildXor(BuilderRef, lhs : ValueRef, rhs : ValueRef, name : Char*) : ValueRef fun build_not = LLVMBuildNot(BuilderRef, value : ValueRef, name : Char*) : ValueRef + fun build_neg = LLVMBuildNeg(BuilderRef, value : ValueRef, name : Char*) : ValueRef + fun build_fneg = LLVMBuildFNeg(BuilderRef, value : ValueRef, name : Char*) : ValueRef fun build_malloc = LLVMBuildMalloc(BuilderRef, ty : TypeRef, name : Char*) : ValueRef fun build_array_malloc = LLVMBuildArrayMalloc(BuilderRef, ty : TypeRef, val : ValueRef, name : Char*) : ValueRef From d738ac9796a24b31782c972c82e7f694b0579d96 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 7 Aug 2024 23:15:10 +0200 Subject: [PATCH 005/193] Add support for negative start index in `Slice#[start, count]` (#14778) Allows `slice[-3, 3]` to return the last 3 elements of the slice, for example, similar to how Array#[-start, count] behaves, with the difference that Slice returns exactly *count* elements, while Array returns up to *count* elements. Introduces a couple changes: - negative start now returns from the end of the slice instead of returning nil/raising IndexError - negative count now raises ArgumentError instead of returning nil/raising IndexError I believe the current behavior is buggy (unexpected result, underspecified documentation), but this can be considered a breaking change. --- spec/std/slice_spec.cr | 23 +++++++++++++++++++---- src/slice.cr | 30 ++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/spec/std/slice_spec.cr b/spec/std/slice_spec.cr index 505db8f09109..1b21a4489bbd 100644 --- a/spec/std/slice_spec.cr +++ b/spec/std/slice_spec.cr @@ -104,25 +104,34 @@ describe "Slice" do it "does []? with start and count" do slice = Slice.new(4) { |i| i + 1 } + slice1 = slice[1, 2]? slice1.should_not be_nil slice1 = slice1.not_nil! slice1.size.should eq(2) + slice1.to_unsafe.should eq(slice.to_unsafe + 1) slice1[0].should eq(2) slice1[1].should eq(3) - slice[-1, 1]?.should be_nil + slice2 = slice[-1, 1]? + slice2.should_not be_nil + slice2 = slice2.not_nil! + slice2.size.should eq(1) + slice2.to_unsafe.should eq(slice.to_unsafe + 3) + slice[3, 2]?.should be_nil slice[0, 5]?.should be_nil - slice[3, -1]?.should be_nil + expect_raises(ArgumentError, "Negative count: -1") { slice[3, -1]? } end it "does []? with range" do slice = Slice.new(4) { |i| i + 1 } + slice1 = slice[1..2]? slice1.should_not be_nil slice1 = slice1.not_nil! slice1.size.should eq(2) + slice1.to_unsafe.should eq(slice.to_unsafe + 1) slice1[0].should eq(2) slice1[1].should eq(3) @@ -134,15 +143,20 @@ describe "Slice" do it "does [] with start and count" do slice = Slice.new(4) { |i| i + 1 } + slice1 = slice[1, 2] slice1.size.should eq(2) + slice1.to_unsafe.should eq(slice.to_unsafe + 1) slice1[0].should eq(2) slice1[1].should eq(3) - expect_raises(IndexError) { slice[-1, 1] } + slice2 = slice[-1, 1] + slice2.size.should eq(1) + slice2.to_unsafe.should eq(slice.to_unsafe + 3) + expect_raises(IndexError) { slice[3, 2] } expect_raises(IndexError) { slice[0, 5] } - expect_raises(IndexError) { slice[3, -1] } + expect_raises(ArgumentError, "Negative count: -1") { slice[3, -1] } end it "does empty?" do @@ -659,6 +673,7 @@ describe "Slice" do subslice = slice[2..4] subslice.read_only?.should be_false subslice.size.should eq(3) + subslice.to_unsafe.should eq(slice.to_unsafe + 2) subslice.should eq(Slice.new(3) { |i| i + 3 }) end diff --git a/src/slice.cr b/src/slice.cr index 196a29a768dd..c03544ffd859 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -222,35 +222,49 @@ struct Slice(T) end # Returns a new slice that starts at *start* elements from this slice's start, - # and of *count* size. + # and of exactly *count* size. # + # Negative *start* is added to `#size`, thus it's treated as index counting + # from the end of the array, `-1` designating the last element. + # + # Raises `ArgumentError` if *count* is negative. # Returns `nil` if the new slice falls outside this slice. # # ``` # slice = Slice.new(5) { |i| i + 10 } # slice # => Slice[10, 11, 12, 13, 14] # - # slice[1, 3]? # => Slice[11, 12, 13] - # slice[1, 33]? # => nil + # slice[1, 3]? # => Slice[11, 12, 13] + # slice[1, 33]? # => nil + # slice[-3, 2]? # => Slice[12, 13] + # slice[-3, 10]? # => nil # ``` def []?(start : Int, count : Int) : Slice(T)? - return unless 0 <= start <= @size - return unless 0 <= count <= @size - start + # we skip the calculated count because the subslice must contain exactly + # *count* elements + start, _ = Indexable.normalize_start_and_count(start, count, size) { return } + return unless count <= @size - start Slice.new(@pointer + start, count, read_only: @read_only) end # Returns a new slice that starts at *start* elements from this slice's start, - # and of *count* size. + # and of exactly *count* size. + # + # Negative *start* is added to `#size`, thus it's treated as index counting + # from the end of the array, `-1` designating the last element. # + # Raises `ArgumentError` if *count* is negative. # Raises `IndexError` if the new slice falls outside this slice. # # ``` # slice = Slice.new(5) { |i| i + 10 } # slice # => Slice[10, 11, 12, 13, 14] # - # slice[1, 3] # => Slice[11, 12, 13] - # slice[1, 33] # raises IndexError + # slice[1, 3] # => Slice[11, 12, 13] + # slice[1, 33] # raises IndexError + # slice[-3, 2] # => Slice[12, 13] + # slice[-3, 10] # raises IndexError # ``` def [](start : Int, count : Int) : Slice(T) self[start, count]? || raise IndexError.new From edce0a3627d9f5b707e6d4d1ca8c5dc7c3ead362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 7 Aug 2024 23:15:28 +0200 Subject: [PATCH 006/193] Enable full backtrace for exception in process spawn (#14796) If an exception raises in the code that prepares a forked process for `exec`, the error message of this exception is written through a pipe to the original process, which then raises an exception for the calling code (`Process.run`). The error only includes a message, no stack trace. So the only stack trace you get is that of the handler which reads the message from the pipe and raises in the original process. But that's not very relevant. We want to know the location of the original exception. This patch changes from sending just the exception message, to printing the entire backtrace (`inspect_with_backtrace`). --- src/crystal/system/unix/process.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 83f95cc8648c..4a540fa53a3d 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -243,8 +243,9 @@ struct Crystal::System::Process writer_pipe.write_bytes(Errno.value.to_i) rescue ex writer_pipe.write_byte(0) - writer_pipe.write_bytes(ex.message.try(&.bytesize) || 0) - writer_pipe << ex.message + message = ex.inspect_with_backtrace + writer_pipe.write_bytes(message.bytesize) + writer_pipe << message writer_pipe.close ensure LibC._exit 127 From 2a8ced5e57d555089c7e47a000fb4071058391f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 8 Aug 2024 10:31:21 +0200 Subject: [PATCH 007/193] Refactor GitHub changelog generator print special infra (#14795) --- scripts/github-changelog.cr | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index f7ae12e74dad..2f89bd923153 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -367,32 +367,37 @@ puts puts "[#{milestone.title}]: https://github.com/#{repository}/releases/#{milestone.title}" puts +def print_items(prs) + prs.each do |pr| + puts "- #{pr}" + end + puts + + prs.each(&.print_ref_label(STDOUT)) + puts +end + SECTION_TITLES.each do |id, title| prs = sections[id]? || next puts "### #{title}" puts - topics = prs.group_by(&.primary_topic) + if id == "infra" + prs.sort_by!(&.infra_sort_tuple) + print_items prs + else + topics = prs.group_by(&.primary_topic) - topic_titles = topics.keys.sort_by! { |k| TOPIC_ORDER.index(k) || Int32::MAX } + topic_titles = topics.keys.sort_by! { |k| TOPIC_ORDER.index(k) || Int32::MAX } - topic_titles.each do |topic_title| - topic_prs = topics[topic_title]? || next + topic_titles.each do |topic_title| + topic_prs = topics[topic_title]? || next - if id == "infra" - topic_prs.sort_by!(&.infra_sort_tuple) - else - topic_prs.sort! puts "#### #{topic_title}" puts - end - topic_prs.each do |pr| - puts "- #{pr}" + topic_prs.sort! + print_items topic_prs end - puts - - topic_prs.each(&.print_ref_label(STDOUT)) - puts end end From c0cdaa2f93fdc7e00687c722ebbd5b52d119ef0b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 8 Aug 2024 16:31:34 +0800 Subject: [PATCH 008/193] Use `Time::Span` in `Benchmark.ips` (#14805) --- src/benchmark.cr | 16 ++++++++++++++-- src/benchmark/ips.cr | 9 ++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/benchmark.cr b/src/benchmark.cr index bd77a93ae026..a0f4933ddf2a 100644 --- a/src/benchmark.cr +++ b/src/benchmark.cr @@ -102,10 +102,10 @@ module Benchmark # to which one can report the benchmarks. See the module's description. # # The optional parameters *calculation* and *warmup* set the duration of - # those stages in seconds. For more detail on these stages see + # those stages. For more detail on these stages see # `Benchmark::IPS`. When the *interactive* parameter is `true`, results are # displayed and updated as they are calculated, otherwise all at once after they finished. - def ips(calculation = 5, warmup = 2, interactive = STDOUT.tty?, &) + def ips(calculation : Time::Span = 5.seconds, warmup : Time::Span = 2.seconds, interactive : Bool = STDOUT.tty?, &) {% if !flag?(:release) %} puts "Warning: benchmarking without the `--release` flag won't yield useful results" {% end %} @@ -117,6 +117,18 @@ module Benchmark job end + # Instruction per second interface of the `Benchmark` module. Yields a `Job` + # to which one can report the benchmarks. See the module's description. + # + # The optional parameters *calculation* and *warmup* set the duration of + # those stages in seconds. For more detail on these stages see + # `Benchmark::IPS`. When the *interactive* parameter is `true`, results are + # displayed and updated as they are calculated, otherwise all at once after they finished. + @[Deprecated("Use `#ips(Time::Span, Time::Span, Bool, &)` instead.")] + def ips(calculation = 5, warmup = 2, interactive = STDOUT.tty?, &) + ips(calculation.seconds, warmup.seconds, !!interactive) { |job| yield job } + end + # Returns the time used to execute the given block. def measure(label = "", &) : BM::Tms t0, r0 = Process.times, Time.monotonic diff --git a/src/benchmark/ips.cr b/src/benchmark/ips.cr index cb952325eca0..def5b09c7c66 100644 --- a/src/benchmark/ips.cr +++ b/src/benchmark/ips.cr @@ -20,13 +20,16 @@ module Benchmark @warmup_time : Time::Span @calculation_time : Time::Span - def initialize(calculation = 5, warmup = 2, interactive = STDOUT.tty?) + def initialize(calculation @calculation_time : Time::Span = 5.seconds, warmup @warmup_time : Time::Span = 2.seconds, interactive : Bool = STDOUT.tty?) @interactive = !!interactive - @warmup_time = warmup.seconds - @calculation_time = calculation.seconds @items = [] of Entry end + @[Deprecated("Use `.new(Time::Span, Time::Span, Bool)` instead.")] + def self.new(calculation = 5, warmup = 2, interactive = STDOUT.tty?) + new(calculation.seconds, warmup.seconds, !!interactive) + end + # Adds code to be benchmarked def report(label = "", &action) : Benchmark::IPS::Entry item = Entry.new(label, action) From 94bdba1227c1950c7ae0f3e5588b90597e832ba2 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 8 Aug 2024 04:31:53 -0400 Subject: [PATCH 009/193] Add `underscore_to_space` option to `String#titleize` (#14822) --- spec/std/string_spec.cr | 10 ++++++++++ src/string.cr | 35 ++++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 00310bfcbc47..2ea13d52010d 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -724,6 +724,10 @@ describe "String" do it { assert_prints " spáçes before".titleize, " Spáçes Before" } it { assert_prints "testá-se múitô".titleize, "Testá-se Múitô" } it { assert_prints "iO iO".titleize(Unicode::CaseOptions::Turkic), "İo İo" } + it { assert_prints "foo_Bar".titleize, "Foo_bar" } + it { assert_prints "foo_bar".titleize, "Foo_bar" } + it { assert_prints "testá_se múitô".titleize(underscore_to_space: true), "Testá Se Múitô" } + it { assert_prints "foo_bar".titleize(underscore_to_space: true), "Foo Bar" } it "handles multi-character mappings correctly (#13533)" do assert_prints "fflİ İffl dz DZ".titleize, "Ffli̇ İffl Dz Dz" @@ -735,6 +739,12 @@ describe "String" do String.build { |io| "\xB5!\xE0\xC1\xB5?".titleize(io) }.should eq("\xB5!\xE0\xC1\xB5?".scrub) String.build { |io| "a\xA0b".titleize(io) }.should eq("A\xA0b".scrub) end + + describe "with IO" do + it { String.build { |io| "foo_Bar".titleize io }.should eq "Foo_bar" } + it { String.build { |io| "foo_bar".titleize io }.should eq "Foo_bar" } + it { String.build { |io| "foo_bar".titleize(io, underscore_to_space: true) }.should eq "Foo Bar" } + end end describe "chomp" do diff --git a/src/string.cr b/src/string.cr index d3bc7d6998b2..08bbb87fc505 100644 --- a/src/string.cr +++ b/src/string.cr @@ -1506,15 +1506,17 @@ class String end end - # Returns a new `String` with the first letter after any space converted to uppercase and every - # other letter converted to lowercase. + # Returns a new `String` with the first letter after any space converted to uppercase and every other letter converted to lowercase. + # Optionally, if *underscore_to_space* is `true`, underscores (`_`) will be converted to a space and the following letter converted to uppercase. # # ``` - # "hEllO tAb\tworld".titleize # => "Hello Tab\tWorld" - # " spaces before".titleize # => " Spaces Before" - # "x-men: the last stand".titleize # => "X-men: The Last Stand" + # "hEllO tAb\tworld".titleize # => "Hello Tab\tWorld" + # " spaces before".titleize # => " Spaces Before" + # "x-men: the last stand".titleize # => "X-men: The Last Stand" + # "foo_bar".titleize # => "Foo_bar" + # "foo_bar".titleize(underscore_to_space: true) # => "Foo Bar" # ``` - def titleize(options : Unicode::CaseOptions = :none) : String + def titleize(options : Unicode::CaseOptions = :none, *, underscore_to_space : Bool = false) : String return self if empty? if single_byte_optimizable? && (options.none? || options.ascii?) @@ -1525,9 +1527,15 @@ class String byte = to_unsafe[i] if byte < 0x80 char = byte.unsafe_chr - replaced_char = upcase_next ? char.upcase : char.downcase + replaced_char, upcase_next = if upcase_next + {char.upcase, false} + elsif underscore_to_space && '_' == char + {' ', true} + else + {char.downcase, char.ascii_whitespace?} + end + buffer[i] = replaced_char.ord.to_u8! - upcase_next = char.ascii_whitespace? else buffer[i] = byte upcase_next = false @@ -1537,26 +1545,31 @@ class String end end - String.build(bytesize) { |io| titleize io, options } + String.build(bytesize) { |io| titleize io, options, underscore_to_space: underscore_to_space } end # Writes a titleized version of `self` to the given *io*. + # Optionally, if *underscore_to_space* is `true`, underscores (`_`) will be converted to a space and the following letter converted to uppercase. # # ``` # io = IO::Memory.new # "x-men: the last stand".titleize io # io.to_s # => "X-men: The Last Stand" # ``` - def titleize(io : IO, options : Unicode::CaseOptions = :none) : Nil + def titleize(io : IO, options : Unicode::CaseOptions = :none, *, underscore_to_space : Bool = false) : Nil upcase_next = true each_char_with_index do |char, i| if upcase_next + upcase_next = false char.titlecase(options) { |c| io << c } + elsif underscore_to_space && '_' == char + upcase_next = true + io << ' ' else + upcase_next = char.whitespace? char.downcase(options) { |c| io << c } end - upcase_next = char.whitespace? end end From f1eabf5768e6cec5a92acda78e2d9ce58d3c6a0c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 8 Aug 2024 21:21:11 +0800 Subject: [PATCH 010/193] Always use unstable sort for simple types (#14825) When calling `#sort!` without a block, if two elements have the same binary representations whenever they compare equal using `#<=>`, then they are indistinguishable from each other and swapping is a no-op. This allows the use of unstable sorting which runs slightly faster and requires no temporary allocations, as opposed to stable sorting which allocates memory linear in the collection size. Primitive floats do not support it, as the signed zeros compare equal but have opposite sign bits. For simplicity, unions also aren't touched; either they don't have a meaningful, non-null `#<=>` defined across the variant types, or they break the criterion (e.g. `1_i32` and `1_i8` have different type IDs). `#sort` always delegates to `#sort!`. This does not affect `#sort_by!` since the projected element type alone doesn't guarantee the original elements themselves can be swapped in the same way. --- src/slice.cr | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/slice.cr b/src/slice.cr index c03544ffd859..d843ceb17c63 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -995,13 +995,23 @@ struct Slice(T) # the result could also be `[b, a]`. # # If stability is expendable, `#unstable_sort!` provides a performance - # advantage over stable sort. + # advantage over stable sort. As an optimization, if `T` is any primitive + # integer type, `Char`, any enum type, any `Pointer` instance, `Symbol`, or + # `Time::Span`, then an unstable sort is automatically used. # # Raises `ArgumentError` if the comparison between any two elements returns `nil`. def sort! : self - Slice.merge_sort!(self) + # If two values `x, y : T` have the same binary representation whenever they + # compare equal, i.e. `x <=> y == 0` implies + # `pointerof(x).memcmp(pointerof(y), 1) == 0`, then swapping the two values + # is a no-op and therefore a stable sort isn't required + {% if T.union_types.size == 1 && (T <= Int::Primitive || T <= Char || T <= Enum || T <= Pointer || T <= Symbol || T <= Time::Span) %} + unstable_sort! + {% else %} + Slice.merge_sort!(self) - self + self + {% end %} end # Sorts all elements in `self` based on the return value of the comparison From 467103dfaad8d32da89409f23a2e0f8be62c8e84 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 15:22:37 +0200 Subject: [PATCH 011/193] Update GH Actions (#14535) --- .github/workflows/macos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 7f27b3cc9c14..c19041c3f52d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -17,12 +17,12 @@ jobs: - name: Download Crystal source uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v26 + - uses: cachix/install-nix-action@v27 with: install_url: https://releases.nixos.org/nix/nix-2.9.2/install extra_nix_config: | experimental-features = nix-command - - uses: cachix/cachix-action@v14 + - uses: cachix/cachix-action@v15 with: name: crystal-ci signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' From f16794d933605b25354fee401e210c1f2e818b26 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 8 Aug 2024 09:22:57 -0400 Subject: [PATCH 012/193] Add JSON parsing UTF-8 spec (#14823) --- spec/std/json/parser_spec.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/std/json/parser_spec.cr b/spec/std/json/parser_spec.cr index 96cfd52277a2..0147cfa92964 100644 --- a/spec/std/json/parser_spec.cr +++ b/spec/std/json/parser_spec.cr @@ -22,6 +22,7 @@ describe JSON::Parser do it_parses "true", true it_parses "false", false it_parses "null", nil + it_parses %("\\nПривет, мир!"), "\nПривет, мир!" it_parses "[]", [] of Int32 it_parses "[1]", [1] From e4390a3e66e056002bcca57d8025444f81322a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 8 Aug 2024 18:18:46 +0200 Subject: [PATCH 013/193] Update distribution-scripts (#14877) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/da59efb2dfd70dcd7272eaecceffb636ef547427 This includes the following changes: * crystal-lang/distribution-scripts#326 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9118ce51ec2c..b3f2310d7808 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "96e431e170979125018bd4fd90111a3147477eec" + default: "da59efb2dfd70dcd7272eaecceffb636ef547427" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From 74f009346c1c8dc4187a6dd69f8ec055a7526b1d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 9 Aug 2024 03:11:26 +0800 Subject: [PATCH 014/193] Add `Crystal::Macros::TypeNode#has_inner_pointers?` (#14847) This allows `Pointer.malloc` and `Reference#allocate` to be implemented without compiler primitives eventually, see #13589 and #13481. This might be helpful to diagnostic tools related to the GC too. --- spec/compiler/macro/macro_methods_spec.cr | 59 +++++++++++++++++++++++ src/compiler/crystal/macros.cr | 19 ++++++++ src/compiler/crystal/macros/methods.cr | 2 + src/primitives.cr | 8 +-- 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 29de1a51c2be..38b08f44568a 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -2407,6 +2407,65 @@ module Crystal end end end + + describe "#has_inner_pointers?" do + it "works on structs" do + assert_macro("{{x.has_inner_pointers?}}", %(false)) do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.struct) + klass.struct = true + klass.declare_instance_var("@var", program.int32) + {x: TypeNode.new(klass)} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.struct) + klass.struct = true + klass.declare_instance_var("@var", program.string) + {x: TypeNode.new(klass)} + end + end + + it "works on references" do + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + {x: TypeNode.new(klass)} + end + end + + it "works on ReferenceStorage" do + assert_macro("{{x.has_inner_pointers?}}", %(false)) do |program| + reference_storage = GenericReferenceStorageType.new program, program, "ReferenceStorage", program.struct, ["T"] + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.declare_instance_var("@var", program.int32) + {x: TypeNode.new(reference_storage.instantiate([klass] of TypeVar))} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + reference_storage = GenericReferenceStorageType.new program, program, "ReferenceStorage", program.struct, ["T"] + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.declare_instance_var("@var", program.string) + {x: TypeNode.new(reference_storage.instantiate([klass] of TypeVar))} + end + end + + it "works on primitive values" do + assert_macro("{{x.has_inner_pointers?}}", %(false)) do |program| + {x: TypeNode.new(program.int32)} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + {x: TypeNode.new(program.void)} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + {x: TypeNode.new(program.pointer_of(program.int32))} + end + + assert_macro("{{x.has_inner_pointers?}}", %(true)) do |program| + {x: TypeNode.new(program.proc_of(program.void))} + end + end + end end describe "type declaration methods" do diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index c0d4f6e0a071..ff422ce553a2 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -2853,5 +2853,24 @@ module Crystal::Macros # `self` is an ancestor of *other*. def >=(other : TypeNode) : BoolLiteral end + + # Returns whether `self` contains any inner pointers. + # + # Primitive types, except `Void`, are expected to not contain inner pointers. + # `Proc` and `Pointer` contain inner pointers. + # Unions, structs and collection types (tuples, static arrays) + # have inner pointers if any of their contained types has inner pointers. + # All other types, including classes, are expected to contain inner pointers. + # + # Types that do not have inner pointers may opt to use atomic allocations, + # i.e. `GC.malloc_atomic` rather than `GC.malloc`. The compiler ensures + # that, for any type `T`: + # + # * `Pointer(T).malloc` is atomic if and only if `T` has no inner pointers; + # * `T.allocate` is atomic if and only if `T` is a reference type and + # `ReferenceStorage(T)` has no inner pointers. + # NOTE: Like `#instance_vars` this method must be called from within a method. The result may be incorrect when used in top-level code. + def has_inner_pointers? : BoolLiteral + end end end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index a44bba1b76f9..d3a1a1cc15a6 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -2013,6 +2013,8 @@ module Crystal SymbolLiteral.new("public") end end + when "has_inner_pointers?" + interpret_check_args { BoolLiteral.new(type.has_inner_pointers?) } else super end diff --git a/src/primitives.cr b/src/primitives.cr index 9383ba642165..e033becdfbd2 100644 --- a/src/primitives.cr +++ b/src/primitives.cr @@ -206,12 +206,8 @@ struct Pointer(T) # ``` # # The implementation uses `GC.malloc` if the compiler is aware that the - # allocated type contains inner address pointers. Otherwise it uses - # `GC.malloc_atomic`. Primitive types are expected to not contain pointers, - # except `Void`. `Proc` and `Pointer` are expected to contain pointers. - # For unions, structs and collection types (tuples, static array) - # it depends on the contained types. All other types, including classes are - # expected to contain inner address pointers. + # allocated type contains inner address pointers. See + # `Crystal::Macros::TypeNode#has_inner_pointers?` for details. # # To override this implicit behaviour, `GC.malloc` and `GC.malloc_atomic` # can be used directly instead. From 4c2eaf06d05be415f8140fb61893b53fb74f6390 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 9 Aug 2024 03:13:59 +0800 Subject: [PATCH 015/193] Fix `File#truncate` and `#lock` for Win32 append-mode files (#14706) --- spec/std/file_spec.cr | 35 +++++++++++++++++++++ src/crystal/system/unix/file.cr | 3 ++ src/crystal/system/wasi/file.cr | 3 ++ src/crystal/system/win32/file.cr | 20 +++++++++--- src/crystal/system/win32/file_descriptor.cr | 13 ++++++-- src/file.cr | 2 +- 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 44f947997b34..942ae8a1143d 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1049,6 +1049,41 @@ describe "File" do end end + it "does not overwrite existing content in append mode" do + with_tempfile("append-override.txt") do |filename| + File.write(filename, "0123456789") + + File.open(filename, "a") do |file| + file.seek(5) + file.write "abcd".to_slice + end + + File.read(filename).should eq "0123456789abcd" + end + end + + it "truncates file opened in append mode (#14702)" do + with_tempfile("truncate-append.txt") do |path| + File.write(path, "0123456789") + + File.open(path, "a") do |file| + file.truncate(4) + end + + File.read(path).should eq "0123" + end + end + + it "locks file opened in append mode (#14702)" do + with_tempfile("truncate-append.txt") do |path| + File.write(path, "0123456789") + + File.open(path, "a") do |file| + file.flock_exclusive { } + end + end + end + it "can navigate with pos" do File.open(datapath("test_file.txt")) do |file| file.pos = 3 diff --git a/src/crystal/system/unix/file.cr b/src/crystal/system/unix/file.cr index a353cf29cd3c..fafd1d0d0a16 100644 --- a/src/crystal/system/unix/file.cr +++ b/src/crystal/system/unix/file.cr @@ -24,6 +24,9 @@ module Crystal::System::File {fd, fd < 0 ? Errno.value : Errno::NONE} end + protected def system_set_mode(mode : String) + end + def self.info?(path : String, follow_symlinks : Bool) : ::File::Info? stat = uninitialized LibC::Stat if follow_symlinks diff --git a/src/crystal/system/wasi/file.cr b/src/crystal/system/wasi/file.cr index 0d197550e3db..a48463eded4e 100644 --- a/src/crystal/system/wasi/file.cr +++ b/src/crystal/system/wasi/file.cr @@ -2,6 +2,9 @@ require "../unix/file" # :nodoc: module Crystal::System::File + protected def system_set_mode(mode : String) + end + def self.chmod(path, mode) raise NotImplementedError.new "Crystal::System::File.chmod" end diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 83d6afcf18ca..9039cc40a7ac 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -9,6 +9,11 @@ require "c/ntifs" require "c/winioctl" module Crystal::System::File + # On Windows we cannot rely on the system mode `FILE_APPEND_DATA` and + # keep track of append mode explicitly. When writing data, this ensures to only + # write at the end of the file. + @system_append = false + def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) : FileDescriptor::Handle perm = ::File::Permissions.new(perm) if perm.is_a? Int32 # Only the owner writable bit is used, since windows only supports @@ -52,10 +57,9 @@ module Crystal::System::File LibC::FILE_GENERIC_READ end - if flags.bits_set? LibC::O_APPEND - access |= LibC::FILE_APPEND_DATA - access &= ~LibC::FILE_WRITE_DATA - end + # do not handle `O_APPEND`, because Win32 append mode relies on removing + # `FILE_WRITE_DATA` which breaks file truncation and locking; instead, + # simply set the end of the file as the write offset in `#write_blocking` if flags.bits_set? LibC::O_TRUNC if flags.bits_set? LibC::O_CREAT @@ -96,6 +100,14 @@ module Crystal::System::File {access, disposition, attributes} end + protected def system_set_mode(mode : String) + @system_append = true if mode.starts_with?('a') + end + + private def write_blocking(handle, slice) + write_blocking(handle, slice, pos: @system_append ? UInt64::MAX : nil) + end + NOT_FOUND_ERRORS = { WinError::ERROR_FILE_NOT_FOUND, WinError::ERROR_PATH_NOT_FOUND, diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index dc8d479532be..b39f98fbdf0c 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -52,8 +52,17 @@ module Crystal::System::FileDescriptor end end - private def write_blocking(handle, slice) - ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, nil) + private def write_blocking(handle, slice, pos = nil) + overlapped = LibC::OVERLAPPED.new + if pos + overlapped.union.offset.offset = LibC::DWORD.new!(pos) + overlapped.union.offset.offsetHigh = LibC::DWORD.new!(pos >> 32) + overlapped_ptr = pointerof(overlapped) + else + overlapped_ptr = Pointer(LibC::OVERLAPPED).null + end + + ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, overlapped_ptr) if ret.zero? case error = WinError.value when .error_access_denied? diff --git a/src/file.cr b/src/file.cr index ff6c68ef4d03..202a05ab01f0 100644 --- a/src/file.cr +++ b/src/file.cr @@ -173,7 +173,7 @@ class File < IO::FileDescriptor def self.new(filename : Path | String, mode = "r", perm = DEFAULT_CREATE_PERMISSIONS, encoding = nil, invalid = nil, blocking = true) filename = filename.to_s fd = Crystal::System::File.open(filename, mode, perm: perm) - new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid) + new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid).tap { |f| f.system_set_mode(mode) } end getter path : String From b9ab9968b85ce2b99a875abd4360b1a432e27a98 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 9 Aug 2024 05:00:09 -0400 Subject: [PATCH 016/193] Hide `Hash::Entry` from public API docs (#14881) --- src/hash.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hash.cr b/src/hash.cr index cfa556f921ed..e14b92ee482e 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -2149,6 +2149,7 @@ class Hash(K, V) hash end + # :nodoc: struct Entry(K, V) getter key, value, hash From a12ab5b769e122c9583588d7b4ae5cb4ce2162a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Cla=C3=9Fen?= Date: Tue, 13 Aug 2024 11:03:39 +0200 Subject: [PATCH 017/193] Fix typos in docs for `Set` and `Hash` (#14889) --- src/hash.cr | 2 +- src/set.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hash.cr b/src/hash.cr index e14b92ee482e..e2fe7dad186c 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -1055,7 +1055,7 @@ class Hash(K, V) self end - # Returns `true` of this Hash is comparing keys by `object_id`. + # Returns `true` if this Hash is comparing keys by `object_id`. # # See `compare_by_identity`. getter? compare_by_identity : Bool diff --git a/src/set.cr b/src/set.cr index c998fab949a1..1bcc5178fbb0 100644 --- a/src/set.cr +++ b/src/set.cr @@ -73,7 +73,7 @@ struct Set(T) self end - # Returns `true` of this Set is comparing objects by `object_id`. + # Returns `true` if this Set is comparing objects by `object_id`. # # See `compare_by_identity`. def compare_by_identity? : Bool From 6d89f24f13fa988458d5b5fde983a234e4a625d1 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 13 Aug 2024 17:04:03 +0800 Subject: [PATCH 018/193] Support return types in codegen specs (#14888) --- spec/compiler/codegen/and_spec.cr | 48 ++++++++++++------------ spec/spec_helper.cr | 12 ++++++ src/compiler/crystal/codegen/codegen.cr | 50 +++++++++++++++++++++++++ src/llvm/jit_compiler.cr | 4 ++ src/llvm/lib_llvm/execution_engine.cr | 1 + 5 files changed, 91 insertions(+), 24 deletions(-) diff --git a/spec/compiler/codegen/and_spec.cr b/spec/compiler/codegen/and_spec.cr index 337cceb138eb..7aa3cdfd6c7b 100644 --- a/spec/compiler/codegen/and_spec.cr +++ b/spec/compiler/codegen/and_spec.cr @@ -2,42 +2,42 @@ require "../../spec_helper" describe "Code gen: and" do it "codegens and with bool false and false" do - run("false && false").to_b.should be_false + run("false && false", Bool).should be_false end it "codegens and with bool false and true" do - run("false && true").to_b.should be_false + run("false && true", Bool).should be_false end it "codegens and with bool true and true" do - run("true && true").to_b.should be_true + run("true && true", Bool).should be_true end it "codegens and with bool true and false" do - run("true && false").to_b.should be_false + run("true && false", Bool).should be_false end it "codegens and with bool and int 1" do - run("struct Bool; def to_i!; 0; end; end; (false && 2).to_i!").to_i.should eq(0) + run("struct Bool; def to_i!; 0; end; end; (false && 2).to_i!", Int32).should eq(0) end it "codegens and with bool and int 2" do - run("struct Bool; def to_i!; 0; end; end; (true && 2).to_i!").to_i.should eq(2) + run("struct Bool; def to_i!; 0; end; end; (true && 2).to_i!", Int32).should eq(2) end it "codegens and with primitive type other than bool" do - run("1 && 2").to_i.should eq(2) + run("1 && 2", Int32).should eq(2) end it "codegens and with primitive type other than bool with union" do - run("(1 && 1.5).to_f").to_f64.should eq(1.5) + run("(1 && 1.5).to_f", Float64).should eq(1.5) end it "codegens and with primitive type other than bool" do run(%( struct Nil; def to_i!; 0; end; end (nil && 2).to_i! - )).to_i.should eq(0) + ), Int32).should eq(0) end it "codegens and with nilable as left node 1" do @@ -47,7 +47,7 @@ describe "Code gen: and" do a = Reference.new a = nil (a && 2).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens and with nilable as left node 2" do @@ -56,7 +56,7 @@ describe "Code gen: and" do a = nil a = Reference.new (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with non-false union as left node" do @@ -64,7 +64,7 @@ describe "Code gen: and" do a = 1.5 a = 1 (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with nil union as left node 1" do @@ -73,7 +73,7 @@ describe "Code gen: and" do a = nil a = 1 (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with nil union as left node 2" do @@ -82,7 +82,7 @@ describe "Code gen: and" do a = 1 a = nil (a && 2).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens and with bool union as left node 1" do @@ -91,7 +91,7 @@ describe "Code gen: and" do a = false a = 1 (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with bool union as left node 2" do @@ -100,7 +100,7 @@ describe "Code gen: and" do a = 1 a = false (a && 2).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens and with bool union as left node 3" do @@ -109,7 +109,7 @@ describe "Code gen: and" do a = 1 a = true (a && 2).to_i! - ").to_i.should eq(2) + ", Int32).should eq(2) end it "codegens and with bool union as left node 1" do @@ -120,7 +120,7 @@ describe "Code gen: and" do a = nil a = 2 (a && 3).to_i! - ").to_i.should eq(3) + ", Int32).should eq(3) end it "codegens and with bool union as left node 2" do @@ -131,7 +131,7 @@ describe "Code gen: and" do a = 2 a = false (a && 3).to_i! - ").to_i.should eq(1) + ", Int32).should eq(1) end it "codegens and with bool union as left node 3" do @@ -142,7 +142,7 @@ describe "Code gen: and" do a = 2 a = true (a && 3).to_i! - ").to_i.should eq(3) + ", Int32).should eq(3) end it "codegens and with bool union as left node 4" do @@ -153,14 +153,14 @@ describe "Code gen: and" do a = true a = nil (a && 3).to_i! - ").to_i.should eq(0) + ", Int32).should eq(0) end it "codegens assign in right node, after must be nilable" do run(" a = 1 == 2 && (b = Reference.new) b.nil? - ").to_b.should be_true + ", Bool).should be_true end it "codegens assign in right node, inside if must not be nil" do @@ -173,7 +173,7 @@ describe "Code gen: and" do else 0 end - ").to_i.should eq(1) + ", Int32).should eq(1) end it "codegens assign in right node, after if must be nilable" do @@ -181,6 +181,6 @@ describe "Code gen: and" do if 1 == 2 && (b = Reference.new) end b.nil? - ").to_b.should be_true + ", Bool).should be_true end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index ca5bc61ad3c4..31412035ff74 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -319,6 +319,18 @@ def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug:: end end +def run(code, return_type : T.class, filename : String? = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil, *, file = __FILE__) forall T + if inject_primitives + code = %(require "primitives"\n#{code}) + end + + if code.includes?(%(require "prelude")) || flags + fail "TODO: support the prelude in typed codegen specs", file: file + else + new_program.run(code, return_type: T, filename: filename, debug: debug) + end +end + def test_c(c_code, crystal_code, *, file = __FILE__, &) with_temp_c_object_file(c_code, file: file) do |o_filename| yield run(%( diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index a46d255901e5..67882e9d75dc 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -69,6 +69,56 @@ module Crystal end end + def run(code, return_type : T.class, filename : String? = nil, debug = Debug::Default) forall T + parser = new_parser(code) + parser.filename = filename + node = parser.parse + node = normalize node + node = semantic node + evaluate node, T, debug: debug + end + + def evaluate(node, return_type : T.class, debug = Debug::Default) : T forall T + visitor = CodeGenVisitor.new self, node, single_module: true, debug: debug + visitor.accept node + visitor.process_finished_hooks + visitor.finish + + llvm_mod = visitor.modules[""].mod + llvm_mod.target = target_machine.triple + + main = visitor.typed_fun?(llvm_mod, MAIN_NAME).not_nil! + llvm_context = llvm_mod.context + + # void (*__evaluate_wrapper)(void*) + wrapper_type = LLVM::Type.function([llvm_context.void_pointer], llvm_context.void) + wrapper = llvm_mod.functions.add("__evaluate_wrapper", wrapper_type) do |func| + func.basic_blocks.append "entry" do |builder| + argc = llvm_context.int32.const_int(0) + argv = llvm_context.void_pointer.pointer.null + ret = builder.call(main.type, main.func, [argc, argv]) + unless node.type.void? || node.type.nil_type? + out_ptr = func.params[0] + {% if LibLLVM::IS_LT_150 %} + out_ptr = builder.bit_cast out_ptr, main.type.return_type.pointer + {% end %} + builder.store(ret, out_ptr) + end + builder.ret + end + end + + llvm_mod.verify + + result = uninitialized T + LLVM::JITCompiler.new(llvm_mod) do |jit| + func_ptr = jit.function_address("__evaluate_wrapper") + func = Proc(T*, Nil).new(func_ptr, Pointer(Void).null) + func.call(pointerof(result)) + end + result + end + def codegen(node, single_module = false, debug = Debug::Default, frame_pointers = FramePointers::Auto) visitor = CodeGenVisitor.new self, node, single_module: single_module, diff --git a/src/llvm/jit_compiler.cr b/src/llvm/jit_compiler.cr index 33d03e697107..4acae901f381 100644 --- a/src/llvm/jit_compiler.cr +++ b/src/llvm/jit_compiler.cr @@ -39,6 +39,10 @@ class LLVM::JITCompiler LibLLVM.get_pointer_to_global(self, value) end + def function_address(name : String) : Void* + Pointer(Void).new(LibLLVM.get_function_address(self, name.check_no_null_byte)) + end + def to_unsafe @unwrap end diff --git a/src/llvm/lib_llvm/execution_engine.cr b/src/llvm/lib_llvm/execution_engine.cr index f9de5c10ea39..bfc2e23154db 100644 --- a/src/llvm/lib_llvm/execution_engine.cr +++ b/src/llvm/lib_llvm/execution_engine.cr @@ -30,4 +30,5 @@ lib LibLLVM fun run_function = LLVMRunFunction(ee : ExecutionEngineRef, f : ValueRef, num_args : UInt, args : GenericValueRef*) : GenericValueRef fun get_execution_engine_target_machine = LLVMGetExecutionEngineTargetMachine(ee : ExecutionEngineRef) : TargetMachineRef fun get_pointer_to_global = LLVMGetPointerToGlobal(ee : ExecutionEngineRef, global : ValueRef) : Void* + fun get_function_address = LLVMGetFunctionAddress(ee : ExecutionEngineRef, name : Char*) : UInt64 end From 529fa4d66df1bb92351039ceb78fa36adf5fe097 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 13 Aug 2024 18:50:43 +0800 Subject: [PATCH 019/193] Add minimal LLVM OrcV2 bindings (#14887) --- src/llvm.cr | 7 +++++ src/llvm/lib_llvm/error.cr | 3 +++ src/llvm/lib_llvm/lljit.cr | 16 +++++++++++ src/llvm/lib_llvm/orc.cr | 26 ++++++++++++++++++ src/llvm/orc/jit_dylib.cr | 16 +++++++++++ src/llvm/orc/lljit.cr | 42 +++++++++++++++++++++++++++++ src/llvm/orc/lljit_builder.cr | 35 ++++++++++++++++++++++++ src/llvm/orc/thread_safe_context.cr | 30 +++++++++++++++++++++ src/llvm/orc/thread_safe_module.cr | 36 +++++++++++++++++++++++++ 9 files changed, 211 insertions(+) create mode 100644 src/llvm/lib_llvm/lljit.cr create mode 100644 src/llvm/lib_llvm/orc.cr create mode 100644 src/llvm/orc/jit_dylib.cr create mode 100644 src/llvm/orc/lljit.cr create mode 100644 src/llvm/orc/lljit_builder.cr create mode 100644 src/llvm/orc/thread_safe_context.cr create mode 100644 src/llvm/orc/thread_safe_module.cr diff --git a/src/llvm.cr b/src/llvm.cr index 6fb8767cad54..84c9dc89aa8f 100644 --- a/src/llvm.cr +++ b/src/llvm.cr @@ -140,6 +140,13 @@ module LLVM string end + protected def self.assert(error : LibLLVM::ErrorRef) + if error + chars = LibLLVM.get_error_message(error) + raise String.new(chars).tap { LibLLVM.dispose_error_message(chars) } + end + end + {% unless LibLLVM::IS_LT_130 %} def self.run_passes(module mod : Module, passes : String, target_machine : TargetMachine, options : PassBuilderOptions) LibLLVM.run_passes(mod, passes, target_machine, options) diff --git a/src/llvm/lib_llvm/error.cr b/src/llvm/lib_llvm/error.cr index b816a7e2088b..5a035b5f80a5 100644 --- a/src/llvm/lib_llvm/error.cr +++ b/src/llvm/lib_llvm/error.cr @@ -1,3 +1,6 @@ lib LibLLVM type ErrorRef = Void* + + fun get_error_message = LLVMGetErrorMessage(err : ErrorRef) : Char* + fun dispose_error_message = LLVMDisposeErrorMessage(err_msg : Char*) end diff --git a/src/llvm/lib_llvm/lljit.cr b/src/llvm/lib_llvm/lljit.cr new file mode 100644 index 000000000000..640973024af4 --- /dev/null +++ b/src/llvm/lib_llvm/lljit.cr @@ -0,0 +1,16 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +lib LibLLVM + alias OrcLLJITBuilderRef = Void* + alias OrcLLJITRef = Void* + + fun orc_create_lljit_builder = LLVMOrcCreateLLJITBuilder : OrcLLJITBuilderRef + fun orc_dispose_lljit_builder = LLVMOrcDisposeLLJITBuilder(builder : OrcLLJITBuilderRef) + + fun orc_create_lljit = LLVMOrcCreateLLJIT(result : OrcLLJITRef*, builder : OrcLLJITBuilderRef) : ErrorRef + fun orc_dispose_lljit = LLVMOrcDisposeLLJIT(j : OrcLLJITRef) : ErrorRef + + fun orc_lljit_get_main_jit_dylib = LLVMOrcLLJITGetMainJITDylib(j : OrcLLJITRef) : OrcJITDylibRef + fun orc_lljit_add_llvm_ir_module = LLVMOrcLLJITAddLLVMIRModule(j : OrcLLJITRef, jd : OrcJITDylibRef, tsm : OrcThreadSafeModuleRef) : ErrorRef + fun orc_lljit_lookup = LLVMOrcLLJITLookup(j : OrcLLJITRef, result : OrcExecutorAddress*, name : Char*) : ErrorRef +end diff --git a/src/llvm/lib_llvm/orc.cr b/src/llvm/lib_llvm/orc.cr new file mode 100644 index 000000000000..a1650b3dfb96 --- /dev/null +++ b/src/llvm/lib_llvm/orc.cr @@ -0,0 +1,26 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +lib LibLLVM + # OrcJITTargetAddress before LLVM 13.0 (also an alias of UInt64) + alias OrcExecutorAddress = UInt64 + alias OrcSymbolStringPoolEntryRef = Void* + alias OrcJITDylibRef = Void* + alias OrcDefinitionGeneratorRef = Void* + alias OrcSymbolPredicate = Void*, OrcSymbolStringPoolEntryRef -> Int + alias OrcThreadSafeContextRef = Void* + alias OrcThreadSafeModuleRef = Void* + + fun orc_create_dynamic_library_search_generator_for_process = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( + result : OrcDefinitionGeneratorRef*, global_prefx : Char, + filter : OrcSymbolPredicate, filter_ctx : Void* + ) : ErrorRef + + fun orc_jit_dylib_add_generator = LLVMOrcJITDylibAddGenerator(jd : OrcJITDylibRef, dg : OrcDefinitionGeneratorRef) + + fun orc_create_new_thread_safe_context = LLVMOrcCreateNewThreadSafeContext : OrcThreadSafeContextRef + fun orc_thread_safe_context_get_context = LLVMOrcThreadSafeContextGetContext(ts_ctx : OrcThreadSafeContextRef) : ContextRef + fun orc_dispose_thread_safe_context = LLVMOrcDisposeThreadSafeContext(ts_ctx : OrcThreadSafeContextRef) + + fun orc_create_new_thread_safe_module = LLVMOrcCreateNewThreadSafeModule(m : ModuleRef, ts_ctx : OrcThreadSafeContextRef) : OrcThreadSafeModuleRef + fun orc_dispose_thread_safe_module = LLVMOrcDisposeThreadSafeModule(tsm : OrcThreadSafeModuleRef) +end diff --git a/src/llvm/orc/jit_dylib.cr b/src/llvm/orc/jit_dylib.cr new file mode 100644 index 000000000000..929dc5e5e6a4 --- /dev/null +++ b/src/llvm/orc/jit_dylib.cr @@ -0,0 +1,16 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::JITDylib + protected def initialize(@unwrap : LibLLVM::OrcJITDylibRef) + end + + def to_unsafe + @unwrap + end + + def link_symbols_from_current_process : Nil + LLVM.assert LibLLVM.orc_create_dynamic_library_search_generator_for_process(out dg, 0, nil, nil) + LibLLVM.orc_jit_dylib_add_generator(self, dg) + end +end diff --git a/src/llvm/orc/lljit.cr b/src/llvm/orc/lljit.cr new file mode 100644 index 000000000000..6271dea6ea56 --- /dev/null +++ b/src/llvm/orc/lljit.cr @@ -0,0 +1,42 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::LLJIT + protected def initialize(@unwrap : LibLLVM::OrcLLJITRef) + end + + def self.new(builder : LLJITBuilder) + builder.take_ownership { raise "Failed to take ownership of LLVM::Orc::LLJITBuilder" } + LLVM.assert LibLLVM.orc_create_lljit(out unwrap, builder) + new(unwrap) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LLVM.assert LibLLVM.orc_dispose_lljit(self) + @unwrap = LibLLVM::OrcLLJITRef.null + end + + def finalize + if @unwrap + LibLLVM.orc_dispose_lljit(self) + end + end + + def main_jit_dylib : JITDylib + JITDylib.new(LibLLVM.orc_lljit_get_main_jit_dylib(self)) + end + + def add_llvm_ir_module(dylib : JITDylib, tsm : ThreadSafeModule) : Nil + tsm.take_ownership { raise "Failed to take ownership of LLVM::Orc::ThreadSafeModule" } + LLVM.assert LibLLVM.orc_lljit_add_llvm_ir_module(self, dylib, tsm) + end + + def lookup(name : String) : Void* + LLVM.assert LibLLVM.orc_lljit_lookup(self, out address, name.check_no_null_byte) + Pointer(Void).new(address) + end +end diff --git a/src/llvm/orc/lljit_builder.cr b/src/llvm/orc/lljit_builder.cr new file mode 100644 index 000000000000..8147e5947376 --- /dev/null +++ b/src/llvm/orc/lljit_builder.cr @@ -0,0 +1,35 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::LLJITBuilder + protected def initialize(@unwrap : LibLLVM::OrcLLJITBuilderRef) + @dispose_on_finalize = true + end + + def self.new + new(LibLLVM.orc_create_lljit_builder) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LibLLVM.orc_dispose_lljit_builder(self) + @unwrap = LibLLVM::OrcLLJITBuilderRef.null + end + + def finalize + if @dispose_on_finalize && @unwrap + dispose + end + end + + def take_ownership(&) : Nil + if @dispose_on_finalize + @dispose_on_finalize = false + else + yield + end + end +end diff --git a/src/llvm/orc/thread_safe_context.cr b/src/llvm/orc/thread_safe_context.cr new file mode 100644 index 000000000000..38c4ece7a50a --- /dev/null +++ b/src/llvm/orc/thread_safe_context.cr @@ -0,0 +1,30 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::ThreadSafeContext + protected def initialize(@unwrap : LibLLVM::OrcThreadSafeContextRef) + end + + def self.new + new(LibLLVM.orc_create_new_thread_safe_context) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LibLLVM.orc_dispose_thread_safe_context(self) + @unwrap = LibLLVM::OrcThreadSafeContextRef.null + end + + def finalize + if @unwrap + dispose + end + end + + def context : LLVM::Context + LLVM::Context.new(LibLLVM.orc_thread_safe_context_get_context(self), false) + end +end diff --git a/src/llvm/orc/thread_safe_module.cr b/src/llvm/orc/thread_safe_module.cr new file mode 100644 index 000000000000..5e29667fd9cd --- /dev/null +++ b/src/llvm/orc/thread_safe_module.cr @@ -0,0 +1,36 @@ +{% skip_file if LibLLVM::IS_LT_110 %} + +@[Experimental("The C API wrapped by this type is marked as experimental by LLVM.")] +class LLVM::Orc::ThreadSafeModule + protected def initialize(@unwrap : LibLLVM::OrcThreadSafeModuleRef) + @dispose_on_finalize = true + end + + def self.new(llvm_mod : LLVM::Module, ts_ctx : ThreadSafeContext) + llvm_mod.take_ownership { raise "Failed to take ownership of LLVM::Module" } + new(LibLLVM.orc_create_new_thread_safe_module(llvm_mod, ts_ctx)) + end + + def to_unsafe + @unwrap + end + + def dispose : Nil + LibLLVM.orc_dispose_thread_safe_module(self) + @unwrap = LibLLVM::OrcThreadSafeModuleRef.null + end + + def finalize + if @dispose_on_finalize && @unwrap + dispose + end + end + + def take_ownership(&) : Nil + if @dispose_on_finalize + @dispose_on_finalize = false + else + yield + end + end +end From 5dca1ba606558e28a8ca2dbbf5b47d825e135b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 13 Aug 2024 23:45:41 +0200 Subject: [PATCH 020/193] Move `#evented_read`, `#evented_write` into `Crystal::LibEvent::EventLoop` (#14883) This is a simple refactor, moving two methods from `IO::Evented` to `LibEvent::EventLoop`. --- .../system/unix/event_loop_libevent.cr | 47 +++++++++++++++++-- src/crystal/system/wasi/event_loop.cr | 45 ++++++++++++++++-- src/io/evented.cr | 43 ++--------------- 3 files changed, 87 insertions(+), 48 deletions(-) diff --git a/src/crystal/system/unix/event_loop_libevent.cr b/src/crystal/system/unix/event_loop_libevent.cr index 32c9c8409b17..b67bad63ff2f 100644 --- a/src/crystal/system/unix/event_loop_libevent.cr +++ b/src/crystal/system/unix/event_loop_libevent.cr @@ -70,7 +70,7 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_read("Error reading file_descriptor") do + evented_read(file_descriptor, "Error reading file_descriptor") do LibC.read(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for reading", target: file_descriptor @@ -80,7 +80,7 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop end def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_write("Error writing file_descriptor") do + evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for writing", target: file_descriptor @@ -94,13 +94,13 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop end def read(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_read("Error reading socket") do + evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 end end def write(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_write("Error writing to socket") do + evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0).to_i32 end end @@ -114,7 +114,7 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop addrlen = LibC::SocklenT.new(sizeof(LibC::SockaddrStorage)) - bytes_read = socket.evented_read("Error receiving datagram") do + bytes_read = evented_read(socket, "Error receiving datagram") do LibC.recvfrom(socket.fd, slice, slice.size, 0, sockaddr, pointerof(addrlen)) end @@ -185,4 +185,41 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop def close(socket : ::Socket) : Nil socket.evented_close end + + def evented_read(target, errno_msg : String, &) : Int32 + loop do + bytes_read = yield + if bytes_read != -1 + # `to_i32` is acceptable because `Slice#size` is an Int32 + return bytes_read.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_readable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_readers + end + + def evented_write(target, errno_msg : String, &) : Int32 + begin + loop do + bytes_written = yield + if bytes_written != -1 + return bytes_written.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_writable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_writers + end + end end diff --git a/src/crystal/system/wasi/event_loop.cr b/src/crystal/system/wasi/event_loop.cr index 5aaf54452571..ba657b917154 100644 --- a/src/crystal/system/wasi/event_loop.cr +++ b/src/crystal/system/wasi/event_loop.cr @@ -30,7 +30,7 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_read("Error reading file_descriptor") do + evented_read(file_descriptor, "Error reading file_descriptor") do LibC.read(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for reading", target: file_descriptor @@ -40,7 +40,7 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop end def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - file_descriptor.evented_write("Error writing file_descriptor") do + evented_write(file_descriptor, "Error writing file_descriptor") do LibC.write(file_descriptor.fd, slice, slice.size).tap do |return_code| if return_code == -1 && Errno.value == Errno::EBADF raise IO::Error.new "File not open for writing", target: file_descriptor @@ -54,13 +54,13 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop end def read(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_read("Error reading socket") do + evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 end end def write(socket : ::Socket, slice : Bytes) : Int32 - socket.evented_write("Error writing to socket") do + evented_write(socket, "Error writing to socket") do LibC.send(socket.fd, slice, slice.size, 0) end end @@ -84,6 +84,43 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop def close(socket : ::Socket) : Nil socket.evented_close end + + def evented_read(target, errno_msg : String, &) : Int32 + loop do + bytes_read = yield + if bytes_read != -1 + # `to_i32` is acceptable because `Slice#size` is an Int32 + return bytes_read.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_readable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_readers + end + + def evented_write(target, errno_msg : String, &) : Int32 + begin + loop do + bytes_written = yield + if bytes_written != -1 + return bytes_written.to_i32 + end + + if Errno.value == Errno::EAGAIN + target.wait_writable + else + raise IO::Error.from_errno(errno_msg, target: target) + end + end + ensure + target.evented_resume_pending_writers + end + end end struct Crystal::Wasi::Event diff --git a/src/io/evented.cr b/src/io/evented.cr index ccc040932285..d2b3a66c336f 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -13,43 +13,6 @@ module IO::Evented @read_event = Crystal::ThreadLocalValue(Crystal::EventLoop::Event).new @write_event = Crystal::ThreadLocalValue(Crystal::EventLoop::Event).new - def evented_read(errno_msg : String, &) : Int32 - loop do - bytes_read = yield - if bytes_read != -1 - # `to_i32` is acceptable because `Slice#size` is an Int32 - return bytes_read.to_i32 - end - - if Errno.value == Errno::EAGAIN - wait_readable - else - raise IO::Error.from_errno(errno_msg, target: self) - end - end - ensure - resume_pending_readers - end - - def evented_write(errno_msg : String, &) : Int32 - begin - loop do - bytes_written = yield - if bytes_written != -1 - return bytes_written.to_i32 - end - - if Errno.value == Errno::EAGAIN - wait_writable - else - raise IO::Error.from_errno(errno_msg, target: self) - end - end - ensure - resume_pending_writers - end - end - # :nodoc: def resume_read(timed_out = false) : Nil @read_timed_out = timed_out @@ -132,13 +95,15 @@ module IO::Evented end end - private def resume_pending_readers + # :nodoc: + def evented_resume_pending_readers if (readers = @readers.get?) && !readers.empty? add_read_event end end - private def resume_pending_writers + # :nodoc: + def evented_resume_pending_writers if (writers = @writers.get?) && !writers.empty? add_write_event end From d54a91fd9e8a9c5ca59fc946d456ddafc38a05f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 13 Aug 2024 23:46:04 +0200 Subject: [PATCH 021/193] Avoid flush in finalizers for `Socket` and `IO::FileDescriptor` (#14882) Trying to flush means that it can try to write, hence call into the event loop, that may have to wait for the fd to be writable, which means calling into `epoll_wait`. This all while the world is stopped during a garbage collection run. The event loop implementation may need to allocate, or we get an error and try to raise an exception that will also try to allocate memory... again during a GC collection. For `Socket` this change has little impact becase `sync` is true by default which means every byte is sent directly without buffering. `IO::FileDescriptor` (and `File`) is buffered by default, so this may lead to loss of data if you don't properly close the file descriptor after use. --- spec/std/io/file_descriptor_spec.cr | 34 +++++++++++++++------ spec/std/socket/socket_spec.cr | 28 +++++++++++++++++ src/crystal/system/file.cr | 7 ----- src/crystal/system/file_descriptor.cr | 8 +++++ src/crystal/system/socket.cr | 8 +++++ src/crystal/system/unix/file_descriptor.cr | 10 ++++-- src/crystal/system/unix/socket.cr | 12 +++++++- src/crystal/system/win32/file_descriptor.cr | 6 ++++ src/crystal/system/win32/socket.cr | 12 +++++++- src/io/file_descriptor.cr | 13 +++++++- src/socket.cr | 10 +++++- 11 files changed, 126 insertions(+), 22 deletions(-) diff --git a/spec/std/io/file_descriptor_spec.cr b/spec/std/io/file_descriptor_spec.cr index e497ac1061a3..2e10ea99c030 100644 --- a/spec/std/io/file_descriptor_spec.cr +++ b/spec/std/io/file_descriptor_spec.cr @@ -48,17 +48,33 @@ describe IO::FileDescriptor do end end - it "closes on finalize" do - pipes = [] of IO::FileDescriptor - assert_finalizes("fd") do - a, b = IO.pipe - pipes << b - a + describe "#finalize" do + it "closes" do + pipes = [] of IO::FileDescriptor + assert_finalizes("fd") do + a, b = IO.pipe + pipes << b + a + end + + expect_raises(IO::Error) do + pipes.each do |p| + p.puts "123" + end + end end - expect_raises(IO::Error) do - pipes.each do |p| - p.puts "123" + it "does not flush" do + with_tempfile "fd-finalize-flush" do |path| + file = File.new(path, "w") + file << "foo" + file.flush + file << "bar" + file.finalize + + File.read(path).should eq "foo" + ensure + file.try(&.close) rescue nil end end end diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index d4e7051d12bd..2127e196b746 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -169,4 +169,32 @@ describe Socket, tags: "network" do socket.close_on_exec?.should be_true end {% end %} + + describe "#finalize" do + it "does not flush" do + port = unused_local_port + server = Socket.tcp(Socket::Family::INET) + server.bind("127.0.0.1", port) + server.listen + + spawn do + client = server.not_nil!.accept + client.sync = false + client << "foo" + client.flush + client << "bar" + client.finalize + ensure + client.try(&.close) rescue nil + end + + socket = Socket.tcp(Socket::Family::INET) + socket.connect(Socket::IPAddress.new("127.0.0.1", port)) + + socket.gets.should eq "foo" + ensure + socket.try &.close + server.try &.close + end + end end diff --git a/src/crystal/system/file.cr b/src/crystal/system/file.cr index 75985c107fd5..452bfb6e4ead 100644 --- a/src/crystal/system/file.cr +++ b/src/crystal/system/file.cr @@ -87,13 +87,6 @@ module Crystal::System::File private def self.error_is_file_exists?(errno) errno.in?(Errno::EEXIST, WinError::ERROR_FILE_EXISTS) end - - # Closes the internal file descriptor without notifying libevent. - # This is directly used after the fork of a process to close the - # parent's Crystal::System::Signal.@@pipe reference before re initializing - # the event loop. In the case of a fork that will exec there is even - # no need to initialize the event loop at all. - # def file_descriptor_close end {% if flag?(:wasi) %} diff --git a/src/crystal/system/file_descriptor.cr b/src/crystal/system/file_descriptor.cr index 0180627d59ce..0652ed56e52a 100644 --- a/src/crystal/system/file_descriptor.cr +++ b/src/crystal/system/file_descriptor.cr @@ -14,6 +14,14 @@ module Crystal::System::FileDescriptor # cooked mode otherwise. # private def system_raw(enable : Bool, & : ->) + # Closes the internal file descriptor without notifying the event loop. + # This is directly used after the fork of a process to close the + # parent's Crystal::System::Signal.@@pipe reference before re initializing + # the event loop. In the case of a fork that will exec there is even + # no need to initialize the event loop at all. + # Also used in `IO::FileDescriptor#finalize`. + # def file_descriptor_close + private def system_read(slice : Bytes) : Int32 event_loop.read(self, slice) end diff --git a/src/crystal/system/socket.cr b/src/crystal/system/socket.cr index 2669b4c57bca..10f902e9f0c1 100644 --- a/src/crystal/system/socket.cr +++ b/src/crystal/system/socket.cr @@ -91,6 +91,14 @@ module Crystal::System::Socket # private def system_close + # Closes the internal handle without notifying the event loop. + # This is directly used after the fork of a process to close the + # parent's Crystal::System::Signal.@@pipe reference before re initializing + # the event loop. In the case of a fork that will exec there is even + # no need to initialize the event loop at all. + # Also used in `Socket#finalize` + # def socket_close + private def event_loop : Crystal::EventLoop::Socket Crystal::EventLoop.current end diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 0c3ece9cfff8..d235114849b4 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -120,7 +120,7 @@ module Crystal::System::FileDescriptor file_descriptor_close end - def file_descriptor_close : Nil + def file_descriptor_close(&) : Nil # Clear the @volatile_fd before actually closing it in order to # reduce the chance of reading an outdated fd value _fd = @volatile_fd.swap(-1) @@ -130,11 +130,17 @@ module Crystal::System::FileDescriptor when Errno::EINTR, Errno::EINPROGRESS # ignore else - raise IO::Error.from_errno("Error closing file", target: self) + yield end end end + def file_descriptor_close + file_descriptor_close do + raise IO::Error.from_errno("Error closing file", target: self) + end + end + private def system_flock_shared(blocking) flock LibC::FlockOp::SH, blocking end diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index 33ac70659b9f..7c39e140849c 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -208,6 +208,10 @@ module Crystal::System::Socket # always lead to undefined results. This is not specific to libevent. event_loop.close(self) + socket_close + end + + private def socket_close(&) # Clear the @volatile_fd before actually closing it in order to # reduce the chance of reading an outdated fd value fd = @volatile_fd.swap(-1) @@ -219,11 +223,17 @@ module Crystal::System::Socket when Errno::EINTR, Errno::EINPROGRESS # ignore else - raise ::Socket::Error.from_errno("Error closing socket") + yield end end end + private def socket_close + socket_close do + raise ::Socket::Error.from_errno("Error closing socket") + end + end + private def system_local_address sockaddr6 = uninitialized LibC::SockaddrIn6 sockaddr = pointerof(sockaddr6).as(LibC::Sockaddr*) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index b39f98fbdf0c..d19e43b79547 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -187,6 +187,12 @@ module Crystal::System::FileDescriptor def file_descriptor_close if LibC.CloseHandle(windows_handle) == 0 + yield + end + end + + def file_descriptor_close + file_descriptor_close do raise IO::Error.from_winerror("Error closing file", target: self) end end diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 17e4ca875dbb..78645d51f320 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -366,6 +366,10 @@ module Crystal::System::Socket end def system_close + socket_close + end + + private def socket_close handle = @volatile_fd.swap(LibC::INVALID_SOCKET) ret = LibC.closesocket(handle) @@ -375,11 +379,17 @@ module Crystal::System::Socket when WinError::WSAEINTR, WinError::WSAEINPROGRESS # ignore else - raise ::Socket::Error.from_os_error("Error closing socket", err) + yield err end end end + def socket_close + socket_close do |err| + raise ::Socket::Error.from_os_error("Error closing socket", err) + end + end + private def system_local_address sockaddr6 = uninitialized LibC::SockaddrIn6 sockaddr = pointerof(sockaddr6).as(LibC::Sockaddr*) diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index d4459e9bbe0c..8940a118041f 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -233,10 +233,21 @@ class IO::FileDescriptor < IO system_flock_unlock end + # Finalizes the file descriptor resource. + # + # This involves releasing the handle to the operating system, i.e. closing it. + # It does *not* implicitly call `#flush`, so data waiting in the buffer may be + # lost. + # It's recommended to always close the file descriptor explicitly via `#close` + # (or implicitly using the `.open` constructor). + # + # Resource release can be disabled with `close_on_finalize = false`. + # + # This method is a no-op if the file descriptor has already been closed. def finalize return if closed? || !close_on_finalize? - close rescue nil + file_descriptor_close { } # ignore error end def closed? : Bool diff --git a/src/socket.cr b/src/socket.cr index ca484c0140cc..1d367f805343 100644 --- a/src/socket.cr +++ b/src/socket.cr @@ -419,10 +419,18 @@ class Socket < IO self.class.fcntl fd, cmd, arg end + # Finalizes the socket resource. + # + # This involves releasing the handle to the operating system, i.e. closing it. + # It does *not* implicitly call `#flush`, so data waiting in the buffer may be + # lost. By default write buffering is disabled, though (`sync? == true`). + # It's recommended to always close the socket explicitly via `#close`. + # + # This method is a no-op if the file descriptor has already been closed. def finalize return if closed? - close rescue nil + socket_close { } # ignore error end def closed? : Bool From 78c9282c704ca1d1ce83cefb0154ad24e7371d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Wed, 14 Aug 2024 10:12:20 +0200 Subject: [PATCH 022/193] Fix: Don't link to undocumented types in API docs (#14878) Co-authored-by: Sijawusz Pur Rahnama --- src/compiler/crystal/tools/doc/html/type.html | 10 +++++----- src/compiler/crystal/tools/doc/templates.cr | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/compiler/crystal/tools/doc/html/type.html b/src/compiler/crystal/tools/doc/html/type.html index 10c7e51fedd3..e9918c6fe429 100644 --- a/src/compiler/crystal/tools/doc/html/type.html +++ b/src/compiler/crystal/tools/doc/html/type.html @@ -45,10 +45,10 @@

<%= type.formatted_alias_definition %> <% end %> -<%= OtherTypesTemplate.new("Included Modules", type, type.included_modules) %> -<%= OtherTypesTemplate.new("Extended Modules", type, type.extended_modules) %> -<%= OtherTypesTemplate.new("Direct Known Subclasses", type, type.subclasses) %> -<%= OtherTypesTemplate.new("Direct including types", type, type.including_types) %> +<%= OtherTypesTemplate.new("Included Modules", type, included_modules_with_docs) %> +<%= OtherTypesTemplate.new("Extended Modules", type, extended_modules_with_docs) %> +<%= OtherTypesTemplate.new("Direct Known Subclasses", type, subclasses_with_docs) %> +<%= OtherTypesTemplate.new("Direct including types", type, including_types_with_docs) %> <% if locations = type.locations %>

@@ -99,7 +99,7 @@

<%= MethodSummaryTemplate.new("Instance Method Summary", type.instance_methods) %>
- <% type.ancestors.each do |ancestor| %> + <% ancestors_with_docs.each do |ancestor| %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.instance_methods, "Instance") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.constructors, "Constructor") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.class_methods, "Class") %> diff --git a/src/compiler/crystal/tools/doc/templates.cr b/src/compiler/crystal/tools/doc/templates.cr index 91ad32e1d0d1..4aaf5ac9029e 100644 --- a/src/compiler/crystal/tools/doc/templates.cr +++ b/src/compiler/crystal/tools/doc/templates.cr @@ -30,6 +30,12 @@ module Crystal::Doc end record TypeTemplate, type : Type, types : Array(Type), project_info : ProjectInfo do + {% for method in %w[ancestors included_modules extended_modules subclasses including_types] %} + def {{method.id}}_with_docs + type.{{method.id}}.select!(&.in?(types)) + end + {% end %} + ECR.def_to_s "#{__DIR__}/html/type.html" end From 38304b351bd4723cac1637cce3e2623f5789735b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 10:14:13 +0200 Subject: [PATCH 023/193] Update `LibCrypto` bindings for LibreSSL 3.5+ (#14872) * Fix libssl bindings for LibreSSL 3.5 * [CI] Add test for libreSSL 3.5 * [CI] Add test for libreSSL 3.8 --- .github/workflows/openssl.yml | 30 ++++++++++++++++++++++++++++++ src/openssl/lib_crypto.cr | 8 +++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 46d440d1f6e7..b932ce542e45 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -56,3 +56,33 @@ jobs: run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs run: bin/crystal spec --order=random spec/std/openssl/ + libressl35: + runs-on: ubuntu-latest + name: "LibreSSL 3.5" + container: crystallang/crystal:1.13.1-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Uninstall openssl + run: apk del openssl-dev openssl-libs-static + - name: Install libressl 3.5 + run: apk add "libressl-dev=~3.5" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.16/community + - name: Check LibSSL version + run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' + - name: Run OpenSSL specs + run: bin/crystal spec --order=random spec/std/openssl/ + libressl38: + runs-on: ubuntu-latest + name: "LibreSSL 3.5" + container: crystallang/crystal:1.13.1-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Uninstall openssl + run: apk del openssl-dev openssl-libs-static + - name: Install libressl 3.8 + run: apk add "libressl-dev=~3.8" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community + - name: Check LibSSL version + run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' + - name: Run OpenSSL specs + run: bin/crystal spec --order=random spec/std/openssl/ diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index aef6a238f663..8d450b28ff17 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -57,7 +57,10 @@ lib LibCrypto struct Bio method : Void* - callback : (Void*, Int, Char*, Int, Long, Long) -> Long + callback : BIO_callback_fn + {% if compare_versions(LIBRESSL_VERSION, "3.5.0") >= 0 %} + callback_ex : BIO_callback_fn_ex + {% end %} cb_arg : Char* init : Int shutdown : Int @@ -72,6 +75,9 @@ lib LibCrypto num_write : ULong end + alias BIO_callback_fn = (Bio*, Int, Char*, Int, Long, Long) -> Long + alias BIO_callback_fn_ex = (Bio*, Int, Char, SizeT, Int, Long, Int, SizeT*) -> Long + PKCS5_SALT_LEN = 8 EVP_MAX_KEY_LENGTH = 32 EVP_MAX_IV_LENGTH = 16 From 19becd57e49ed061fc43d921334bc252627599b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 10:14:40 +0200 Subject: [PATCH 024/193] [CI] Add test for OpenSSL 3.3 (#14873) --- .github/workflows/openssl.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index b932ce542e45..8cdb888e3621 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -7,7 +7,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: - openssl3: + openssl3_0: runs-on: ubuntu-latest name: "OpenSSL 3.0" container: crystallang/crystal:1.13.1-alpine @@ -24,6 +24,19 @@ jobs: run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs run: bin/crystal spec --order=random spec/std/openssl/ + openssl3_3: + runs-on: ubuntu-latest + name: "OpenSSL 3.3" + container: crystallang/crystal:1.13.1-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v4 + - name: Install openssl 3.3 + run: apk add "openssl-dev=~3.3" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community + - name: Check LibSSL version + run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' + - name: Run OpenSSL specs + run: bin/crystal spec --order=random spec/std/openssl/ openssl111: runs-on: ubuntu-latest name: "OpenSSL 1.1.1" From 9ecd838d1e81d25606c58e5ca945e6e5531ee140 Mon Sep 17 00:00:00 2001 From: kojix2 <2xijok@gmail.com> Date: Wed, 14 Aug 2024 23:42:48 +0900 Subject: [PATCH 025/193] Fix avoid linking `libpcre` when unused (#14891) --- src/regex/pcre.cr | 2 +- src/regex/pcre2.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/regex/pcre.cr b/src/regex/pcre.cr index e6cf6eaca7b0..c80714708a0b 100644 --- a/src/regex/pcre.cr +++ b/src/regex/pcre.cr @@ -6,7 +6,7 @@ module Regex::PCRE String.new(LibPCRE.version) end - class_getter version_number : {Int32, Int32} = begin + class_getter version_number : {Int32, Int32} do version = self.version dot = version.index('.') || raise RuntimeError.new("Invalid libpcre2 version") space = version.index(' ', dot) || raise RuntimeError.new("Invalid libpcre2 version") diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index da811225842f..abbb502eb78c 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -13,7 +13,7 @@ module Regex::PCRE2 end end - class_getter version_number : {Int32, Int32} = begin + class_getter version_number : {Int32, Int32} do version = self.version dot = version.index('.') || raise RuntimeError.new("Invalid libpcre2 version") space = version.index(' ', dot) || raise RuntimeError.new("Invalid libpcre2 version") From f0fece0a0a7a637a944b1c9bea004334148b6036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 16:43:03 +0200 Subject: [PATCH 026/193] [CI] Update GitHub runner to `macos-14` (#14833) Co-authored-by: Sijawusz Pur Rahnama --- .github/workflows/macos.yml | 12 ++++++++++-- .github/workflows/smoke.yml | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c19041c3f52d..d4c93a68aabb 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -11,8 +11,16 @@ env: CI_NIX_SHELL: true jobs: - x86_64-darwin-test: - runs-on: macos-13 + darwin-test: + runs-on: ${{ matrix.runs-on }} + name: ${{ matrix.arch }} + strategy: + matrix: + include: + - runs-on: macos-13 + arch: x86_64-darwin + - runs-on: macos-14 + arch: aarch64-darwin steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 8deffd149dbd..7ae103e528cf 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,7 +51,6 @@ jobs: matrix: target: - aarch64-linux-android - - aarch64-darwin - arm-linux-gnueabihf - i386-linux-gnu - i386-linux-musl From 76c6b2f58475fcdfba5d50e09589a91cb3c2a2a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 16:43:37 +0200 Subject: [PATCH 027/193] Deprecate `Pointer.new(Int)` (#14875) --- src/pointer.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pointer.cr b/src/pointer.cr index c3ebbf3e56fc..06565298d376 100644 --- a/src/pointer.cr +++ b/src/pointer.cr @@ -420,6 +420,7 @@ struct Pointer(T) # ptr = Pointer(Int32).new(5678) # ptr.address # => 5678 # ``` + @[Deprecated("Call `.new(UInt64)` directly instead")] def self.new(address : Int) new address.to_u64! end From 4f310103d167e5c317088b0a40300fc25dfb8eac Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Wed, 14 Aug 2024 17:30:20 -0400 Subject: [PATCH 028/193] Add location info to some `MacroIf` nodes (#14885) --- spec/compiler/parser/parser_spec.cr | 90 +++++++++++++++++++++++++++ src/compiler/crystal/syntax/parser.cr | 5 +- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 22e9c5feb385..db69fa357d59 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -2606,6 +2606,96 @@ module Crystal node.end_location.not_nil!.line_number.should eq(5) end + it "sets correct locations of macro if / else" do + parser = Parser.new(<<-CR) + {% if 1 == val %} + "one!" + "bar" + {% else %} + "not one" + "bar" + {% end %} + CR + + node = parser.parse.as MacroIf + + location = node.cond.location.should_not be_nil + location.line_number.should eq 1 + location = node.cond.end_location.should_not be_nil + location.line_number.should eq 1 + + location = node.then.location.should_not be_nil + location.line_number.should eq 1 + location = node.then.end_location.should_not be_nil + location.line_number.should eq 4 + + location = node.else.location.should_not be_nil + location.line_number.should eq 4 + location = node.else.end_location.should_not be_nil + location.line_number.should eq 7 + end + + it "sets correct locations of macro if / elsif" do + parser = Parser.new(<<-CR) + {% if 1 == val %} + "one!" + "bar" + {% elsif 2 == val %} + "not one" + "bar" + {% end %} + CR + + node = parser.parse.as MacroIf + + location = node.cond.location.should_not be_nil + location.line_number.should eq 1 + location = node.cond.end_location.should_not be_nil + location.line_number.should eq 1 + + location = node.then.location.should_not be_nil + location.line_number.should eq 1 + location = node.then.end_location.should_not be_nil + location.line_number.should eq 4 + + location = node.else.location.should_not be_nil + location.line_number.should eq 4 + location = node.else.end_location.should_not be_nil + location.line_number.should eq 7 + end + + it "sets correct locations of macro if / else / elsif" do + parser = Parser.new(<<-CR) + {% if 1 == val %} + "one!" + "bar" + {% elsif 2 == val %} + "not one" + "bar" + {% else %} + "biz" + "blah" + {% end %} + CR + + node = parser.parse.as MacroIf + + location = node.cond.location.should_not be_nil + location.line_number.should eq 1 + location = node.cond.end_location.should_not be_nil + location.line_number.should eq 1 + + location = node.then.location.should_not be_nil + location.line_number.should eq 1 + location = node.then.end_location.should_not be_nil + location.line_number.should eq 4 + + location = node.else.location.should_not be_nil + location.line_number.should eq 4 + location = node.else.end_location.should_not be_nil + location.line_number.should eq 10 + end + it "sets correct location of trailing ensure" do parser = Parser.new("foo ensure bar") node = parser.parse.as(ExceptionHandler) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index bd7c67a975b8..15bd221fd8b2 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3211,7 +3211,7 @@ module Crystal case @token.type when .macro_literal? - pieces << MacroLiteral.new(@token.value.to_s) + pieces << MacroLiteral.new(@token.value.to_s).at(@token.location).at_end(token_end_location) when .macro_expression_start? pieces << MacroExpression.new(parse_macro_expression) check_macro_expression_end @@ -3475,7 +3475,8 @@ module Crystal end when Keyword::ELSIF unexpected_token if is_unless - a_else = parse_macro_if(start_location, macro_state, false) + start_loc = @token.location + a_else = parse_macro_if(start_location, macro_state, false).at(start_loc) if check_end check_ident :end From 7cee8841b9602fca293a02bd0da1be37c068d47d Mon Sep 17 00:00:00 2001 From: "WukongRework.exe BROKE" Date: Wed, 14 Aug 2024 17:31:21 -0400 Subject: [PATCH 029/193] Add `LLVM::Builder#finalize` (#14892) `LLVM::Builder` is a class that wraps a `LibLLVM::BuilderRef`. It has a [protected `dispose` method](https://github.com/crystal-lang/crystal/blob/master/src/llvm/builder.cr#L381) which does call `LibLLVM.dispose_builder`. This is ok when a builder is created through a `LLVM::Context` as the `finalize` method disposes of all builders [here](https://github.com/crystal-lang/crystal/blob/master/src/llvm/context.cr#L147). However in some locations, when a builder is created not through a `LLVM::Context`, the builder is leaked. This can be seen through [`LLVM::BasicBlockCollection::append`](https://github.com/crystal-lang/crystal/blob/master/src/llvm/basic_block_collection.cr#L18) where a builder is created in context through the use of `LibLLVM` (not going through the bookkeeping of a `LLVM::Context`) and yielded to the resulting block. --- src/llvm/builder.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/llvm/builder.cr b/src/llvm/builder.cr index 3f2060b32084..b406d84145e5 100644 --- a/src/llvm/builder.cr +++ b/src/llvm/builder.cr @@ -387,6 +387,10 @@ class LLVM::Builder LibLLVM.dispose_builder(@unwrap) end + def finalize + dispose + end + # The next lines are for ease debugging when a types/values # are incorrectly used across contexts. From 93ac143279a7bde3448a385d3bfc15b0ed1d8312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 14 Aug 2024 23:32:04 +0200 Subject: [PATCH 030/193] Refactor interpreter stack code to avoid duplicate macro expansion (#14876) --- src/compiler/crystal/interpreter/interpreter.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index aa90d83f413f..f73cba958851 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -1000,16 +1000,16 @@ class Crystal::Repl::Interpreter private macro stack_pop(t) %aligned_size = align(sizeof({{t}})) %value = uninitialized {{t}} - (stack - %aligned_size).copy_to(pointerof(%value).as(UInt8*), sizeof({{t}})) + (stack - %aligned_size).copy_to(pointerof(%value).as(UInt8*), sizeof(typeof(%value))) stack_shrink_by(%aligned_size) %value end private macro stack_push(value) %temp = {{value}} - stack.copy_from(pointerof(%temp).as(UInt8*), sizeof(typeof({{value}}))) + %size = sizeof(typeof(%temp)) - %size = sizeof(typeof({{value}})) + stack.copy_from(pointerof(%temp).as(UInt8*), %size) %aligned_size = align(%size) stack += %size stack_grow_by(%aligned_size - %size) From 1a243ad220c7d6df8d9d33ac5f96f0bdc8335d21 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 15 Aug 2024 17:04:48 +0800 Subject: [PATCH 031/193] Implement `#sort_by` inside macros using `Enumerable#sort_by` (#14895) Ensures that the block in `ArrayLiteral#sort_by` and `TupleLiteral#sort_by` is called exactly once for each element, by not using `Enumerable#sort` under the hood. --- spec/compiler/macro/macro_methods_spec.cr | 14 ++++++++++---- src/compiler/crystal/macros/methods.cr | 21 +++++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 38b08f44568a..385e165a3504 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -928,6 +928,16 @@ module Crystal assert_macro %({{["c".id, "b", "a".id].sort}}), %([a, "b", c]) end + it "executes sort_by" do + assert_macro %({{["abc", "a", "ab"].sort_by { |x| x.size }}}), %(["a", "ab", "abc"]) + end + + it "calls block exactly once for each element in #sort_by" do + assert_macro <<-CRYSTAL, %(5) + {{ (i = 0; ["abc", "a", "ab", "abcde", "abcd"].sort_by { i += 1 }; i) }} + CRYSTAL + end + it "executes uniq" do assert_macro %({{[1, 1, 1, 2, 3, 1, 2, 3, 4].uniq}}), %([1, 2, 3, 4]) end @@ -1020,10 +1030,6 @@ module Crystal assert_macro %({{{:a => 1, :b => 3}.size}}), "2" end - it "executes sort_by" do - assert_macro %({{["abc", "a", "ab"].sort_by { |x| x.size }}}), %(["a", "ab", "abc"]) - end - it "executes empty?" do assert_macro %({{{:a => 1}.empty?}}), "false" end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index d3a1a1cc15a6..8a7aa569fa95 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -3232,12 +3232,17 @@ end private def sort_by(object, klass, block, interpreter) block_arg = block.args.first? - klass.new(object.elements.sort { |x, y| - block_arg.try { |arg| interpreter.define_var(arg.name, x) } - x_result = interpreter.accept(block.body) - block_arg.try { |arg| interpreter.define_var(arg.name, y) } - y_result = interpreter.accept(block.body) - - x_result.interpret_compare(y_result) - }) + klass.new(object.elements.sort_by do |elem| + block_arg.try { |arg| interpreter.define_var(arg.name, elem) } + result = interpreter.accept(block.body) + InterpretCompareWrapper.new(result) + end) +end + +private record InterpretCompareWrapper, node : Crystal::ASTNode do + include Comparable(self) + + def <=>(other : self) + node.interpret_compare(other.node) + end end From be4a20be82bd82fe1ba6a5af8412dbe5ffb9ede8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 15 Aug 2024 21:42:42 +0800 Subject: [PATCH 032/193] Fix codegen spec for `ProcPointer` of virtual type (#14903) --- spec/compiler/codegen/proc_spec.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/compiler/codegen/proc_spec.cr b/spec/compiler/codegen/proc_spec.cr index 217f2b8ba9a5..48db694429e5 100644 --- a/spec/compiler/codegen/proc_spec.cr +++ b/spec/compiler/codegen/proc_spec.cr @@ -966,7 +966,6 @@ describe "Code gen: proc" do )).to_i.should eq(1) end - # FIXME: JIT compilation of this spec is broken, forcing normal compilation (#10961) it "doesn't crash when taking a proc pointer to a virtual type (#9823)" do run(%( abstract struct Parent @@ -990,7 +989,7 @@ describe "Code gen: proc" do end Child1.new.as(Parent).get - ), flags: [] of String) + ), Proc(Int32, Int32, Int32)) end it "doesn't crash when taking a proc pointer that multidispatches on the top-level (#3822)" do From fa0240b5be832004adc1172e2183b0caac8241fd Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 01:17:33 +0800 Subject: [PATCH 033/193] Support LLVM OrcV2 codegen specs (#14886) --- spec/spec_helper.cr | 2 +- src/compiler/crystal/codegen/codegen.cr | 41 ++++++++++++++++++++----- src/llvm/lib_llvm/lljit.cr | 1 + src/llvm/orc/jit_dylib.cr | 4 +-- src/llvm/orc/lljit.cr | 4 +++ 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 31412035ff74..ae7e9a5cefca 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -284,7 +284,7 @@ def create_spec_compiler compiler end -def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil, *, file = __FILE__) +def run(code, filename : String? = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil, *, file = __FILE__) : LLVM::GenericValue | SpecRunOutput if inject_primitives code = %(require "primitives"\n#{code}) end diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index 67882e9d75dc..f040f87e17f5 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -17,7 +17,7 @@ module Crystal ONCE = "__crystal_once" class Program - def run(code, filename = nil, debug = Debug::Default) + def run(code, filename : String? = nil, debug = Debug::Default) parser = new_parser(code) parser.filename = filename node = parser.parse @@ -79,7 +79,17 @@ module Crystal end def evaluate(node, return_type : T.class, debug = Debug::Default) : T forall T - visitor = CodeGenVisitor.new self, node, single_module: true, debug: debug + llvm_context = + {% if LibLLVM::IS_LT_110 %} + LLVM::Context.new + {% else %} + begin + ts_ctx = LLVM::Orc::ThreadSafeContext.new + ts_ctx.context + end + {% end %} + + visitor = CodeGenVisitor.new self, node, single_module: true, debug: debug, llvm_context: llvm_context visitor.accept node visitor.process_finished_hooks visitor.finish @@ -88,7 +98,6 @@ module Crystal llvm_mod.target = target_machine.triple main = visitor.typed_fun?(llvm_mod, MAIN_NAME).not_nil! - llvm_context = llvm_mod.context # void (*__evaluate_wrapper)(void*) wrapper_type = LLVM::Type.function([llvm_context.void_pointer], llvm_context.void) @@ -111,11 +120,27 @@ module Crystal llvm_mod.verify result = uninitialized T - LLVM::JITCompiler.new(llvm_mod) do |jit| - func_ptr = jit.function_address("__evaluate_wrapper") + + {% if LibLLVM::IS_LT_110 %} + LLVM::JITCompiler.new(llvm_mod) do |jit| + func_ptr = jit.function_address("__evaluate_wrapper") + func = Proc(T*, Nil).new(func_ptr, Pointer(Void).null) + func.call(pointerof(result)) + end + {% else %} + lljit_builder = LLVM::Orc::LLJITBuilder.new + lljit = LLVM::Orc::LLJIT.new(lljit_builder) + + dylib = lljit.main_jit_dylib + dylib.link_symbols_from_current_process(lljit.global_prefix) + tsm = LLVM::Orc::ThreadSafeModule.new(llvm_mod, ts_ctx) + lljit.add_llvm_ir_module(dylib, tsm) + + func_ptr = lljit.lookup("__evaluate_wrapper") func = Proc(T*, Nil).new(func_ptr, Pointer(Void).null) func.call(pointerof(result)) - end + {% end %} + result end @@ -245,9 +270,9 @@ module Crystal def initialize(@program : Program, @node : ASTNode, @single_module : Bool = false, @debug = Debug::Default, - @frame_pointers : FramePointers = :auto) + @frame_pointers : FramePointers = :auto, + @llvm_context : LLVM::Context = LLVM::Context.new) @abi = @program.target_machine.abi - @llvm_context = LLVM::Context.new # LLVM::Context.register(@llvm_context, "main") @llvm_mod = @llvm_context.new_module("main_module") @main_mod = @llvm_mod diff --git a/src/llvm/lib_llvm/lljit.cr b/src/llvm/lib_llvm/lljit.cr index 640973024af4..93c2089c9db0 100644 --- a/src/llvm/lib_llvm/lljit.cr +++ b/src/llvm/lib_llvm/lljit.cr @@ -11,6 +11,7 @@ lib LibLLVM fun orc_dispose_lljit = LLVMOrcDisposeLLJIT(j : OrcLLJITRef) : ErrorRef fun orc_lljit_get_main_jit_dylib = LLVMOrcLLJITGetMainJITDylib(j : OrcLLJITRef) : OrcJITDylibRef + fun orc_lljit_get_global_prefix = LLVMOrcLLJITGetGlobalPrefix(j : OrcLLJITRef) : Char fun orc_lljit_add_llvm_ir_module = LLVMOrcLLJITAddLLVMIRModule(j : OrcLLJITRef, jd : OrcJITDylibRef, tsm : OrcThreadSafeModuleRef) : ErrorRef fun orc_lljit_lookup = LLVMOrcLLJITLookup(j : OrcLLJITRef, result : OrcExecutorAddress*, name : Char*) : ErrorRef end diff --git a/src/llvm/orc/jit_dylib.cr b/src/llvm/orc/jit_dylib.cr index 929dc5e5e6a4..b1050725110b 100644 --- a/src/llvm/orc/jit_dylib.cr +++ b/src/llvm/orc/jit_dylib.cr @@ -9,8 +9,8 @@ class LLVM::Orc::JITDylib @unwrap end - def link_symbols_from_current_process : Nil - LLVM.assert LibLLVM.orc_create_dynamic_library_search_generator_for_process(out dg, 0, nil, nil) + def link_symbols_from_current_process(global_prefix : Char) : Nil + LLVM.assert LibLLVM.orc_create_dynamic_library_search_generator_for_process(out dg, global_prefix.ord.to_u8, nil, nil) LibLLVM.orc_jit_dylib_add_generator(self, dg) end end diff --git a/src/llvm/orc/lljit.cr b/src/llvm/orc/lljit.cr index 6271dea6ea56..62fcc7f0519f 100644 --- a/src/llvm/orc/lljit.cr +++ b/src/llvm/orc/lljit.cr @@ -30,6 +30,10 @@ class LLVM::Orc::LLJIT JITDylib.new(LibLLVM.orc_lljit_get_main_jit_dylib(self)) end + def global_prefix : Char + LibLLVM.orc_lljit_get_global_prefix(self).unsafe_chr + end + def add_llvm_ir_module(dylib : JITDylib, tsm : ThreadSafeModule) : Nil tsm.take_ownership { raise "Failed to take ownership of LLVM::Orc::ThreadSafeModule" } LLVM.assert LibLLVM.orc_lljit_add_llvm_ir_module(self, dylib, tsm) From f451be644a4f7c5194ca21d5367d21a772989906 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 03:48:53 +0800 Subject: [PATCH 034/193] Fix misaligned store in `Bool` to union upcasts (#14906) The code path for `Nil` looks similar, but it is perfectly fine: it directly stores `[8 x i64] zeroinitializer` to the data field, whose default alignment naturally matches. --- spec/compiler/codegen/union_type_spec.cr | 19 +++++++++++++++++++ src/compiler/crystal/codegen/unions.cr | 9 ++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/spec/compiler/codegen/union_type_spec.cr b/spec/compiler/codegen/union_type_spec.cr index eb561a92dbdd..8ea7d058bff9 100644 --- a/spec/compiler/codegen/union_type_spec.cr +++ b/spec/compiler/codegen/union_type_spec.cr @@ -215,4 +215,23 @@ describe "Code gen: union type" do Union(Nil, Int32).foo )).to_string.should eq("TupleLiteral") end + + it "respects union payload alignment when upcasting Bool (#14898)" do + mod = codegen(<<-CRYSTAL) + x = uninitialized Bool | UInt8[64] + x = true + CRYSTAL + + str = mod.to_s + {% if LibLLVM::IS_LT_150 %} + str.should contain("store i512 1, i512* %2, align 8") + {% else %} + str.should contain("store i512 1, ptr %1, align 8") + {% end %} + + # an i512 store defaults to 16-byte alignment, which is undefined behavior + # as it overestimates the actual alignment of `x`'s data field (x86 in + # particular segfaults on misaligned 16-byte stores) + str.should_not contain("align 16") + end end diff --git a/src/compiler/crystal/codegen/unions.cr b/src/compiler/crystal/codegen/unions.cr index b2b63a17c5ab..fdf1d81a4c95 100644 --- a/src/compiler/crystal/codegen/unions.cr +++ b/src/compiler/crystal/codegen/unions.cr @@ -81,16 +81,19 @@ module Crystal def store_bool_in_union(target_type, union_pointer, value) struct_type = llvm_type(target_type) + union_value_type = struct_type.struct_element_types[1] store type_id(value, @program.bool), union_type_id(struct_type, union_pointer) # To store a boolean in a union - # we sign-extend it to the size in bits of the union - union_size = @llvm_typer.size_of(struct_type.struct_element_types[1]) + # we zero-extend it to the size in bits of the union + union_size = @llvm_typer.size_of(union_value_type) int_type = llvm_context.int((union_size * 8).to_i32) bool_as_extended_int = builder.zext(value, int_type) casted_value_ptr = pointer_cast(union_value(struct_type, union_pointer), int_type.pointer) - store bool_as_extended_int, casted_value_ptr + inst = store bool_as_extended_int, casted_value_ptr + set_alignment(inst, @llvm_typer.align_of(union_value_type)) + inst end def store_nil_in_union(target_type, union_pointer) From 3bf34106ca718c220629d1977e8db72e935dadad Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 16 Aug 2024 03:24:41 -0400 Subject: [PATCH 035/193] Fix handle empty string in `String#to_f(whitespace: false)` (#14902) Co-authored-by: Sijawusz Pur Rahnama --- spec/std/string_spec.cr | 2 ++ src/string.cr | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 2ea13d52010d..6bb4bd2c0c62 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -321,6 +321,7 @@ describe "String" do it { expect_raises(ArgumentError) { "1__234".to_i } } it { expect_raises(ArgumentError) { "1_234".to_i } } it { expect_raises(ArgumentError) { " 1234 ".to_i(whitespace: false) } } + it { expect_raises(ArgumentError) { "".to_i(whitespace: false) } } it { expect_raises(ArgumentError) { "0x123".to_i } } it { expect_raises(ArgumentError) { "0b123".to_i } } it { expect_raises(ArgumentError) { "000b123".to_i(prefix: true) } } @@ -515,6 +516,7 @@ describe "String" do "nan".to_f?(whitespace: false).try(&.nan?).should be_true " nan".to_f?(whitespace: false).should be_nil "nan ".to_f?(whitespace: false).should be_nil + expect_raises(ArgumentError) { "".to_f(whitespace: false) } "nani".to_f?(strict: true).should be_nil " INF".to_f?.should eq Float64::INFINITY "INF".to_f?.should eq Float64::INFINITY diff --git a/src/string.cr b/src/string.cr index 08bbb87fc505..cf96401253b8 100644 --- a/src/string.cr +++ b/src/string.cr @@ -752,7 +752,8 @@ class String end private def to_f_impl(whitespace : Bool = true, strict : Bool = true, &) - return unless whitespace || '0' <= self[0] <= '9' || self[0].in?('-', '+', 'i', 'I', 'n', 'N') + return unless first_char = self[0]? + return unless whitespace || '0' <= first_char <= '9' || first_char.in?('-', '+', 'i', 'I', 'n', 'N') v, endptr = yield From 7ee895fa8703aa7955b65bd9f12b8f6cf32835a9 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 21:57:53 +0800 Subject: [PATCH 036/193] Don't spawn subprocess if codegen spec uses flags but not the prelude (#14904) --- spec/compiler/codegen/macro_spec.cr | 5 +++++ spec/spec_helper.cr | 12 ++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/spec/compiler/codegen/macro_spec.cr b/spec/compiler/codegen/macro_spec.cr index 0cae55711568..fcf1092192b4 100644 --- a/spec/compiler/codegen/macro_spec.cr +++ b/spec/compiler/codegen/macro_spec.cr @@ -1885,4 +1885,9 @@ describe "Code gen: macro" do {% end %} )).to_i.should eq(10) end + + it "accepts compile-time flags" do + run("{{ flag?(:foo) ? 1 : 0 }}", flags: %w(foo)).to_i.should eq(1) + run("{{ flag?(:foo) ? 1 : 0 }}", Int32, flags: %w(foo)).should eq(1) + end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index ae7e9a5cefca..d3ccdf13fc87 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -294,7 +294,7 @@ def run(code, filename : String? = nil, inject_primitives = true, debug = Crysta # in the current executable!), so instead we compile # the program and run it, printing the last # expression and using that to compare the result. - if code.includes?(%(require "prelude")) || flags + if code.includes?(%(require "prelude")) ast = Parser.parse(code).as(Expressions) last = ast.expressions.last assign = Assign.new(Var.new("__tempvar"), last) @@ -315,7 +315,9 @@ def run(code, filename : String? = nil, inject_primitives = true, debug = Crysta return SpecRunOutput.new(output) end else - new_program.run(code, filename: filename, debug: debug) + program = new_program + program.flags.concat(flags) if flags + program.run(code, filename: filename, debug: debug) end end @@ -324,10 +326,12 @@ def run(code, return_type : T.class, filename : String? = nil, inject_primitives code = %(require "primitives"\n#{code}) end - if code.includes?(%(require "prelude")) || flags + if code.includes?(%(require "prelude")) fail "TODO: support the prelude in typed codegen specs", file: file else - new_program.run(code, return_type: T, filename: filename, debug: debug) + program = new_program + program.flags.concat(flags) if flags + program.run(code, return_type: T, filename: filename, debug: debug) end end From 75ced20b1417b43f83930a22e416984b9d79923f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 18 Aug 2024 19:30:00 +0200 Subject: [PATCH 037/193] Extract `select` from `src/channel.cr` (#14912) --- src/channel.cr | 263 +-------------------------- src/channel/select.cr | 158 ++++++++++++++++ src/channel/select/select_action.cr | 45 +++++ src/channel/select/timeout_action.cr | 66 +++++++ 4 files changed, 270 insertions(+), 262 deletions(-) create mode 100644 src/channel/select.cr create mode 100644 src/channel/select/select_action.cr create mode 100644 src/channel/select/timeout_action.cr diff --git a/src/channel.cr b/src/channel.cr index dfd61ff51cc4..4e23f8bb9b09 100644 --- a/src/channel.cr +++ b/src/channel.cr @@ -1,6 +1,7 @@ require "fiber" require "crystal/spin_lock" require "crystal/pointer_linked_list" +require "channel/select" # A `Channel` enables concurrent communication between fibers. # @@ -26,106 +27,15 @@ class Channel(T) @lock = Crystal::SpinLock.new @queue : Deque(T)? - # :nodoc: - record NotReady # :nodoc: record UseDefault - # :nodoc: - module SelectAction(S) - abstract def execute : DeliveryState - abstract def wait(context : SelectContext(S)) - abstract def wait_result_impl(context : SelectContext(S)) - abstract def unwait_impl(context : SelectContext(S)) - abstract def result : S - abstract def lock_object_id - abstract def lock - abstract def unlock - - def create_context_and_wait(shared_state) - context = SelectContext.new(shared_state, self) - self.wait(context) - context - end - - # wait_result overload allow implementors to define - # wait_result_impl with the right type and Channel.select_impl - # to allow dispatching over unions that will not happen - def wait_result(context : SelectContext) - raise "BUG: Unexpected call to #{typeof(self)}#wait_result(context : #{typeof(context)})" - end - - def wait_result(context : SelectContext(S)) - wait_result_impl(context) - end - - # idem wait_result/wait_result_impl - def unwait(context : SelectContext) - raise "BUG: Unexpected call to #{typeof(self)}#unwait(context : #{typeof(context)})" - end - - def unwait(context : SelectContext(S)) - unwait_impl(context) - end - - # Implementor that returns `Channel::UseDefault` in `#execute` - # must redefine `#default_result` - def default_result - raise "Unreachable" - end - end - - private enum SelectState - None = 0 - Active = 1 - Done = 2 - end - - private class SelectContextSharedState - @state : Atomic(SelectState) - - def initialize(value : SelectState) - @state = Atomic(SelectState).new(value) - end - - def compare_and_set(cmp : SelectState, new : SelectState) : {SelectState, Bool} - @state.compare_and_set(cmp, new) - end - end - - private class SelectContext(S) - @state : SelectContextSharedState - property action : SelectAction(S) - @activated = false - - def initialize(@state, @action : SelectAction(S)) - end - - def activated? : Bool - @activated - end - - def try_trigger : Bool - _, succeed = @state.compare_and_set(:active, :done) - if succeed - @activated = true - end - succeed - end - end - class ClosedError < Exception def initialize(msg = "Channel is closed") super(msg) end end - private enum DeliveryState - None - Delivered - Closed - end - private module SenderReceiverCloseAction def close self.state = DeliveryState::Closed @@ -398,112 +308,6 @@ class Channel(T) nil end - # :nodoc: - def self.select(*ops : SelectAction) - self.select ops - end - - # :nodoc: - def self.select(ops : Indexable(SelectAction)) - i, m = select_impl(ops, false) - raise "BUG: Blocking select returned not ready status" if m.is_a?(NotReady) - return i, m - end - - # :nodoc: - def self.non_blocking_select(*ops : SelectAction) - self.non_blocking_select ops - end - - # :nodoc: - def self.non_blocking_select(ops : Indexable(SelectAction)) - select_impl(ops, true) - end - - private def self.select_impl(ops : Indexable(SelectAction), non_blocking) - # ops_locks is a duplicate of ops that can be sorted without disturbing the - # index positions of ops - if ops.responds_to?(:unstable_sort_by!) - # If the collection type implements `unstable_sort_by!` we can dup it. - # This applies to two types: - # * `Array`: `Array#to_a` does not dup and would return the same instance, - # thus we'd be sorting ops and messing up the index positions. - # * `StaticArray`: This avoids a heap allocation because we can dup a - # static array on the stack. - ops_locks = ops.dup - elsif ops.responds_to?(:to_static_array) - # If the collection type implements `to_static_array` we can create a - # copy without allocating an array. This applies to `Tuple` types, which - # the compiler generates for `select` expressions. - ops_locks = ops.to_static_array - else - ops_locks = ops.to_a - end - - # Sort the operations by the channel they contain - # This is to avoid deadlocks between concurrent `select` calls - ops_locks.unstable_sort_by!(&.lock_object_id) - - each_skip_duplicates(ops_locks, &.lock) - - ops.each_with_index do |op, index| - state = op.execute - - case state - in .delivered? - each_skip_duplicates(ops_locks, &.unlock) - return index, op.result - in .closed? - each_skip_duplicates(ops_locks, &.unlock) - return index, op.default_result - in .none? - # do nothing - end - end - - if non_blocking - each_skip_duplicates(ops_locks, &.unlock) - return ops.size, NotReady.new - end - - # Because `channel#close` may clean up a long list, `select_context.try_trigger` may - # be called after the select return. In order to prevent invalid address access, - # the state is allocated in the heap. - shared_state = SelectContextSharedState.new(SelectState::Active) - contexts = ops.map &.create_context_and_wait(shared_state) - - each_skip_duplicates(ops_locks, &.unlock) - Fiber.suspend - - contexts.each_with_index do |context, index| - op = ops[index] - op.lock - op.unwait(context) - op.unlock - end - - contexts.each_with_index do |context, index| - if context.activated? - return index, ops[index].wait_result(context) - end - end - - raise "BUG: Fiber was awaken from select but no action was activated" - end - - private def self.each_skip_duplicates(ops_locks, &) - # Avoid deadlocks from trying to lock the same lock twice. - # `ops_lock` is sorted by `lock_object_id`, so identical onces will be in - # a row and we skip repeats while iterating. - last_lock_id = nil - ops_locks.each do |op| - if op.lock_object_id != last_lock_id - last_lock_id = op.lock_object_id - yield op - end - end - end - # :nodoc: def send_select_action(value : T) SendAction.new(self, value) @@ -699,69 +503,4 @@ class Channel(T) raise ClosedError.new end end - - # :nodoc: - class TimeoutAction - include SelectAction(Nil) - - # Total amount of time to wait - @timeout : Time::Span - @select_context : SelectContext(Nil)? - - def initialize(@timeout : Time::Span) - end - - def execute : DeliveryState - DeliveryState::None - end - - def result : Nil - nil - end - - def wait(context : SelectContext(Nil)) : Nil - @select_context = context - Fiber.timeout(@timeout, self) - end - - def wait_result_impl(context : SelectContext(Nil)) - nil - end - - def unwait_impl(context : SelectContext(Nil)) - Fiber.cancel_timeout - end - - def lock_object_id : UInt64 - self.object_id - end - - def lock - end - - def unlock - end - - def time_expired(fiber : Fiber) : Nil - if @select_context.try &.try_trigger - fiber.enqueue - end - end - end -end - -# Timeout keyword for use in `select`. -# -# ``` -# select -# when x = ch.receive -# puts "got #{x}" -# when timeout(1.seconds) -# puts "timeout" -# end -# ``` -# -# NOTE: It won't trigger if the `select` has an `else` case (i.e.: a non-blocking select). -def timeout_select_action(timeout : Time::Span) : Channel::TimeoutAction - Channel::TimeoutAction.new(timeout) end diff --git a/src/channel/select.cr b/src/channel/select.cr new file mode 100644 index 000000000000..5628fd460e6e --- /dev/null +++ b/src/channel/select.cr @@ -0,0 +1,158 @@ +class Channel(T) + # :nodoc: + record NotReady + + private enum SelectState + None = 0 + Active = 1 + Done = 2 + end + + private class SelectContextSharedState + @state : Atomic(SelectState) + + def initialize(value : SelectState) + @state = Atomic(SelectState).new(value) + end + + def compare_and_set(cmp : SelectState, new : SelectState) : {SelectState, Bool} + @state.compare_and_set(cmp, new) + end + end + + private class SelectContext(S) + @state : SelectContextSharedState + property action : SelectAction(S) + @activated = false + + def initialize(@state, @action : SelectAction(S)) + end + + def activated? : Bool + @activated + end + + def try_trigger : Bool + _, succeed = @state.compare_and_set(:active, :done) + if succeed + @activated = true + end + succeed + end + end + + private enum DeliveryState + None + Delivered + Closed + end + + # :nodoc: + def self.select(*ops : SelectAction) + self.select ops + end + + # :nodoc: + def self.select(ops : Indexable(SelectAction)) + i, m = select_impl(ops, false) + raise "BUG: Blocking select returned not ready status" if m.is_a?(NotReady) + return i, m + end + + # :nodoc: + def self.non_blocking_select(*ops : SelectAction) + self.non_blocking_select ops + end + + # :nodoc: + def self.non_blocking_select(ops : Indexable(SelectAction)) + select_impl(ops, true) + end + + private def self.select_impl(ops : Indexable(SelectAction), non_blocking) + # ops_locks is a duplicate of ops that can be sorted without disturbing the + # index positions of ops + if ops.responds_to?(:unstable_sort_by!) + # If the collection type implements `unstable_sort_by!` we can dup it. + # This applies to two types: + # * `Array`: `Array#to_a` does not dup and would return the same instance, + # thus we'd be sorting ops and messing up the index positions. + # * `StaticArray`: This avoids a heap allocation because we can dup a + # static array on the stack. + ops_locks = ops.dup + elsif ops.responds_to?(:to_static_array) + # If the collection type implements `to_static_array` we can create a + # copy without allocating an array. This applies to `Tuple` types, which + # the compiler generates for `select` expressions. + ops_locks = ops.to_static_array + else + ops_locks = ops.to_a + end + + # Sort the operations by the channel they contain + # This is to avoid deadlocks between concurrent `select` calls + ops_locks.unstable_sort_by!(&.lock_object_id) + + each_skip_duplicates(ops_locks, &.lock) + + ops.each_with_index do |op, index| + state = op.execute + + case state + in .delivered? + each_skip_duplicates(ops_locks, &.unlock) + return index, op.result + in .closed? + each_skip_duplicates(ops_locks, &.unlock) + return index, op.default_result + in .none? + # do nothing + end + end + + if non_blocking + each_skip_duplicates(ops_locks, &.unlock) + return ops.size, NotReady.new + end + + # Because `channel#close` may clean up a long list, `select_context.try_trigger` may + # be called after the select return. In order to prevent invalid address access, + # the state is allocated in the heap. + shared_state = SelectContextSharedState.new(SelectState::Active) + contexts = ops.map &.create_context_and_wait(shared_state) + + each_skip_duplicates(ops_locks, &.unlock) + Fiber.suspend + + contexts.each_with_index do |context, index| + op = ops[index] + op.lock + op.unwait(context) + op.unlock + end + + contexts.each_with_index do |context, index| + if context.activated? + return index, ops[index].wait_result(context) + end + end + + raise "BUG: Fiber was awaken from select but no action was activated" + end + + private def self.each_skip_duplicates(ops_locks, &) + # Avoid deadlocks from trying to lock the same lock twice. + # `ops_lock` is sorted by `lock_object_id`, so identical onces will be in + # a row and we skip repeats while iterating. + last_lock_id = nil + ops_locks.each do |op| + if op.lock_object_id != last_lock_id + last_lock_id = op.lock_object_id + yield op + end + end + end +end + +require "./select/select_action" +require "./select/timeout_action" diff --git a/src/channel/select/select_action.cr b/src/channel/select/select_action.cr new file mode 100644 index 000000000000..d5439fde5587 --- /dev/null +++ b/src/channel/select/select_action.cr @@ -0,0 +1,45 @@ +class Channel(T) + # :nodoc: + module SelectAction(S) + abstract def execute : DeliveryState + abstract def wait(context : SelectContext(S)) + abstract def wait_result_impl(context : SelectContext(S)) + abstract def unwait_impl(context : SelectContext(S)) + abstract def result : S + abstract def lock_object_id + abstract def lock + abstract def unlock + + def create_context_and_wait(shared_state) + context = SelectContext.new(shared_state, self) + self.wait(context) + context + end + + # wait_result overload allow implementors to define + # wait_result_impl with the right type and Channel.select_impl + # to allow dispatching over unions that will not happen + def wait_result(context : SelectContext) + raise "BUG: Unexpected call to #{typeof(self)}#wait_result(context : #{typeof(context)})" + end + + def wait_result(context : SelectContext(S)) + wait_result_impl(context) + end + + # idem wait_result/wait_result_impl + def unwait(context : SelectContext) + raise "BUG: Unexpected call to #{typeof(self)}#unwait(context : #{typeof(context)})" + end + + def unwait(context : SelectContext(S)) + unwait_impl(context) + end + + # Implementor that returns `Channel::UseDefault` in `#execute` + # must redefine `#default_result` + def default_result + raise "Unreachable" + end + end +end diff --git a/src/channel/select/timeout_action.cr b/src/channel/select/timeout_action.cr new file mode 100644 index 000000000000..9240b480db1a --- /dev/null +++ b/src/channel/select/timeout_action.cr @@ -0,0 +1,66 @@ +# Timeout keyword for use in `select`. +# +# ``` +# select +# when x = ch.receive +# puts "got #{x}" +# when timeout(1.seconds) +# puts "timeout" +# end +# ``` +# +# NOTE: It won't trigger if the `select` has an `else` case (i.e.: a non-blocking select). +def timeout_select_action(timeout : Time::Span) : Channel::TimeoutAction + Channel::TimeoutAction.new(timeout) +end + +class Channel(T) + # :nodoc: + class TimeoutAction + include SelectAction(Nil) + + # Total amount of time to wait + @timeout : Time::Span + @select_context : SelectContext(Nil)? + + def initialize(@timeout : Time::Span) + end + + def execute : DeliveryState + DeliveryState::None + end + + def result : Nil + nil + end + + def wait(context : SelectContext(Nil)) : Nil + @select_context = context + Fiber.timeout(@timeout, self) + end + + def wait_result_impl(context : SelectContext(Nil)) + nil + end + + def unwait_impl(context : SelectContext(Nil)) + Fiber.cancel_timeout + end + + def lock_object_id : UInt64 + self.object_id + end + + def lock + end + + def unlock + end + + def time_expired(fiber : Fiber) : Nil + if @select_context.try &.try_trigger + fiber.enqueue + end + end + end +end From 041e8bd8f718632367bb5a4c46736cd00447916a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 18 Aug 2024 19:37:49 +0200 Subject: [PATCH 038/193] Update version in `shard.yml` (#14909) --- scripts/release-update.sh | 3 +++ scripts/update-changelog.sh | 4 ++++ shard.yml | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/release-update.sh b/scripts/release-update.sh index c9fa180f6578..b6216ce3d6df 100755 --- a/scripts/release-update.sh +++ b/scripts/release-update.sh @@ -16,6 +16,9 @@ minor_branch="${CRYSTAL_VERSION%.*}" next_minor="$((${minor_branch#*.} + 1))" echo "${CRYSTAL_VERSION%%.*}.${next_minor}.0-dev" > src/VERSION +# Update shard.yml +sed -i -E "s/version: .*/version: $(cat src/VERSION)/" shard.yml + # Remove SOURCE_DATE_EPOCH (only used in source tree of a release) rm -f src/SOURCE_DATE_EPOCH diff --git a/scripts/update-changelog.sh b/scripts/update-changelog.sh index 6fe0fa2839f3..763e63670f43 100755 --- a/scripts/update-changelog.sh +++ b/scripts/update-changelog.sh @@ -44,6 +44,10 @@ git switch $branch 2>/dev/null || git switch -c $branch; echo "${VERSION}" > src/VERSION git add src/VERSION +# Update shard.yml +sed -i -E "s/version: .*/version: ${VERSION}/" shard.yml +git add shard.yml + # Write release date into src/SOURCE_DATE_EPOCH release_date=$(head -n1 $current_changelog | grep -o -P '(?<=\()[^)]+') echo "$(date --utc --date="${release_date}" +%s)" > src/SOURCE_DATE_EPOCH diff --git a/shard.yml b/shard.yml index 396d91bdffe2..85b76f49c8d8 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.13.0-dev +version: 1.14.0-dev authors: - Crystal Core Team From f3fb7b6485ccabe4ce53f7f385a66514af6598a7 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 19 Aug 2024 01:38:33 +0800 Subject: [PATCH 039/193] Support ARM64 Windows (#14911) Adds ARM64-specific stack unwinding so that the prelude now compiles for Windows on ARM64. For convenience, the x86-64 and ARM64 Windows bindings share the same directory using a symbolic link. The context switch reuses the existing one for AArch64 Linux, since it seems the ABIs are identical; most `Fiber` and `select` specs work, except ones that deal with closed `Channel`s for some reason. --- spec/std/concurrent/select_spec.cr | 148 ++++++++++++--------- src/exception/call_stack/stackwalk.cr | 14 +- src/lib_c/aarch64-windows-msvc | 1 + src/lib_c/x86_64-windows-msvc/c/dbghelp.cr | 1 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 134 ++++++++++++------- 5 files changed, 181 insertions(+), 117 deletions(-) create mode 120000 src/lib_c/aarch64-windows-msvc diff --git a/spec/std/concurrent/select_spec.cr b/spec/std/concurrent/select_spec.cr index f3f439ddd0b3..5285e3dd070c 100644 --- a/spec/std/concurrent/select_spec.cr +++ b/spec/std/concurrent/select_spec.cr @@ -253,19 +253,23 @@ describe "select" do end end - it "raises if channel was closed" do - ch = Channel(String).new - - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed" do + ch = Channel(String).new + + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "non-blocking raise-on-close single-channel" do @@ -295,20 +299,24 @@ describe "select" do end end - it "raises if channel was closed" do - ch = Channel(String).new + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed" do + ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive - else + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + else + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "blocking raise-on-close multi-channel" do @@ -342,37 +350,41 @@ describe "select" do end end - it "raises if channel was closed (1)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed (1)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end - it "raises if channel was closed (2)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + it "raises if channel was closed (2)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive + spawn_and_check(->{ ch2.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "non-blocking raise-on-close multi-channel" do @@ -422,39 +434,43 @@ describe "select" do end end - it "raises if channel was closed (1)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + {% if flag?(:win32) && flag?(:aarch64) %} + pending "raises if channel was closed" + {% else %} + it "raises if channel was closed (1)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive - else + spawn_and_check(->{ ch.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + else + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end - it "raises if channel was closed (2)" do - ch = Channel(String).new - ch2 = Channel(Bool).new + it "raises if channel was closed (2)" do + ch = Channel(String).new + ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| - begin - select - when m = ch.receive - when m = ch2.receive - else + spawn_and_check(->{ ch2.close }) do |w| + begin + select + when m = ch.receive + when m = ch2.receive + else + end + rescue Channel::ClosedError + w.check end - rescue Channel::ClosedError - w.check end end - end + {% end %} end context "blocking nil-on-close single-channel" do diff --git a/src/exception/call_stack/stackwalk.cr b/src/exception/call_stack/stackwalk.cr index 2b9a03b472c7..b3e2ed8f479c 100644 --- a/src/exception/call_stack/stackwalk.cr +++ b/src/exception/call_stack/stackwalk.cr @@ -93,6 +93,8 @@ struct Exception::CallStack {% elsif flag?(:i386) %} # TODO: use WOW64_CONTEXT in place of CONTEXT {% raise "x86 not supported" %} + {% elsif flag?(:aarch64) %} + LibC::IMAGE_FILE_MACHINE_ARM64 {% else %} {% raise "Architecture not supported" %} {% end %} @@ -102,9 +104,15 @@ struct Exception::CallStack stack_frame.addrFrame.mode = LibC::ADDRESS_MODE::AddrModeFlat stack_frame.addrStack.mode = LibC::ADDRESS_MODE::AddrModeFlat - stack_frame.addrPC.offset = context.value.rip - stack_frame.addrFrame.offset = context.value.rbp - stack_frame.addrStack.offset = context.value.rsp + {% if flag?(:x86_64) %} + stack_frame.addrPC.offset = context.value.rip + stack_frame.addrFrame.offset = context.value.rbp + stack_frame.addrStack.offset = context.value.rsp + {% elsif flag?(:aarch64) %} + stack_frame.addrPC.offset = context.value.pc + stack_frame.addrFrame.offset = context.value.x[29] + stack_frame.addrStack.offset = context.value.sp + {% end %} last_frame = nil cur_proc = LibC.GetCurrentProcess diff --git a/src/lib_c/aarch64-windows-msvc b/src/lib_c/aarch64-windows-msvc new file mode 120000 index 000000000000..072348f65d09 --- /dev/null +++ b/src/lib_c/aarch64-windows-msvc @@ -0,0 +1 @@ +x86_64-windows-msvc \ No newline at end of file diff --git a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr index af37cb0c7f0c..2c62d07d3ad8 100644 --- a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr +++ b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr @@ -122,6 +122,7 @@ lib LibC end IMAGE_FILE_MACHINE_AMD64 = DWORD.new!(0x8664) + IMAGE_FILE_MACHINE_ARM64 = DWORD.new!(0xAA64) alias PREAD_PROCESS_MEMORY_ROUTINE64 = HANDLE, DWORD64, Void*, DWORD, DWORD* -> BOOL alias PFUNCTION_TABLE_ACCESS_ROUTINE64 = HANDLE, DWORD64 -> Void* diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index e1f133dcae48..535ad835c87a 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -140,54 +140,84 @@ lib LibC JOB_OBJECT_MSG_EXIT_PROCESS = 7 JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS = 8 - struct CONTEXT - p1Home : DWORD64 - p2Home : DWORD64 - p3Home : DWORD64 - p4Home : DWORD64 - p5Home : DWORD64 - p6Home : DWORD64 - contextFlags : DWORD - mxCsr : DWORD - segCs : WORD - segDs : WORD - segEs : WORD - segFs : WORD - segGs : WORD - segSs : WORD - eFlags : DWORD - dr0 : DWORD64 - dr1 : DWORD64 - dr2 : DWORD64 - dr3 : DWORD64 - dr6 : DWORD64 - dr7 : DWORD64 - rax : DWORD64 - rcx : DWORD64 - rdx : DWORD64 - rbx : DWORD64 - rsp : DWORD64 - rbp : DWORD64 - rsi : DWORD64 - rdi : DWORD64 - r8 : DWORD64 - r9 : DWORD64 - r10 : DWORD64 - r11 : DWORD64 - r12 : DWORD64 - r13 : DWORD64 - r14 : DWORD64 - r15 : DWORD64 - rip : DWORD64 - fltSave : UInt8[512] # DUMMYUNIONNAME - vectorRegister : UInt8[16][26] # M128A[26] - vectorControl : DWORD64 - debugControl : DWORD64 - lastBranchToRip : DWORD64 - lastBranchFromRip : DWORD64 - lastExceptionToRip : DWORD64 - lastExceptionFromRip : DWORD64 - end + {% if flag?(:x86_64) %} + struct CONTEXT + p1Home : DWORD64 + p2Home : DWORD64 + p3Home : DWORD64 + p4Home : DWORD64 + p5Home : DWORD64 + p6Home : DWORD64 + contextFlags : DWORD + mxCsr : DWORD + segCs : WORD + segDs : WORD + segEs : WORD + segFs : WORD + segGs : WORD + segSs : WORD + eFlags : DWORD + dr0 : DWORD64 + dr1 : DWORD64 + dr2 : DWORD64 + dr3 : DWORD64 + dr6 : DWORD64 + dr7 : DWORD64 + rax : DWORD64 + rcx : DWORD64 + rdx : DWORD64 + rbx : DWORD64 + rsp : DWORD64 + rbp : DWORD64 + rsi : DWORD64 + rdi : DWORD64 + r8 : DWORD64 + r9 : DWORD64 + r10 : DWORD64 + r11 : DWORD64 + r12 : DWORD64 + r13 : DWORD64 + r14 : DWORD64 + r15 : DWORD64 + rip : DWORD64 + fltSave : UInt8[512] # DUMMYUNIONNAME + vectorRegister : UInt8[16][26] # M128A[26] + vectorControl : DWORD64 + debugControl : DWORD64 + lastBranchToRip : DWORD64 + lastBranchFromRip : DWORD64 + lastExceptionToRip : DWORD64 + lastExceptionFromRip : DWORD64 + end + {% elsif flag?(:aarch64) %} + struct ARM64_NT_NEON128_DUMMYSTRUCTNAME + low : ULongLong + high : LongLong + end + + union ARM64_NT_NEON128 + dummystructname : ARM64_NT_NEON128_DUMMYSTRUCTNAME + d : Double[2] + s : Float[4] + h : WORD[8] + b : BYTE[16] + end + + struct CONTEXT + contextFlags : DWORD + cpsr : DWORD + x : DWORD64[31] # x29 = fp, x30 = lr + sp : DWORD64 + pc : DWORD64 + v : ARM64_NT_NEON128[32] + fpcr : DWORD + fpsr : DWORD + bcr : DWORD[8] + bvr : DWORD64[8] + wcr : DWORD[8] + wvr : DWORD64[8] + end + {% end %} {% if flag?(:x86_64) %} CONTEXT_AMD64 = DWORD.new!(0x00100000) @@ -211,6 +241,14 @@ lib LibC CONTEXT_EXTENDED_REGISTERS = CONTEXT_i386 | 0x00000020 CONTEXT_FULL = CONTEXT_CONTROL | CONTEXT_INTEGER | CONTEXT_SEGMENTS + {% elsif flag?(:aarch64) %} + CONTEXT_ARM64 = DWORD.new!(0x00400000) + + CONTEXT_ARM64_CONTROL = CONTEXT_ARM64 | 0x1 + CONTEXT_ARM64_INTEGER = CONTEXT_ARM64 | 0x2 + CONTEXT_ARM64_FLOATING_POINT = CONTEXT_ARM64 | 0x4 + + CONTEXT_FULL = CONTEXT_ARM64_CONTROL | CONTEXT_ARM64_INTEGER | CONTEXT_ARM64_FLOATING_POINT {% end %} fun RtlCaptureContext(contextRecord : CONTEXT*) From 95a761e8d18752ac474f4dc4b0b47fd713352110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Aug 2024 11:13:23 +0200 Subject: [PATCH 040/193] Revert "Fix: Don't link to undocumented types in API docs" (#14908) This reverts commit 78c9282c704ca1d1ce83cefb0154ad24e7371d28. --- src/compiler/crystal/tools/doc/html/type.html | 10 +++++----- src/compiler/crystal/tools/doc/templates.cr | 6 ------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/compiler/crystal/tools/doc/html/type.html b/src/compiler/crystal/tools/doc/html/type.html index e9918c6fe429..10c7e51fedd3 100644 --- a/src/compiler/crystal/tools/doc/html/type.html +++ b/src/compiler/crystal/tools/doc/html/type.html @@ -45,10 +45,10 @@

<%= type.formatted_alias_definition %> <% end %> -<%= OtherTypesTemplate.new("Included Modules", type, included_modules_with_docs) %> -<%= OtherTypesTemplate.new("Extended Modules", type, extended_modules_with_docs) %> -<%= OtherTypesTemplate.new("Direct Known Subclasses", type, subclasses_with_docs) %> -<%= OtherTypesTemplate.new("Direct including types", type, including_types_with_docs) %> +<%= OtherTypesTemplate.new("Included Modules", type, type.included_modules) %> +<%= OtherTypesTemplate.new("Extended Modules", type, type.extended_modules) %> +<%= OtherTypesTemplate.new("Direct Known Subclasses", type, type.subclasses) %> +<%= OtherTypesTemplate.new("Direct including types", type, type.including_types) %> <% if locations = type.locations %>

@@ -99,7 +99,7 @@

<%= MethodSummaryTemplate.new("Instance Method Summary", type.instance_methods) %>
- <% ancestors_with_docs.each do |ancestor| %> + <% type.ancestors.each do |ancestor| %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.instance_methods, "Instance") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.constructors, "Constructor") %> <%= MethodsInheritedTemplate.new(type, ancestor, ancestor.class_methods, "Class") %> diff --git a/src/compiler/crystal/tools/doc/templates.cr b/src/compiler/crystal/tools/doc/templates.cr index 4aaf5ac9029e..91ad32e1d0d1 100644 --- a/src/compiler/crystal/tools/doc/templates.cr +++ b/src/compiler/crystal/tools/doc/templates.cr @@ -30,12 +30,6 @@ module Crystal::Doc end record TypeTemplate, type : Type, types : Array(Type), project_info : ProjectInfo do - {% for method in %w[ancestors included_modules extended_modules subclasses including_types] %} - def {{method.id}}_with_docs - type.{{method.id}}.select!(&.in?(types)) - end - {% end %} - ECR.def_to_s "#{__DIR__}/html/type.html" end From 12372105ae21861c28dbaad77f7c9312dd296e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Aug 2024 16:36:58 +0200 Subject: [PATCH 041/193] CI: Refactor SSL workflow with job matrix (#14899) --- .github/workflows/openssl.yml | 107 ++++++++-------------------------- 1 file changed, 23 insertions(+), 84 deletions(-) diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 8cdb888e3621..d518c93a51de 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -7,95 +7,34 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: - openssl3_0: + libssl_test: runs-on: ubuntu-latest - name: "OpenSSL 3.0" + name: "${{ matrix.pkg }}" container: crystallang/crystal:1.13.1-alpine + strategy: + fail-fast: false + matrix: + include: + - pkg: "openssl1.1-compat-dev=~1.1.1" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.18/community + - pkg: "openssl-dev=~3.0" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.17/main + - pkg: "openssl-dev=~3.3" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.20/main + - pkg: "libressl-dev=~3.4" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.15/community + - pkg: "libressl-dev=~3.5" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.16/community + - pkg: "libressl-dev=~3.8" + repository: http://dl-cdn.alpinelinux.org/alpine/v3.20/community steps: - name: Download Crystal source uses: actions/checkout@v4 - - name: Uninstall openssl - run: apk del openssl-dev libxml2-static - - name: Upgrade alpine-keys - run: apk upgrade alpine-keys - - name: Install openssl 3.0 - run: apk add "openssl-dev=~3.0" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.17/main - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - openssl3_3: - runs-on: ubuntu-latest - name: "OpenSSL 3.3" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - name: Install openssl 3.3 - run: apk add "openssl-dev=~3.3" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - openssl111: - runs-on: ubuntu-latest - name: "OpenSSL 1.1.1" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - name: Uninstall openssl - run: apk del openssl-dev - - name: Install openssl 1.1.1 - run: apk add "openssl1.1-compat-dev=~1.1.1" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - libressl34: - runs-on: ubuntu-latest - name: "LibreSSL 3.4" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - name: Uninstall openssl - run: apk del openssl-dev openssl-libs-static - - name: Upgrade alpine-keys - run: apk upgrade alpine-keys - - name: Install libressl 3.4 - run: apk add "libressl-dev=~3.4" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.15/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - libressl35: - runs-on: ubuntu-latest - name: "LibreSSL 3.5" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v2 - - name: Uninstall openssl - run: apk del openssl-dev openssl-libs-static - - name: Install libressl 3.5 - run: apk add "libressl-dev=~3.5" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.16/community - - name: Check LibSSL version - run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - - name: Run OpenSSL specs - run: bin/crystal spec --order=random spec/std/openssl/ - libressl38: - runs-on: ubuntu-latest - name: "LibreSSL 3.5" - container: crystallang/crystal:1.13.1-alpine - steps: - - name: Download Crystal source - uses: actions/checkout@v2 - - name: Uninstall openssl - run: apk del openssl-dev openssl-libs-static - - name: Install libressl 3.8 - run: apk add "libressl-dev=~3.8" --repository=http://dl-cdn.alpinelinux.org/alpine/v3.20/community - - name: Check LibSSL version + - name: Uninstall openssl and conflicts + run: apk del openssl-dev openssl-libs-static libxml2-static + - name: Install ${{ matrix.pkg }} + run: apk add "${{ matrix.pkg }}" --repository=${{ matrix.repository }} + - name: Print LibSSL version run: bin/crystal eval 'require "openssl"; p! LibSSL::OPENSSL_VERSION, LibSSL::LIBRESSL_VERSION' - name: Run OpenSSL specs run: bin/crystal spec --order=random spec/std/openssl/ From 2fcb168588820b785a3e17467c93d2b709c53df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Aug 2024 16:38:13 +0200 Subject: [PATCH 042/193] Add `Slice#same?` (#14728) --- spec/std/slice_spec.cr | 14 ++++++++++++++ src/slice.cr | 15 +++++++++++++++ src/spec/expectations.cr | 12 ++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/spec/std/slice_spec.cr b/spec/std/slice_spec.cr index 1b21a4489bbd..7624b34c852c 100644 --- a/spec/std/slice_spec.cr +++ b/spec/std/slice_spec.cr @@ -503,6 +503,20 @@ describe "Slice" do end end + it "#same?" do + slice = Slice[1, 2, 3] + + slice.should be slice + slice.should_not be slice.dup + slice.should_not be Slice[1, 2, 3] + + (slice + 1).should be slice + 1 + slice.should_not be slice + 1 + + (slice[0, 2]).should be slice[0, 2] + slice.should_not be slice[0, 2] + end + it "does macro []" do slice = Slice[1, 'a', "foo"] slice.should be_a(Slice(Int32 | Char | String)) diff --git a/src/slice.cr b/src/slice.cr index d843ceb17c63..c87816f315d9 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -859,6 +859,21 @@ struct Slice(T) {% end %} end + # Returns `true` if `self` and *other* point to the same memory, i.e. pointer + # and size are identical. + # + # ``` + # slice = Slice[1, 2, 3] + # slice.same?(slice) # => true + # slice == Slice[1, 2, 3] # => false + # slice.same?(slice + 1) # => false + # (slice + 1).same?(slice + 1) # => true + # slice.same?(slice[0, 2]) # => false + # ``` + def same?(other : self) : Bool + to_unsafe == other.to_unsafe && size == other.size + end + def to_slice : self self end diff --git a/src/spec/expectations.cr b/src/spec/expectations.cr index ac93de54975e..193f86d0de21 100644 --- a/src/spec/expectations.cr +++ b/src/spec/expectations.cr @@ -65,11 +65,19 @@ module Spec end def failure_message(actual_value) - "Expected: #{@expected_value.pretty_inspect} (object_id: #{@expected_value.object_id})\n got: #{actual_value.pretty_inspect} (object_id: #{actual_value.object_id})" + "Expected: #{@expected_value.pretty_inspect} (#{identify(@expected_value)})\n got: #{actual_value.pretty_inspect} (#{identify(actual_value)})" end def negative_failure_message(actual_value) - "Expected: value.same? #{@expected_value.pretty_inspect} (object_id: #{@expected_value.object_id})\n got: #{actual_value.pretty_inspect} (object_id: #{actual_value.object_id})" + "Expected: #{@expected_value.pretty_inspect} (#{identify(@expected_value)})\n got: #{actual_value.pretty_inspect} (#{identify(actual_value)})" + end + + private def identify(value) + if value.responds_to?(:object_id) + "object_id: #{value.object_id}" + else + value.to_unsafe + end end end From a9e04578a10c9dc351cadd1b960a8bde4eeb24d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Gw=C3=B3=C5=BAd=C5=BA?= Date: Tue, 20 Aug 2024 01:07:13 +0200 Subject: [PATCH 043/193] Add `HashLiteral#has_key?` and `NamedTupleLiteral#has_key?` (#14890) --- spec/compiler/macro/macro_methods_spec.cr | 14 ++++++++++++ src/compiler/crystal/macros.cr | 8 +++++++ src/compiler/crystal/macros/methods.cr | 27 ++++++++++++++--------- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 385e165a3504..9d425cb7e162 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -1090,6 +1090,12 @@ module Crystal assert_macro %({{ {'z' => 6, 'a' => 9}.of_value }}), %() end + it "executes has_key?" do + assert_macro %({{ {'z' => 6, 'a' => 9}.has_key?('z') }}), %(true) + assert_macro %({{ {'z' => 6, 'a' => 9}.has_key?('x') }}), %(false) + assert_macro %({{ {'z' => nil, 'a' => 9}.has_key?('z') }}), %(true) + end + it "executes type" do assert_macro %({{ x.type }}), %(Headers), {x: HashLiteral.new([] of HashLiteral::Entry, name: Path.new("Headers"))} end @@ -1195,6 +1201,14 @@ module Crystal assert_macro %({% a = {a: 1}; a["a"] = 2 %}{{a["a"]}}), "2" end + it "executes has_key?" do + assert_macro %({{{a: 1}.has_key?("a")}}), "true" + assert_macro %({{{a: 1}.has_key?(:a)}}), "true" + assert_macro %({{{a: nil}.has_key?("a")}}), "true" + assert_macro %({{{a: nil}.has_key?("b")}}), "false" + assert_macro_error %({{{a: 1}.has_key?(true)}}), "expected 'NamedTupleLiteral#has_key?' first argument to be a SymbolLiteral, StringLiteral or MacroId, not BoolLiteral" + end + it "creates a named tuple literal with a var" do assert_macro %({% a = {a: x} %}{{a[:a]}}), "1", {x: 1.int32} end diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index ff422ce553a2..a2ea0aeb85fe 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -800,6 +800,10 @@ module Crystal::Macros def []=(key : ASTNode, value : ASTNode) : ASTNode end + # Similar to `Hash#has_hey?` + def has_key?(key : ASTNode) : BoolLiteral + end + # Returns the type specified at the end of the Hash literal, if any. # # This refers to the key type after brackets in `{} of String => Int32`. @@ -874,6 +878,10 @@ module Crystal::Macros # Adds or replaces a key. def []=(key : SymbolLiteral | StringLiteral | MacroId, value : ASTNode) : ASTNode end + + # Similar to `NamedTuple#has_key?` + def has_key?(key : SymbolLiteral | StringLiteral | MacroId) : ASTNode + end end # A range literal. diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 8a7aa569fa95..3a81015f0ffd 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -965,6 +965,10 @@ module Crystal interpret_check_args { @of.try(&.key) || Nop.new } when "of_value" interpret_check_args { @of.try(&.value) || Nop.new } + when "has_key?" + interpret_check_args do |key| + BoolLiteral.new(entries.any? &.key.==(key)) + end when "type" interpret_check_args { @name || Nop.new } when "clear" @@ -1042,11 +1046,7 @@ module Crystal when "[]" interpret_check_args do |key| case key - when SymbolLiteral - key = key.value - when MacroId - key = key.value - when StringLiteral + when SymbolLiteral, MacroId, StringLiteral key = key.value else raise "argument to [] must be a symbol or string, not #{key.class_desc}:\n\n#{key}" @@ -1058,11 +1058,7 @@ module Crystal when "[]=" interpret_check_args do |key, value| case key - when SymbolLiteral - key = key.value - when MacroId - key = key.value - when StringLiteral + when SymbolLiteral, MacroId, StringLiteral key = key.value else raise "expected 'NamedTupleLiteral#[]=' first argument to be a SymbolLiteral or MacroId, not #{key.class_desc}" @@ -1077,6 +1073,17 @@ module Crystal value end + when "has_key?" + interpret_check_args do |key| + case key + when SymbolLiteral, MacroId, StringLiteral + key = key.value + else + raise "expected 'NamedTupleLiteral#has_key?' first argument to be a SymbolLiteral, StringLiteral or MacroId, not #{key.class_desc}" + end + + BoolLiteral.new(entries.any? &.key.==(key)) + end else super end From 74279908e6e5cf848fe3138df8a9d0e1a326a243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 20 Aug 2024 01:07:21 +0200 Subject: [PATCH 044/193] Add `Pointer::Appender#to_slice` (#14874) --- spec/std/pointer/appender_spec.cr | 14 ++++++++++++++ src/base64.cr | 2 +- src/crystal/system/print_error.cr | 4 ++-- src/crystal/system/win32/file_descriptor.cr | 2 +- src/pointer.cr | 14 ++++++++++++++ 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/spec/std/pointer/appender_spec.cr b/spec/std/pointer/appender_spec.cr index 02ca18e0188e..54aff72c9349 100644 --- a/spec/std/pointer/appender_spec.cr +++ b/spec/std/pointer/appender_spec.cr @@ -25,4 +25,18 @@ describe Pointer::Appender do end appender.size.should eq 4 end + + it "#to_slice" do + data = Slice(Int32).new(5) + appender = data.to_unsafe.appender + appender.to_slice.should eq Slice(Int32).new(0) + appender.to_slice.to_unsafe.should eq data.to_unsafe + + 4.times do |i| + appender << (i + 1) * 2 + appender.to_slice.should eq data[0, i + 1] + end + appender.to_slice.should eq Slice[2, 4, 6, 8] + appender.to_slice.to_unsafe.should eq data.to_unsafe + end end diff --git a/src/base64.cr b/src/base64.cr index 241d00c57bda..951684afc7ef 100644 --- a/src/base64.cr +++ b/src/base64.cr @@ -163,7 +163,7 @@ module Base64 buf = Pointer(UInt8).malloc(decode_size(slice.size)) appender = buf.appender from_base64(slice) { |byte| appender << byte } - Slice.new(buf, appender.size.to_i32) + appender.to_slice end # Writes the base64-decoded version of *data* to *io*. diff --git a/src/crystal/system/print_error.cr b/src/crystal/system/print_error.cr index 796579bf256a..b55e05e51ec6 100644 --- a/src/crystal/system/print_error.cr +++ b/src/crystal/system/print_error.cr @@ -23,7 +23,7 @@ module Crystal::System String.each_utf16_char(bytes) do |char| if appender.size > utf8.size - char.bytesize # buffer is full (char won't fit) - print_error utf8.to_slice[0...appender.size] + print_error appender.to_slice appender = utf8.to_unsafe.appender end @@ -33,7 +33,7 @@ module Crystal::System end if appender.size > 0 - print_error utf8.to_slice[0...appender.size] + print_error appender.to_slice end end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index d19e43b79547..7899f75407f7 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -438,7 +438,7 @@ private module ConsoleUtils appender << byte end end - @@buffer = @@utf8_buffer[0, appender.size] + @@buffer = appender.to_slice end private def self.read_console(handle : LibC::HANDLE, slice : Slice(UInt16)) : Int32 diff --git a/src/pointer.cr b/src/pointer.cr index 06565298d376..87da18b25fa5 100644 --- a/src/pointer.cr +++ b/src/pointer.cr @@ -52,6 +52,20 @@ struct Pointer(T) def pointer @pointer end + + # Creates a slice pointing at the values appended by this instance. + # + # ``` + # slice = Slice(Int32).new(5) + # appender = slice.to_unsafe.appender + # appender << 1 + # appender << 2 + # appender << 3 + # appender.to_slice # => Slice[1, 2, 3] + # ``` + def to_slice : Slice(T) + @start.to_slice(size) + end end include Comparable(self) From ee894e0e6db90d308a1cc72c96fd99d237ef8a4a Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Mon, 19 Aug 2024 19:08:02 -0400 Subject: [PATCH 045/193] Add documentation for `NoReturn` and `Void` (#14817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller Co-authored-by: Sijawusz Pur Rahnama --- src/compiler/crystal/tools/doc/generator.cr | 2 ++ src/compiler/crystal/tools/doc/type.cr | 19 +++++++++++-- src/docs_pseudo_methods.cr | 30 +++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 635a6be65731..a2f4db47dee0 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -217,6 +217,8 @@ class Crystal::Doc::Generator def crystal_builtin?(type) return false unless project_info.crystal_stdlib? + # TODO: Enabling this allows links to `NoReturn` to work, but has two `NoReturn`s show up in the sidebar + # return true if type.is_a?(NamedType) && {"NoReturn", "Void"}.includes?(type.name) return false unless type.is_a?(Const) || type.is_a?(NonGenericModuleType) crystal_type = @program.types["Crystal"] diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 9a40bd23e189..624c8f017fe7 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -3,6 +3,13 @@ require "./item" class Crystal::Doc::Type include Item + PSEUDO_CLASS_PREFIX = "CRYSTAL_PSEUDO__" + PSEUDO_CLASS_NOTE = <<-DOC + + NOTE: This is a pseudo-class provided directly by the Crystal compiler. + It cannot be reopened nor overridden. + DOC + getter type : Crystal::Type def initialize(@generator : Generator, type : Crystal::Type) @@ -39,7 +46,11 @@ class Crystal::Doc::Type when Program "Top Level Namespace" when NamedType - type.name + if @generator.project_info.crystal_stdlib? + type.name.lchop(PSEUDO_CLASS_PREFIX) + else + type.name + end when NoReturnType "NoReturn" when VoidType @@ -403,7 +414,11 @@ class Crystal::Doc::Type end def doc - @type.doc + if (t = type).is_a?(NamedType) && t.name.starts_with?(PSEUDO_CLASS_PREFIX) + "#{@type.doc}#{PSEUDO_CLASS_NOTE}" + else + @type.doc + end end def lookup_path(path_or_names : Path | Array(String)) diff --git a/src/docs_pseudo_methods.cr b/src/docs_pseudo_methods.cr index d4f1fb832263..36eb1f09eaff 100644 --- a/src/docs_pseudo_methods.cr +++ b/src/docs_pseudo_methods.cr @@ -200,3 +200,33 @@ class Object def __crystal_pseudo_responds_to?(name : Symbol) : Bool end end + +# Some expressions won't return to the current scope and therefore have no return type. +# This is expressed as the special return type `NoReturn`. +# +# Typical examples for non-returning methods and keywords are `return`, `exit`, `raise`, `next`, and `break`. +# +# This is for example useful for deconstructing union types: +# +# ``` +# string = STDIN.gets +# typeof(string) # => String? +# typeof(raise "Empty input") # => NoReturn +# typeof(string || raise "Empty input") # => String +# ``` +# +# The compiler recognizes that in case string is Nil, the right hand side of the expression `string || raise` will be evaluated. +# Since `typeof(raise "Empty input")` is `NoReturn` the execution would not return to the current scope in that case. +# That leaves only `String` as resulting type of the expression. +# +# Every expression whose code paths all result in `NoReturn` will be `NoReturn` as well. +# `NoReturn` does not show up in a union type because it would essentially be included in every expression's type. +# It is only used when an expression will never return to the current scope. +# +# `NoReturn` can be explicitly set as return type of a method or function definition but will usually be inferred by the compiler. +struct CRYSTAL_PSEUDO__NoReturn +end + +# Similar in usage to `Nil`. `Void` is prefered for C lib bindings. +struct CRYSTAL_PSEUDO__Void +end From 827c59adba86135c72c7a5555979ae85314a57f7 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 6 Aug 2024 15:50:50 +0800 Subject: [PATCH 046/193] Fix misaligned stack access in the interpreter (#14843) --- src/compiler/crystal/interpreter/interpreter.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index eca73ecae6bc..aa90d83f413f 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -999,14 +999,15 @@ class Crystal::Repl::Interpreter private macro stack_pop(t) %aligned_size = align(sizeof({{t}})) - %value = (stack - %aligned_size).as({{t}}*).value + %value = uninitialized {{t}} + (stack - %aligned_size).copy_to(pointerof(%value).as(UInt8*), sizeof({{t}})) stack_shrink_by(%aligned_size) %value end private macro stack_push(value) %temp = {{value}} - stack.as(Pointer(typeof({{value}}))).value = %temp + stack.copy_from(pointerof(%temp).as(UInt8*), sizeof(typeof({{value}}))) %size = sizeof(typeof({{value}})) %aligned_size = align(%size) From a60b0766d26ee1f6005c671369432d57ec843a9c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 7 Aug 2024 04:48:06 +0800 Subject: [PATCH 047/193] Fix `ReferenceStorage(T)` atomic if `T` has no inner pointers (#14845) It turns out the fix in #14730 made all `ReferenceStorage` objects non-atomic; `Crystal::ReferenceStorageType#reference_type` returns a reference type, whose `#has_inner_pointers?` always returns true since the reference itself is a pointer. This PR fixes that again by adding a special case for `ReferenceStorage`. --- src/compiler/crystal/codegen/types.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/codegen/types.cr b/src/compiler/crystal/codegen/types.cr index 470fe7424dcd..7ce1640bb5e7 100644 --- a/src/compiler/crystal/codegen/types.cr +++ b/src/compiler/crystal/codegen/types.cr @@ -70,7 +70,7 @@ module Crystal when NamedTupleInstanceType self.entries.any? &.type.has_inner_pointers? when ReferenceStorageType - self.reference_type.has_inner_pointers? + self.reference_type.all_instance_vars.each_value.any? &.type.has_inner_pointers? when PrimitiveType false when EnumType From d63d459d24228a9f916b7dfb584d15689b75a05c Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 9 Aug 2024 05:00:09 -0400 Subject: [PATCH 048/193] Hide `Hash::Entry` from public API docs (#14881) --- src/hash.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hash.cr b/src/hash.cr index 8d48e1cd8c08..96b87c7d3e22 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -2148,6 +2148,7 @@ class Hash(K, V) hash end + # :nodoc: struct Entry(K, V) getter key, value, hash From 77314b0c5fd5e2d2e1f7e07415834acdb795b553 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 16 Aug 2024 03:48:53 +0800 Subject: [PATCH 049/193] Fix misaligned store in `Bool` to union upcasts (#14906) The code path for `Nil` looks similar, but it is perfectly fine: it directly stores `[8 x i64] zeroinitializer` to the data field, whose default alignment naturally matches. --- spec/compiler/codegen/union_type_spec.cr | 19 +++++++++++++++++++ src/compiler/crystal/codegen/unions.cr | 9 ++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/spec/compiler/codegen/union_type_spec.cr b/spec/compiler/codegen/union_type_spec.cr index eb561a92dbdd..8ea7d058bff9 100644 --- a/spec/compiler/codegen/union_type_spec.cr +++ b/spec/compiler/codegen/union_type_spec.cr @@ -215,4 +215,23 @@ describe "Code gen: union type" do Union(Nil, Int32).foo )).to_string.should eq("TupleLiteral") end + + it "respects union payload alignment when upcasting Bool (#14898)" do + mod = codegen(<<-CRYSTAL) + x = uninitialized Bool | UInt8[64] + x = true + CRYSTAL + + str = mod.to_s + {% if LibLLVM::IS_LT_150 %} + str.should contain("store i512 1, i512* %2, align 8") + {% else %} + str.should contain("store i512 1, ptr %1, align 8") + {% end %} + + # an i512 store defaults to 16-byte alignment, which is undefined behavior + # as it overestimates the actual alignment of `x`'s data field (x86 in + # particular segfaults on misaligned 16-byte stores) + str.should_not contain("align 16") + end end diff --git a/src/compiler/crystal/codegen/unions.cr b/src/compiler/crystal/codegen/unions.cr index b2b63a17c5ab..fdf1d81a4c95 100644 --- a/src/compiler/crystal/codegen/unions.cr +++ b/src/compiler/crystal/codegen/unions.cr @@ -81,16 +81,19 @@ module Crystal def store_bool_in_union(target_type, union_pointer, value) struct_type = llvm_type(target_type) + union_value_type = struct_type.struct_element_types[1] store type_id(value, @program.bool), union_type_id(struct_type, union_pointer) # To store a boolean in a union - # we sign-extend it to the size in bits of the union - union_size = @llvm_typer.size_of(struct_type.struct_element_types[1]) + # we zero-extend it to the size in bits of the union + union_size = @llvm_typer.size_of(union_value_type) int_type = llvm_context.int((union_size * 8).to_i32) bool_as_extended_int = builder.zext(value, int_type) casted_value_ptr = pointer_cast(union_value(struct_type, union_pointer), int_type.pointer) - store bool_as_extended_int, casted_value_ptr + inst = store bool_as_extended_int, casted_value_ptr + set_alignment(inst, @llvm_typer.align_of(union_value_type)) + inst end def store_nil_in_union(target_type, union_pointer) From 879ec124747c287605e349183a3c9143174659e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 20 Aug 2024 15:49:27 +0200 Subject: [PATCH 050/193] Changelog for 1.13.2 (#14914) --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 2 +- src/VERSION | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 382f76969ec0..f97d0bedeb1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## [1.13.2] (2024-08-20) + +[1.13.2]: https://github.com/crystal-lang/crystal/releases/1.13.2 + +### Bugfixes + +#### stdlib + +- *(collection)* Fix explicitly clear deleted `Hash::Entry` ([#14862], thanks @HertzDevil) + +[#14862]: https://github.com/crystal-lang/crystal/pull/14862 + +#### compiler + +- *(codegen)* Fix `ReferenceStorage(T)` atomic if `T` has no inner pointers ([#14845], thanks @HertzDevil) +- *(codegen)* Fix misaligned store in `Bool` to union upcasts ([#14906], thanks @HertzDevil) +- *(interpreter)* Fix misaligned stack access in the interpreter ([#14843], thanks @HertzDevil) + +[#14845]: https://github.com/crystal-lang/crystal/pull/14845 +[#14906]: https://github.com/crystal-lang/crystal/pull/14906 +[#14843]: https://github.com/crystal-lang/crystal/pull/14843 + +### Infrastructure + +- Changelog for 1.13.2 ([#14914], thanks @straight-shoota) + +[#14914]: https://github.com/crystal-lang/crystal/pull/14914 + ## [1.13.1] (2024-07-12) [1.13.1]: https://github.com/crystal-lang/crystal/releases/1.13.1 diff --git a/shard.yml b/shard.yml index 396d91bdffe2..0dd8c2abf3a1 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.13.0-dev +version: 1.13.2 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH index efabb39ec223..0ea6bd82d669 100644 --- a/src/SOURCE_DATE_EPOCH +++ b/src/SOURCE_DATE_EPOCH @@ -1 +1 @@ -1720742400 +1724112000 diff --git a/src/VERSION b/src/VERSION index b50dd27dd92e..61ce01b30118 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.13.1 +1.13.2 From 41f75ca500b9a3c4b2767c9c993e720bd1e64a37 Mon Sep 17 00:00:00 2001 From: Margret Riegert Date: Tue, 20 Aug 2024 10:36:45 -0400 Subject: [PATCH 051/193] Add `URI.from_json_object_key?` and `URI#to_json_object_key` (#14834) --- spec/std/uri/json_spec.cr | 14 ++++++++++++++ src/uri/json.cr | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 spec/std/uri/json_spec.cr diff --git a/spec/std/uri/json_spec.cr b/spec/std/uri/json_spec.cr new file mode 100644 index 000000000000..a21f503958a5 --- /dev/null +++ b/spec/std/uri/json_spec.cr @@ -0,0 +1,14 @@ +require "spec" +require "uri/json" + +describe "URI" do + describe "serializes" do + it "#to_json" do + URI.parse("https://example.com").to_json.should eq %q("https://example.com") + end + + it "from_json_object_key?" do + URI.from_json_object_key?("https://example.com").should eq(URI.parse("https://example.com")) + end + end +end diff --git a/src/uri/json.cr b/src/uri/json.cr index 9767c9e98a02..00b58f419be5 100644 --- a/src/uri/json.cr +++ b/src/uri/json.cr @@ -25,4 +25,18 @@ class URI def to_json(builder : JSON::Builder) builder.string self end + + # Deserializes the given JSON *key* into a `URI` + # + # NOTE: `require "uri/json"` is required to opt-in to this feature. + def self.from_json_object_key?(key : String) : URI? + parse key + rescue URI::Error + nil + end + + # :nodoc: + def to_json_object_key : String + to_s + end end From 1baf3a726f76adda630d5ee4f384dd00e319a2de Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Tue, 20 Aug 2024 09:38:58 -0500 Subject: [PATCH 052/193] Add `WaitGroup.wait` and `WaitGroup#spawn` (#14837) This commit allows for usage of WaitGroup in a way that is significantly more readable. WaitGroup.wait do |wg| wg.spawn { http.get "/foo" } wg.spawn { http.get "/bar" } end --- spec/std/wait_group_spec.cr | 13 +++++++++++++ src/wait_group.cr | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/spec/std/wait_group_spec.cr b/spec/std/wait_group_spec.cr index 459af8d5c898..6c2f46daa562 100644 --- a/spec/std/wait_group_spec.cr +++ b/spec/std/wait_group_spec.cr @@ -160,6 +160,19 @@ describe WaitGroup do extra.get.should eq(32) end + it "takes a block to WaitGroup.wait" do + fiber_count = 10 + completed = Array.new(fiber_count) { false } + + WaitGroup.wait do |wg| + fiber_count.times do |i| + wg.spawn { completed[i] = true } + end + end + + completed.should eq [true] * 10 + end + # the test takes far too much time for the interpreter to complete {% unless flag?(:interpreted) %} it "stress add/done/wait" do diff --git a/src/wait_group.cr b/src/wait_group.cr index 2fd49c593b56..89510714c727 100644 --- a/src/wait_group.cr +++ b/src/wait_group.cr @@ -42,12 +42,46 @@ class WaitGroup end end + # Yields a `WaitGroup` instance and waits at the end of the block for all of + # the work enqueued inside it to complete. + # + # ``` + # WaitGroup.wait do |wg| + # items.each do |item| + # wg.spawn { process item } + # end + # end + # ``` + def self.wait : Nil + instance = new + yield instance + instance.wait + end + def initialize(n : Int32 = 0) @waiting = Crystal::PointerLinkedList(Waiting).new @lock = Crystal::SpinLock.new @counter = Atomic(Int32).new(n) end + # Increment the counter by 1, perform the work inside the block in a separate + # fiber, decrementing the counter after it completes or raises. Returns the + # `Fiber` that was spawned. + # + # ``` + # wg = WaitGroup.new + # wg.spawn { do_something } + # wg.wait + # ``` + def spawn(&block) : Fiber + add + ::spawn do + block.call + ensure + done + end + end + # Increments the counter by how many fibers we want to wait for. # # A negative value decrements the counter. When the counter reaches zero, From bc569acfbd6faaeb45823e468d3713c52f6c51df Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 21 Aug 2024 17:06:23 +0800 Subject: [PATCH 053/193] Fix internal error when calling `#is_a?` on `External` nodes (#14918) --- spec/compiler/macro/macro_methods_spec.cr | 8 ++++++++ src/compiler/crystal/macros/types.cr | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 9d425cb7e162..10ba78d5bdc6 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -2654,6 +2654,14 @@ module Crystal end end + describe External do + it "executes is_a?" do + assert_macro %({{x.is_a?(External)}}), "true", {x: External.new("foo", [] of Arg, Nop.new, "foo")} + assert_macro %({{x.is_a?(Def)}}), "true", {x: External.new("foo", [] of Arg, Nop.new, "foo")} + assert_macro %({{x.is_a?(ASTNode)}}), "true", {x: External.new("foo", [] of Arg, Nop.new, "foo")} + end + end + describe Primitive do it "executes name" do assert_macro %({{x.name}}), %(:abc), {x: Primitive.new("abc")} diff --git a/src/compiler/crystal/macros/types.cr b/src/compiler/crystal/macros/types.cr index 7a7777e8aef3..3a40a9bc90aa 100644 --- a/src/compiler/crystal/macros/types.cr +++ b/src/compiler/crystal/macros/types.cr @@ -46,7 +46,8 @@ module Crystal @macro_types["Arg"] = NonGenericMacroType.new self, "Arg", ast_node @macro_types["ProcNotation"] = NonGenericMacroType.new self, "ProcNotation", ast_node - @macro_types["Def"] = NonGenericMacroType.new self, "Def", ast_node + @macro_types["Def"] = def_type = NonGenericMacroType.new self, "Def", ast_node + @macro_types["External"] = NonGenericMacroType.new self, "External", def_type @macro_types["Macro"] = NonGenericMacroType.new self, "Macro", ast_node @macro_types["UnaryExpression"] = unary_expression = NonGenericMacroType.new self, "UnaryExpression", ast_node @@ -102,7 +103,6 @@ module Crystal # bottom type @macro_types["NoReturn"] = @macro_no_return = NoReturnMacroType.new self - # unimplemented types (see https://github.com/crystal-lang/crystal/issues/3274#issuecomment-860092436) @macro_types["Self"] = NonGenericMacroType.new self, "Self", ast_node @macro_types["Underscore"] = NonGenericMacroType.new self, "Underscore", ast_node @macro_types["Select"] = NonGenericMacroType.new self, "Select", ast_node From f4022151d3f43bce605b8776c3a341637e10fea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 21 Aug 2024 17:08:09 +0200 Subject: [PATCH 054/193] Fix return type restriction for `ENV.fetch` (#14919) --- spec/std/env_spec.cr | 4 ++++ src/env.cr | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/std/env_spec.cr b/spec/std/env_spec.cr index 038bdc74b9b1..c48afb0ff6f9 100644 --- a/spec/std/env_spec.cr +++ b/spec/std/env_spec.cr @@ -137,6 +137,10 @@ describe "ENV" do ENV.fetch("2") end end + + it "fetches arbitrary default value" do + ENV.fetch("nonexistent", true).should be_true + end end it "handles unicode" do diff --git a/src/env.cr b/src/env.cr index b28e4014ea22..13779f3051aa 100644 --- a/src/env.cr +++ b/src/env.cr @@ -60,7 +60,7 @@ module ENV # Retrieves a value corresponding to the given *key*. Return the second argument's value # if the *key* does not exist. - def self.fetch(key, default) : String? + def self.fetch(key, default : T) : String | T forall T fetch(key) { default } end From c45d3767f6c1d5551968e9a5db01cfad883217b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 21 Aug 2024 17:09:02 +0200 Subject: [PATCH 055/193] Update previous Crystal release 1.13.2 (#14925) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shell.nix | 12 ++++++------ 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3f2310d7808..39984bc5aadb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 8828efe88a10..ba32bb2dd2d6 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: build-interpreter: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build strategy: matrix: part: [0, 1, 2, 3] @@ -67,7 +67,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.1-build + image: crystallang/crystal:1.13.2-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 32761dbb8c75..d1128ebdbca8 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.1] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.2] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 767d401138e7..152e2b5294b5 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -58,7 +58,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.1" + crystal: "1.13.2" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index d518c93a51de..4eede5adf78c 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -10,7 +10,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.13.1-alpine + container: crystallang/crystal:1.13.2-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 8816c31dc9b0..186192288895 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -10,7 +10,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.13.1-alpine + container: crystallang/crystal:1.13.2-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.13.1-alpine + container: crystallang/crystal:1.13.2-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 2b446ec6726f..7ce32ee2d625 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -12,7 +12,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-latest - container: crystallang/crystal:1.13.1-build + container: crystallang/crystal:1.13.2-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 6e36608d608d..d2ed6469d264 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -24,7 +24,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.1" + crystal: "1.13.2" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 74a1f228ceff..4ca0eb96577e 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.1-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.2-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.1}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.2}" case $ARCH in x86_64) diff --git a/shell.nix b/shell.nix index 259cecf9b304..db69834a8a89 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz"; - sha256 = "sha256:0wrfv7bgqwfi76p9s48zg4j953kvjsj5cv59slhhc62lllx926zm"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; + sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-darwin-universal.tar.gz"; - sha256 = "sha256:0wrfv7bgqwfi76p9s48zg4j953kvjsj5cv59slhhc62lllx926zm"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; + sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.1/crystal-1.13.1-1-linux-x86_64.tar.gz"; - sha256 = "sha256:1dghcv8qgjcbq1r0d2saa21xzp4h7pkan6fnmn6hpickib678g7x"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-linux-x86_64.tar.gz"; + sha256 = "sha256:0186q0y97135kvxa8bmzgqc24idv19jg4vglany0pkpzy8b3qs0s"; }; }.${pkgs.stdenv.system}); From bb75d3b789e40883a71a59e78dd26a9708e4d20c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 22 Aug 2024 05:34:43 +0800 Subject: [PATCH 056/193] Fix `SOURCE_DATE_EPOCH` in `Makefile.win` (#14922) --- Makefile.win | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.win b/Makefile.win index 89c0f9972a14..0613acc8a207 100644 --- a/Makefile.win +++ b/Makefile.win @@ -64,7 +64,7 @@ CRYSTAL_CONFIG_LIBRARY_PATH := $$ORIGIN\lib CRYSTAL_CONFIG_BUILD_COMMIT := $(shell git rev-parse --short HEAD) CRYSTAL_CONFIG_PATH := $$ORIGIN\src CRYSTAL_VERSION ?= $(shell type src\VERSION) -SOURCE_DATE_EPOCH ?= $(shell type src/SOURCE_DATE_EPOCH || git show -s --format=%ct HEAD) +SOURCE_DATE_EPOCH ?= $(or $(shell type src\SOURCE_DATE_EPOCH 2>NUL),$(shell git show -s --format=%ct HEAD)) export_vars = $(eval export CRYSTAL_CONFIG_BUILD_COMMIT CRYSTAL_CONFIG_PATH SOURCE_DATE_EPOCH) export_build_vars = $(eval export CRYSTAL_CONFIG_LIBRARY_PATH) LLVM_CONFIG ?= From 0f906a4f4374e69e2ff5348a8029ef0709606e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 22 Aug 2024 11:37:41 +0200 Subject: [PATCH 057/193] Fix `Expectations::Be` for module type (#14926) --- spec/std/spec/expectations_spec.cr | 23 +++++++++++++++++++++++ src/spec/expectations.cr | 10 ++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/spec/std/spec/expectations_spec.cr b/spec/std/spec/expectations_spec.cr index 4acce2bfbad9..0831bca226ca 100644 --- a/spec/std/spec/expectations_spec.cr +++ b/spec/std/spec/expectations_spec.cr @@ -1,5 +1,17 @@ require "spec" +private module MyModule; end + +private class Foo + include MyModule +end + +private record NoObjectId, to_unsafe : Int32 do + def same?(other : self) : Bool + to_unsafe == other.to_unsafe + end +end + describe "expectations" do describe "accept a custom failure message" do it { 1.should be < 3, "custom message!" } @@ -25,6 +37,17 @@ describe "expectations" do array = [1] array.should_not be [1] end + + it "works with type that does not implement `#object_id`" do + a = NoObjectId.new(1) + a.should be a + a.should_not be NoObjectId.new(2) + end + + it "works with module type (#14920)" do + a = Foo.new + a.as(MyModule).should be a.as(MyModule) + end end describe "be_a" do diff --git a/src/spec/expectations.cr b/src/spec/expectations.cr index 193f86d0de21..f50658a5d787 100644 --- a/src/spec/expectations.cr +++ b/src/spec/expectations.cr @@ -73,11 +73,13 @@ module Spec end private def identify(value) - if value.responds_to?(:object_id) - "object_id: #{value.object_id}" - else - value.to_unsafe + if value.responds_to?(:to_unsafe) + if !value.responds_to?(:object_id) + return value.to_unsafe + end end + + "object_id: #{value.object_id}" end end From eb01f2a488bfb30083b1da25cade19c99e3c4e4b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 22 Aug 2024 17:37:54 +0800 Subject: [PATCH 058/193] Allow returning `Proc`s from top-level funs (#14917) --- spec/compiler/codegen/proc_spec.cr | 53 +++++++++++++++++++++++++++++ src/compiler/crystal/codegen/fun.cr | 13 ++++--- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/spec/compiler/codegen/proc_spec.cr b/spec/compiler/codegen/proc_spec.cr index 48db694429e5..65b2731e5ac6 100644 --- a/spec/compiler/codegen/proc_spec.cr +++ b/spec/compiler/codegen/proc_spec.cr @@ -862,6 +862,59 @@ describe "Code gen: proc" do )) end + it "returns proc as function pointer inside top-level fun (#14691)" do + run(<<-CRYSTAL, Int32).should eq(8) + def raise(msg) + while true + end + end + + fun add : Int32, Int32 -> Int32 + ->(x : Int32, y : Int32) { x &+ y } + end + + add.call(3, 5) + CRYSTAL + end + + it "returns ProcPointer inside top-level fun (#14691)" do + run(<<-CRYSTAL, Int32).should eq(8) + def raise(msg) + while true + end + end + + fun foo(x : Int32) : Int32 + x &+ 5 + end + + fun bar : Int32 -> Int32 + ->foo(Int32) + end + + bar.call(3) + CRYSTAL + end + + it "raises if returning closure from top-level fun (#14691)" do + run(<<-CRYSTAL).to_b.should be_true + require "prelude" + + @[Raises] + fun foo(x : Int32) : -> Int32 + -> { x } + end + + begin + foo(1) + rescue + true + else + false + end + CRYSTAL + end + it "closures var on ->var.call (#8584)" do run(%( def bar(x) diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index 5b7c9b224c83..616b21b79d24 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -236,17 +236,22 @@ class Crystal::CodeGenVisitor # Check if this def must use the C calling convention and the return # value must be either casted or passed by sret if target_def.c_calling_convention? && target_def.abi_info? + return_type = target_def.body.type + if return_type.proc? + @last = check_proc_is_not_closure(@last, return_type) + end + abi_info = abi_info(target_def) - ret_type = abi_info.return_type - if cast = ret_type.cast + abi_ret_type = abi_info.return_type + if cast = abi_ret_type.cast casted_last = pointer_cast @last, cast.pointer last = load cast, casted_last ret last return end - if (attr = ret_type.attr) && attr == LLVM::Attribute::StructRet - store load(llvm_type(target_def.body.type), @last), context.fun.params[0] + if (attr = abi_ret_type.attr) && attr == LLVM::Attribute::StructRet + store load(llvm_type(return_type), @last), context.fun.params[0] ret return end From a3bfa4ccc45cc3d1570ac1cf830cb68b45c32bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 22 Aug 2024 16:54:04 +0200 Subject: [PATCH 059/193] Fix `crystal tool dependencies` format flat (#14927) --- src/compiler/crystal/tools/dependencies.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/tools/dependencies.cr b/src/compiler/crystal/tools/dependencies.cr index cfb26fbccc43..7ddc48857ad3 100644 --- a/src/compiler/crystal/tools/dependencies.cr +++ b/src/compiler/crystal/tools/dependencies.cr @@ -124,7 +124,7 @@ module Crystal end private def print_indent - @io.print " " * @stack.size unless @stack.empty? + @io.print " " * @stack.size unless @stack.empty? || @format.flat? end end From d031bfa89cd7464579d5d360a178fa3ea96f6e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 22 Aug 2024 16:54:18 +0200 Subject: [PATCH 060/193] Fix `crystal tool dependencies` filters for Windows paths (#14928) --- src/compiler/crystal/tools/dependencies.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/tools/dependencies.cr b/src/compiler/crystal/tools/dependencies.cr index 7ddc48857ad3..91701285639b 100644 --- a/src/compiler/crystal/tools/dependencies.cr +++ b/src/compiler/crystal/tools/dependencies.cr @@ -8,8 +8,8 @@ class Crystal::Command dependency_printer = DependencyPrinter.create(STDOUT, format: DependencyPrinter::Format.parse(config.output_format), verbose: config.verbose) - dependency_printer.includes.concat config.includes.map { |path| ::Path[path].expand.to_s } - dependency_printer.excludes.concat config.excludes.map { |path| ::Path[path].expand.to_s } + dependency_printer.includes.concat config.includes.map { |path| ::Path[path].expand.to_posix.to_s } + dependency_printer.excludes.concat config.excludes.map { |path| ::Path[path].expand.to_posix.to_s } config.compiler.dependency_printer = dependency_printer dependency_printer.start_format From c462cd61cbba7f0f0003570ffef894821010c335 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 23 Aug 2024 17:59:21 +0800 Subject: [PATCH 061/193] Open non-blocking regular files as overlapped on Windows (#14921) --- spec/std/file/tempfile_spec.cr | 6 +++--- spec/std/file_spec.cr | 8 ++++++++ src/crystal/system/file.cr | 2 +- src/crystal/system/unix/file.cr | 6 +++--- src/crystal/system/win32/file.cr | 16 ++++++++++------ src/crystal/system/win32/file_descriptor.cr | 3 +-- src/file.cr | 2 +- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/spec/std/file/tempfile_spec.cr b/spec/std/file/tempfile_spec.cr index 3ede9e52e44d..84d9cd553398 100644 --- a/spec/std/file/tempfile_spec.cr +++ b/spec/std/file/tempfile_spec.cr @@ -200,7 +200,7 @@ describe Crystal::System::File do fd, path = Crystal::System::File.mktemp("A", "Z", dir: tempdir, random: TestRNG.new([7, 8, 9, 10, 11, 12, 13, 14])) path.should eq Path[tempdir, "A789abcdeZ"].to_s ensure - File.from_fd(path, fd).close if fd && path + IO::FileDescriptor.new(fd).close if fd end end @@ -212,7 +212,7 @@ describe Crystal::System::File do fd, path = Crystal::System::File.mktemp("A", "Z", dir: tempdir, random: TestRNG.new([7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22])) path.should eq File.join(tempdir, "AfghijklmZ") ensure - File.from_fd(path, fd).close if fd && path + IO::FileDescriptor.new(fd).close if fd end end @@ -223,7 +223,7 @@ describe Crystal::System::File do expect_raises(File::AlreadyExistsError, "Error creating temporary file") do fd, path = Crystal::System::File.mktemp("A", "Z", dir: tempdir, random: TestRNG.new([7, 8, 9, 10, 11, 12, 13, 14])) ensure - File.from_fd(path, fd).close if fd && path + IO::FileDescriptor.new(fd).close if fd end end end diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 942ae8a1143d..96dbacd73cc9 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -71,6 +71,14 @@ describe "File" do end end + it "opens regular file as non-blocking" do + with_tempfile("regular") do |path| + File.open(path, "w", blocking: false) do |file| + file.blocking.should be_false + end + end + end + {% if flag?(:unix) %} if File.exists?("/dev/tty") it "opens character device" do diff --git a/src/crystal/system/file.cr b/src/crystal/system/file.cr index 452bfb6e4ead..84dbd0fa5c98 100644 --- a/src/crystal/system/file.cr +++ b/src/crystal/system/file.cr @@ -65,7 +65,7 @@ module Crystal::System::File io << suffix end - handle, errno = open(path, mode, perm) + handle, errno = open(path, mode, perm, blocking: true) if error_is_none?(errno) return {handle, path} diff --git a/src/crystal/system/unix/file.cr b/src/crystal/system/unix/file.cr index fafd1d0d0a16..a049659e684f 100644 --- a/src/crystal/system/unix/file.cr +++ b/src/crystal/system/unix/file.cr @@ -3,10 +3,10 @@ require "file/error" # :nodoc: module Crystal::System::File - def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) + def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions, blocking) perm = ::File::Permissions.new(perm) if perm.is_a? Int32 - fd, errno = open(filename, open_flag(mode), perm) + fd, errno = open(filename, open_flag(mode), perm, blocking) unless errno.none? raise ::File::Error.from_os_error("Error opening file with mode '#{mode}'", errno, file: filename) @@ -15,7 +15,7 @@ module Crystal::System::File fd end - def self.open(filename : String, flags : Int32, perm : ::File::Permissions) : {LibC::Int, Errno} + def self.open(filename : String, flags : Int32, perm : ::File::Permissions, blocking _blocking) : {LibC::Int, Errno} filename.check_no_null_byte flags |= LibC::O_CLOEXEC diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 9039cc40a7ac..7b7b443ce310 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -14,7 +14,7 @@ module Crystal::System::File # write at the end of the file. @system_append = false - def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) : FileDescriptor::Handle + def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions, blocking : Bool?) : FileDescriptor::Handle perm = ::File::Permissions.new(perm) if perm.is_a? Int32 # Only the owner writable bit is used, since windows only supports # the read only attribute. @@ -24,7 +24,7 @@ module Crystal::System::File perm = LibC::S_IREAD end - handle, error = open(filename, open_flag(mode), ::File::Permissions.new(perm)) + handle, error = open(filename, open_flag(mode), ::File::Permissions.new(perm), blocking != false) unless error.error_success? raise ::File::Error.from_os_error("Error opening file with mode '#{mode}'", error, file: filename) end @@ -32,8 +32,8 @@ module Crystal::System::File handle end - def self.open(filename : String, flags : Int32, perm : ::File::Permissions) : {FileDescriptor::Handle, WinError} - access, disposition, attributes = self.posix_to_open_opts flags, perm + def self.open(filename : String, flags : Int32, perm : ::File::Permissions, blocking : Bool) : {FileDescriptor::Handle, WinError} + access, disposition, attributes = self.posix_to_open_opts flags, perm, blocking handle = LibC.CreateFileW( System.to_wstr(filename), @@ -48,7 +48,7 @@ module Crystal::System::File {handle.address, handle == LibC::INVALID_HANDLE_VALUE ? WinError.value : WinError::ERROR_SUCCESS} end - private def self.posix_to_open_opts(flags : Int32, perm : ::File::Permissions) + private def self.posix_to_open_opts(flags : Int32, perm : ::File::Permissions, blocking : Bool) access = if flags.bits_set? LibC::O_WRONLY LibC::FILE_GENERIC_WRITE elsif flags.bits_set? LibC::O_RDWR @@ -77,7 +77,7 @@ module Crystal::System::File disposition = LibC::OPEN_EXISTING end - attributes = LibC::FILE_ATTRIBUTE_NORMAL + attributes = 0 unless perm.owner_write? attributes |= LibC::FILE_ATTRIBUTE_READONLY end @@ -97,6 +97,10 @@ module Crystal::System::File attributes |= LibC::FILE_FLAG_RANDOM_ACCESS end + unless blocking + attributes |= LibC::FILE_FLAG_OVERLAPPED + end + {access, disposition, attributes} end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 7899f75407f7..37813307191f 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -89,6 +89,7 @@ module Crystal::System::FileDescriptor private def system_blocking_init(value) @system_blocking = value + Crystal::EventLoop.current.create_completion_port(windows_handle) unless value end private def system_close_on_exec? @@ -264,13 +265,11 @@ module Crystal::System::FileDescriptor w_pipe_flags |= LibC::FILE_FLAG_OVERLAPPED unless write_blocking w_pipe = LibC.CreateNamedPipeA(pipe_name, w_pipe_flags, pipe_mode, 1, PIPE_BUFFER_SIZE, PIPE_BUFFER_SIZE, 0, nil) raise IO::Error.from_winerror("CreateNamedPipeA") if w_pipe == LibC::INVALID_HANDLE_VALUE - Crystal::EventLoop.current.create_completion_port(w_pipe) unless write_blocking r_pipe_flags = LibC::FILE_FLAG_NO_BUFFERING r_pipe_flags |= LibC::FILE_FLAG_OVERLAPPED unless read_blocking r_pipe = LibC.CreateFileW(System.to_wstr(pipe_name), LibC::GENERIC_READ | LibC::FILE_WRITE_ATTRIBUTES, 0, nil, LibC::OPEN_EXISTING, r_pipe_flags, nil) raise IO::Error.from_winerror("CreateFileW") if r_pipe == LibC::INVALID_HANDLE_VALUE - Crystal::EventLoop.current.create_completion_port(r_pipe) unless read_blocking r = IO::FileDescriptor.new(r_pipe.address, read_blocking) w = IO::FileDescriptor.new(w_pipe.address, write_blocking) diff --git a/src/file.cr b/src/file.cr index 202a05ab01f0..5169a6dc703d 100644 --- a/src/file.cr +++ b/src/file.cr @@ -172,7 +172,7 @@ class File < IO::FileDescriptor # additional syscall. def self.new(filename : Path | String, mode = "r", perm = DEFAULT_CREATE_PERMISSIONS, encoding = nil, invalid = nil, blocking = true) filename = filename.to_s - fd = Crystal::System::File.open(filename, mode, perm: perm) + fd = Crystal::System::File.open(filename, mode, perm: perm, blocking: blocking) new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid).tap { |f| f.system_set_mode(mode) } end From d4fc67a271b42afbb91875154f803fbf17047029 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 23 Aug 2024 17:59:43 +0800 Subject: [PATCH 062/193] Include `Crystal::System::Group` instead of extending it (#14930) --- src/crystal/system/group.cr | 10 ++++++++++ src/crystal/system/unix/group.cr | 19 +++++++++++++++---- src/crystal/system/wasi/group.cr | 16 ++++++++++++---- src/system/group.cr | 19 ++++++++++--------- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/crystal/system/group.cr b/src/crystal/system/group.cr index dce631e8c1ab..8a542e2cc63c 100644 --- a/src/crystal/system/group.cr +++ b/src/crystal/system/group.cr @@ -1,3 +1,13 @@ +module Crystal::System::Group + # def system_name : String + + # def system_id : String + + # def self.from_name?(groupname : String) : ::System::Group? + + # def self.from_id?(groupid : String) : ::System::Group? +end + {% if flag?(:wasi) %} require "./wasi/group" {% elsif flag?(:unix) %} diff --git a/src/crystal/system/unix/group.cr b/src/crystal/system/unix/group.cr index d7d408f77608..d4562cc7d286 100644 --- a/src/crystal/system/unix/group.cr +++ b/src/crystal/system/unix/group.cr @@ -4,11 +4,22 @@ require "../unix" module Crystal::System::Group private GETGR_R_SIZE_MAX = 1024 * 16 - private def from_struct(grp) - new(String.new(grp.gr_name), grp.gr_gid.to_s) + def initialize(@name : String, @id : String) end - private def from_name?(groupname : String) + def system_name + @name + end + + def system_id + @id + end + + private def self.from_struct(grp) + ::System::Group.new(String.new(grp.gr_name), grp.gr_gid.to_s) + end + + def self.from_name?(groupname : String) groupname.check_no_null_byte grp = uninitialized LibC::Group @@ -21,7 +32,7 @@ module Crystal::System::Group end end - private def from_id?(groupid : String) + def self.from_id?(groupid : String) groupid = groupid.to_u32? return unless groupid diff --git a/src/crystal/system/wasi/group.cr b/src/crystal/system/wasi/group.cr index 0aa09bd40aa8..c94fffa4fe6e 100644 --- a/src/crystal/system/wasi/group.cr +++ b/src/crystal/system/wasi/group.cr @@ -1,9 +1,17 @@ module Crystal::System::Group - private def from_name?(groupname : String) - raise NotImplementedError.new("Crystal::System::Group#from_name?") + def system_name + raise NotImplementedError.new("Crystal::System::Group#system_name") end - private def from_id?(groupid : String) - raise NotImplementedError.new("Crystal::System::Group#from_id?") + def system_id + raise NotImplementedError.new("Crystal::System::Group#system_id") + end + + def self.from_name?(groupname : String) + raise NotImplementedError.new("Crystal::System::Group.from_name?") + end + + def self.from_id?(groupid : String) + raise NotImplementedError.new("Crystal::System::Group.from_id?") end end diff --git a/src/system/group.cr b/src/system/group.cr index bd992e6af19d..47b9768cca52 100644 --- a/src/system/group.cr +++ b/src/system/group.cr @@ -17,19 +17,20 @@ class System::Group class NotFoundError < Exception end - extend Crystal::System::Group + include Crystal::System::Group # The group's name. - getter name : String + def name : String + system_name + end # The group's identifier. - getter id : String - - def_equals_and_hash @id - - private def initialize(@name, @id) + def id : String + system_id end + def_equals_and_hash id + # Returns the group associated with the given name. # # Raises `NotFoundError` if no such group exists. @@ -41,7 +42,7 @@ class System::Group # # Returns `nil` if no such group exists. def self.find_by?(*, name : String) : System::Group? - from_name?(name) + Crystal::System::Group.from_name?(name) end # Returns the group associated with the given ID. @@ -55,7 +56,7 @@ class System::Group # # Returns `nil` if no such group exists. def self.find_by?(*, id : String) : System::Group? - from_id?(id) + Crystal::System::Group.from_id?(id) end def to_s(io) From cc6859bd32bdaa4bbed5ef95df16f00014c7ba97 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 23 Aug 2024 17:59:52 +0800 Subject: [PATCH 063/193] Include `Crystal::System::User` instead of extending it (#14929) --- src/crystal/system/unix/user.cr | 35 +++++++++++++++++++++++++++++---- src/crystal/system/user.cr | 18 +++++++++++++++++ src/crystal/system/wasi/user.cr | 32 ++++++++++++++++++++++++++---- src/system/user.cr | 35 +++++++++++++++++++++------------ 4 files changed, 99 insertions(+), 21 deletions(-) diff --git a/src/crystal/system/unix/user.cr b/src/crystal/system/unix/user.cr index 8e4f16e8c1c4..c1f91d0f118c 100644 --- a/src/crystal/system/unix/user.cr +++ b/src/crystal/system/unix/user.cr @@ -4,14 +4,41 @@ require "../unix" module Crystal::System::User GETPW_R_SIZE_MAX = 1024 * 16 - private def from_struct(pwd) + def initialize(@username : String, @id : String, @group_id : String, @name : String, @home_directory : String, @shell : String) + end + + def system_username + @username + end + + def system_id + @id + end + + def system_group_id + @group_id + end + + def system_name + @name + end + + def system_home_directory + @home_directory + end + + def system_shell + @shell + end + + private def self.from_struct(pwd) username = String.new(pwd.pw_name) # `pw_gecos` is not part of POSIX and bionic for example always leaves it null user = pwd.pw_gecos ? String.new(pwd.pw_gecos).partition(',')[0] : username - new(username, pwd.pw_uid.to_s, pwd.pw_gid.to_s, user, String.new(pwd.pw_dir), String.new(pwd.pw_shell)) + ::System::User.new(username, pwd.pw_uid.to_s, pwd.pw_gid.to_s, user, String.new(pwd.pw_dir), String.new(pwd.pw_shell)) end - private def from_username?(username : String) + def self.from_username?(username : String) username.check_no_null_byte pwd = uninitialized LibC::Passwd @@ -24,7 +51,7 @@ module Crystal::System::User end end - private def from_id?(id : String) + def self.from_id?(id : String) id = id.to_u32? return unless id diff --git a/src/crystal/system/user.cr b/src/crystal/system/user.cr index ecee92c8dcb5..cb3db8cda026 100644 --- a/src/crystal/system/user.cr +++ b/src/crystal/system/user.cr @@ -1,3 +1,21 @@ +module Crystal::System::User + # def system_username : String + + # def system_id : String + + # def system_group_id : String + + # def system_name : String + + # def system_home_directory : String + + # def system_shell : String + + # def self.from_username?(username : String) : ::System::User? + + # def self.from_id?(id : String) : ::System::User? +end + {% if flag?(:wasi) %} require "./wasi/user" {% elsif flag?(:unix) %} diff --git a/src/crystal/system/wasi/user.cr b/src/crystal/system/wasi/user.cr index 06415897000e..2d1c6e91b770 100644 --- a/src/crystal/system/wasi/user.cr +++ b/src/crystal/system/wasi/user.cr @@ -1,9 +1,33 @@ module Crystal::System::User - private def from_username?(username : String) - raise NotImplementedError.new("Crystal::System::User#from_username?") + def system_username + raise NotImplementedError.new("Crystal::System::User#system_username") end - private def from_id?(id : String) - raise NotImplementedError.new("Crystal::System::User#from_id?") + def system_id + raise NotImplementedError.new("Crystal::System::User#system_id") + end + + def system_group_id + raise NotImplementedError.new("Crystal::System::User#system_group_id") + end + + def system_name + raise NotImplementedError.new("Crystal::System::User#system_name") + end + + def system_home_directory + raise NotImplementedError.new("Crystal::System::User#system_home_directory") + end + + def system_shell + raise NotImplementedError.new("Crystal::System::User#system_shell") + end + + def self.from_username?(username : String) + raise NotImplementedError.new("Crystal::System::User.from_username?") + end + + def self.from_id?(id : String) + raise NotImplementedError.new("Crystal::System::User.from_id?") end end diff --git a/src/system/user.cr b/src/system/user.cr index 7d6c250689da..01c8d11d9e1c 100644 --- a/src/system/user.cr +++ b/src/system/user.cr @@ -17,34 +17,43 @@ class System::User class NotFoundError < Exception end - extend Crystal::System::User + include Crystal::System::User # The user's username. - getter username : String + def username : String + system_username + end # The user's identifier. - getter id : String + def id : String + system_id + end # The user's primary group identifier. - getter group_id : String + def group_id : String + system_group_id + end # The user's real or full name. # # May not be present on all platforms. Returns the same value as `#username` # if neither a real nor full name is available. - getter name : String + def name : String + system_name + end # The user's home directory. - getter home_directory : String + def home_directory : String + system_home_directory + end # The user's login shell. - getter shell : String - - def_equals_and_hash @id - - private def initialize(@username, @id, @group_id, @name, @home_directory, @shell) + def shell : String + system_shell end + def_equals_and_hash id + # Returns the user associated with the given username. # # Raises `NotFoundError` if no such user exists. @@ -56,7 +65,7 @@ class System::User # # Returns `nil` if no such user exists. def self.find_by?(*, name : String) : System::User? - from_username?(name) + Crystal::System::User.from_username?(name) end # Returns the user associated with the given ID. @@ -70,7 +79,7 @@ class System::User # # Returns `nil` if no such user exists. def self.find_by?(*, id : String) : System::User? - from_id?(id) + Crystal::System::User.from_id?(id) end def to_s(io) From cc0dfc16043bbfa832d259563cd738dcf6bd8833 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 23 Aug 2024 10:59:59 -0400 Subject: [PATCH 064/193] Add `URI::Params::Serializable` (#14684) --- spec/std/uri/params/from_www_form_spec.cr | 151 ++++++++++++++++++++++ spec/std/uri/params/serializable_spec.cr | 133 +++++++++++++++++++ spec/std/uri/params/to_www_form_spec.cr | 60 +++++++++ src/docs_main.cr | 1 + src/uri/params/from_www_form.cr | 67 ++++++++++ src/uri/params/serializable.cr | 129 ++++++++++++++++++ src/uri/params/to_www_form.cr | 48 +++++++ 7 files changed, 589 insertions(+) create mode 100644 spec/std/uri/params/from_www_form_spec.cr create mode 100644 spec/std/uri/params/serializable_spec.cr create mode 100644 spec/std/uri/params/to_www_form_spec.cr create mode 100644 src/uri/params/from_www_form.cr create mode 100644 src/uri/params/serializable.cr create mode 100644 src/uri/params/to_www_form.cr diff --git a/spec/std/uri/params/from_www_form_spec.cr b/spec/std/uri/params/from_www_form_spec.cr new file mode 100644 index 000000000000..e0ab818c2e86 --- /dev/null +++ b/spec/std/uri/params/from_www_form_spec.cr @@ -0,0 +1,151 @@ +require "spec" +require "uri/params/serializable" + +private enum Color + Red + Green + Blue +end + +describe ".from_www_form" do + it Array do + Array(Int32).from_www_form(URI::Params.new({"values" => ["1", "2"]}), "values").should eq [1, 2] + Array(Int32).from_www_form(URI::Params.new({"values[]" => ["1", "2"]}), "values").should eq [1, 2] + Array(String).from_www_form(URI::Params.new({"values" => ["", ""]}), "values").should eq ["", ""] + end + + describe Bool do + it "a truthy value" do + Bool.from_www_form("true").should be_true + Bool.from_www_form("on").should be_true + Bool.from_www_form("yes").should be_true + Bool.from_www_form("1").should be_true + end + + it "a falsey value" do + Bool.from_www_form("false").should be_false + Bool.from_www_form("off").should be_false + Bool.from_www_form("no").should be_false + Bool.from_www_form("0").should be_false + end + + it "any other value" do + Bool.from_www_form("foo").should be_nil + Bool.from_www_form("").should be_nil + end + end + + describe String do + it "scalar string" do + String.from_www_form("John Doe").should eq "John Doe" + end + + it "with key" do + String.from_www_form(URI::Params.new({"name" => ["John Doe"]}), "name").should eq "John Doe" + end + + it "with missing key" do + String.from_www_form(URI::Params.new({"" => ["John Doe"]}), "name").should be_nil + end + + it "with alternate casing" do + String.from_www_form(URI::Params.new({"Name" => ["John Doe"]}), "name").should be_nil + end + + it "empty value" do + String.from_www_form(URI::Params.new({"name" => [""]}), "name").should eq "" + end + end + + describe Enum do + it "valid value" do + Color.from_www_form("green").should eq Color::Green + end + + it "invalid value" do + expect_raises ArgumentError do + Color.from_www_form "" + end + end + end + + describe Time do + it "valid value" do + Time.from_www_form("2016-11-16T09:55:48-03:00").to_utc.should eq(Time.utc(2016, 11, 16, 12, 55, 48)) + Time.from_www_form("2016-11-16T09:55:48-0300").to_utc.should eq(Time.utc(2016, 11, 16, 12, 55, 48)) + Time.from_www_form("20161116T095548-03:00").to_utc.should eq(Time.utc(2016, 11, 16, 12, 55, 48)) + end + + it "invalid value" do + expect_raises Time::Format::Error do + Time.from_www_form "" + end + end + end + + describe Nil do + it "valid values" do + Nil.from_www_form("").should be_nil + end + + it "invalid value" do + expect_raises ArgumentError do + Nil.from_www_form "null" + end + end + end + + describe Number do + describe Int do + it "valid numbers" do + Int64.from_www_form("123").should eq 123_i64 + UInt8.from_www_form("7").should eq 7_u8 + Int64.from_www_form("-12").should eq -12_i64 + end + + it "with whitespace" do + expect_raises ArgumentError do + Int32.from_www_form(" 123") + end + end + + it "empty value" do + expect_raises ArgumentError do + Int16.from_www_form "" + end + end + end + + describe Float do + it "valid numbers" do + Float32.from_www_form("123.0").should eq 123_f32 + Float64.from_www_form("123.0").should eq 123_f64 + end + + it "with whitespace" do + expect_raises ArgumentError do + Float64.from_www_form(" 123.0") + end + end + + it "empty value" do + expect_raises Exception do + Float64.from_www_form "" + end + end + end + end + + describe Union do + it "valid" do + String?.from_www_form(URI::Params.parse("name=John Doe"), "name").should eq "John Doe" + String?.from_www_form(URI::Params.parse("name="), "name").should eq "" + end + + it "invalid" do + expect_raises ArgumentError do + (Int32 | Float64).from_www_form(URI::Params.parse("name=John Doe"), "name") + end + end + end +end diff --git a/spec/std/uri/params/serializable_spec.cr b/spec/std/uri/params/serializable_spec.cr new file mode 100644 index 000000000000..bb1fdc7240e9 --- /dev/null +++ b/spec/std/uri/params/serializable_spec.cr @@ -0,0 +1,133 @@ +require "spec" +require "uri/params/serializable" + +private record SimpleType, page : Int32, strict : Bool, per_page : UInt8 do + include URI::Params::Serializable +end + +private record SimpleTypeDefaults, page : Int32, strict : Bool, per_page : Int32 = 10 do + include URI::Params::Serializable +end + +private record SimpleTypeNilable, page : Int32, strict : Bool, per_page : Int32? = nil do + include URI::Params::Serializable +end + +private record SimpleTypeNilableDefault, page : Int32, strict : Bool, per_page : Int32? = 20 do + include URI::Params::Serializable +end + +record Filter, status : String?, total : Float64? do + include URI::Params::Serializable +end + +record Search, filter : Filter?, limit : Int32 = 25, offset : Int32 = 0 do + include URI::Params::Serializable +end + +record GrandChild, name : String do + include URI::Params::Serializable +end + +record Child, status : String?, grand_child : GrandChild do + include URI::Params::Serializable +end + +record Parent, child : Child do + include URI::Params::Serializable +end + +module MyConverter + def self.from_www_form(params : URI::Params, name : String) + params[name].to_i * 10 + end +end + +private record ConverterType, value : Int32 do + include URI::Params::Serializable + + @[URI::Params::Field(converter: MyConverter)] + @value : Int32 +end + +class ParentType + include URI::Params::Serializable + + getter name : String +end + +class ChildType < ParentType +end + +describe URI::Params::Serializable do + describe ".from_www_form" do + it "simple type" do + SimpleType.from_www_form("page=10&strict=true&per_page=5").should eq SimpleType.new(10, true, 5) + end + + it "missing required property" do + expect_raises URI::SerializableError, "Missing required property: 'page'." do + SimpleType.from_www_form("strict=true&per_page=5") + end + end + + it "with default values" do + SimpleTypeDefaults.from_www_form("page=10&strict=off").should eq SimpleTypeDefaults.new(10, false, 10) + end + + it "with nilable values" do + SimpleTypeNilable.from_www_form("page=10&strict=true").should eq SimpleTypeNilable.new(10, true, nil) + end + + it "with nilable default" do + SimpleTypeNilableDefault.from_www_form("page=10&strict=true").should eq SimpleTypeNilableDefault.new(10, true, 20) + end + + it "with custom converter" do + ConverterType.from_www_form("value=10").should eq ConverterType.new(100) + end + + it "child type" do + ChildType.from_www_form("name=Fred").name.should eq "Fred" + end + + describe "nested type" do + it "happy path" do + Search.from_www_form("offset=10&filter[status]=active&filter[total]=3.14") + .should eq Search.new Filter.new("active", 3.14), offset: 10 + end + + it "missing nilable nested data" do + Search.from_www_form("offset=10") + .should eq Search.new Filter.new(nil, nil), offset: 10 + end + + it "missing required nested property" do + expect_raises URI::SerializableError, "Missing required property: 'child[grand_child][name]'." do + Parent.from_www_form("child[status]=active") + end + end + + it "doubly nested" do + Parent.from_www_form("child[status]=active&child[grand_child][name]=Fred") + .should eq Parent.new Child.new("active", GrandChild.new("Fred")) + end + end + end + + describe "#to_www_form" do + it "simple type" do + SimpleType.new(10, true, 5).to_www_form.should eq "page=10&strict=true&per_page=5" + end + + it "nested type path" do + Search.new(Filter.new("active", 3.14), offset: 10).to_www_form + .should eq "filter%5Bstatus%5D=active&filter%5Btotal%5D=3.14&limit=25&offset=10" + end + + it "doubly nested" do + Parent.new(Child.new("active", GrandChild.new("Fred"))).to_www_form + .should eq "child%5Bstatus%5D=active&child%5Bgrand_child%5D%5Bname%5D=Fred" + end + end +end diff --git a/spec/std/uri/params/to_www_form_spec.cr b/spec/std/uri/params/to_www_form_spec.cr new file mode 100644 index 000000000000..c10d44334de5 --- /dev/null +++ b/spec/std/uri/params/to_www_form_spec.cr @@ -0,0 +1,60 @@ +require "spec" +require "uri/params/serializable" + +private enum Color + Red + Green + BlueGreen +end + +describe "#to_www_form" do + it Number do + URI::Params.build do |builder| + 12.to_www_form builder, "value" + end.should eq "value=12" + end + + it Enum do + URI::Params.build do |builder| + Color::BlueGreen.to_www_form builder, "value" + end.should eq "value=blue_green" + end + + it String do + URI::Params.build do |builder| + "12".to_www_form builder, "value" + end.should eq "value=12" + end + + it Bool do + URI::Params.build do |builder| + false.to_www_form builder, "value" + end.should eq "value=false" + end + + it Nil do + URI::Params.build do |builder| + nil.to_www_form builder, "value" + end.should eq "value=" + end + + it Time do + URI::Params.build do |builder| + Time.utc(2024, 8, 6, 9, 48, 10).to_www_form builder, "value" + end.should eq "value=2024-08-06T09%3A48%3A10Z" + end + + describe Array do + it "of a single type" do + URI::Params.build do |builder| + [1, 2, 3].to_www_form builder, "value" + end.should eq "value=1&value=2&value=3" + end + + it "of a union of types" do + URI::Params.build do |builder| + [1, false, "foo"].to_www_form builder, "value" + end.should eq "value=1&value=false&value=foo" + end + end +end diff --git a/src/docs_main.cr b/src/docs_main.cr index 5769678ca131..e670d6d3fa83 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -52,6 +52,7 @@ require "./string_pool" require "./string_scanner" require "./unicode/unicode" require "./uri" +require "./uri/params/serializable" require "./uuid" require "./uuid/json" require "./syscall" diff --git a/src/uri/params/from_www_form.cr b/src/uri/params/from_www_form.cr new file mode 100644 index 000000000000..819c9fc9d82e --- /dev/null +++ b/src/uri/params/from_www_form.cr @@ -0,0 +1,67 @@ +# :nodoc: +def Object.from_www_form(params : URI::Params, name : String) + return unless value = params[name]? + + self.from_www_form value +end + +# :nodoc: +def Array.from_www_form(params : URI::Params, name : String) + name = if params.has_key? name + name + elsif params.has_key? "#{name}[]" + "#{name}[]" + else + return + end + + params.fetch_all(name).map { |item| T.from_www_form(item).as T } +end + +# :nodoc: +def Bool.from_www_form(value : String) + case value + when "true", "1", "yes", "on" then true + when "false", "0", "no", "off" then false + end +end + +# :nodoc: +def Number.from_www_form(value : String) + new value, whitespace: false +end + +# :nodoc: +def String.from_www_form(value : String) + value +end + +# :nodoc: +def Enum.from_www_form(value : String) + parse value +end + +# :nodoc: +def Time.from_www_form(value : String) + Time::Format::ISO_8601_DATE_TIME.parse value +end + +# :nodoc: +def Union.from_www_form(params : URI::Params, name : String) + # Process non nilable types first as they are more likely to work. + {% for type in T.sort_by { |t| t.nilable? ? 1 : 0 } %} + begin + return {{type}}.from_www_form params, name + rescue + # Noop to allow next T to be tried. + end + {% end %} + raise ArgumentError.new "Invalid #{self}: '#{params[name]}'." +end + +# :nodoc: +def Nil.from_www_form(value : String) : Nil + return if value.empty? + + raise ArgumentError.new "Invalid Nil value: '#{value}'." +end diff --git a/src/uri/params/serializable.cr b/src/uri/params/serializable.cr new file mode 100644 index 000000000000..c0d766e85242 --- /dev/null +++ b/src/uri/params/serializable.cr @@ -0,0 +1,129 @@ +require "uri" + +require "./to_www_form" +require "./from_www_form" + +struct URI::Params + annotation Field; end + + # The `URI::Params::Serializable` module automatically generates methods for `x-www-form-urlencoded` serialization when included. + # + # NOTE: To use this module, you must explicitly import it with `require "uri/params/serializable"`. + # + # ### Example + # + # ``` + # require "uri/params/serializable" + # + # struct Applicant + # include URI::Params::Serializable + # + # getter first_name : String + # getter last_name : String + # getter qualities : Array(String) + # end + # + # applicant = Applicant.from_www_form "first_name=John&last_name=Doe&qualities=kind&qualities=smart" + # applicant.first_name # => "John" + # applicant.last_name # => "Doe" + # applicant.qualities # => ["kind", "smart"] + # applicant.to_www_form # => "first_name=John&last_name=Doe&qualities=kind&qualities=smart" + # ``` + # + # ### Usage + # + # Including `URI::Params::Serializable` will create `#to_www_form` and `self.from_www_form` methods on the current class. + # By default, these methods serialize into a www form encoded string containing the value of every instance variable, the keys being the instance variable name. + # Union types are also supported, including unions with nil. + # If multiple types in a union parse correctly, it is undefined which one will be chosen. + # + # To change how individual instance variables are parsed, the annotation `URI::Params::Field` can be placed on the instance variable. + # Annotating property, getter and setter macros is also allowed. + # + # `URI::Params::Field` properties: + # * **converter**: specify an alternate type for parsing. The converter must define `.from_www_form(params : URI::Params, name : String)`. + # An example use case would be customizing the format when parsing `Time` instances, or supporting a type not natively supported. + # + # Deserialization also respects default values of variables: + # ``` + # require "uri/params/serializable" + # + # struct A + # include URI::Params::Serializable + # + # @a : Int32 + # @b : Float64 = 1.0 + # end + # + # A.from_www_form("a=1") # => A(@a=1, @b=1.0) + # ``` + module Serializable + macro included + def self.from_www_form(params : String) + new_from_www_form URI::Params.parse params + end + + # :nodoc: + # + # This is needed so that nested types can pass the name thru internally. + # Has to be public so the generated code can call it, but should be considered an implementation detail. + def self.from_www_form(params : ::URI::Params, name : String) + new_from_www_form(params, name) + end + + protected def self.new_from_www_form(params : ::URI::Params, name : String? = nil) + instance = allocate + instance.initialize(__uri_params: params, name: name) + GC.add_finalizer(instance) if instance.responds_to?(:finalize) + instance + end + + macro inherited + def self.from_www_form(params : String) + new_from_www_form URI::Params.parse params + end + + # :nodoc: + def self.from_www_form(params : ::URI::Params, name : String) + new_from_www_form(params, name) + end + end + end + + # :nodoc: + def initialize(*, __uri_params params : ::URI::Params, name : String?) + {% begin %} + {% for ivar, idx in @type.instance_vars %} + %name{idx} = name.nil? ? {{ivar.name.stringify}} : "#{name}[#{{{ivar.name.stringify}}}]" + %value{idx} = {{(ann = ivar.annotation(URI::Params::Field)) && (converter = ann["converter"]) ? converter : ivar.type}}.from_www_form params, %name{idx} + + unless %value{idx}.nil? + @{{ivar.name.id}} = %value{idx} + else + {% unless ivar.type.resolve.nilable? || ivar.has_default_value? %} + raise URI::SerializableError.new "Missing required property: '#{%name{idx}}'." + {% end %} + end + {% end %} + {% end %} + end + + def to_www_form(*, space_to_plus : Bool = true) : String + URI::Params.build(space_to_plus: space_to_plus) do |form| + {% for ivar in @type.instance_vars %} + @{{ivar.name.id}}.to_www_form form, {{ivar.name.stringify}} + {% end %} + end + end + + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) + {% for ivar in @type.instance_vars %} + @{{ivar.name.id}}.to_www_form builder, "#{name}[#{{{ivar.name.stringify}}}]" + {% end %} + end + end +end + +class URI::SerializableError < URI::Error +end diff --git a/src/uri/params/to_www_form.cr b/src/uri/params/to_www_form.cr new file mode 100644 index 000000000000..3a0007781e64 --- /dev/null +++ b/src/uri/params/to_www_form.cr @@ -0,0 +1,48 @@ +struct Bool + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_s + end +end + +class Array + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + each &.to_www_form builder, name + end +end + +class String + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, self + end +end + +struct Number + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_s + end +end + +struct Nil + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, self + end +end + +struct Enum + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_s.underscore + end +end + +struct Time + # :nodoc: + def to_www_form(builder : URI::Params::Builder, name : String) : Nil + builder.add name, to_rfc3339 + end +end From 0ad3e91668610e1b22379cc8e5c1c5fbba40d34e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 25 Aug 2024 19:15:11 +0800 Subject: [PATCH 065/193] Fix `String#index` and `#rindex` for `Char::REPLACEMENT` (#14937) If the string consists only of ASCII characters and invalid UTF-8 byte sequences, all the latter should correspond to `Char::REPLACEMENT`, and so `#index` and `#rindex` should detect them, but this was broken since #14461. --- spec/std/string_spec.cr | 6 ++++++ src/string.cr | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 6bb4bd2c0c62..5b70deda13c3 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -957,6 +957,7 @@ describe "String" do it { "日本語".index('本').should eq(1) } it { "bar".index('あ').should be_nil } it { "あいう_えお".index('_').should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}').should eq(3) } describe "with offset" do it { "foobarbaz".index('a', 5).should eq(7) } @@ -964,6 +965,8 @@ describe "String" do it { "foo".index('g', 1).should be_nil } it { "foo".index('g', -20).should be_nil } it { "日本語日本語".index('本', 2).should eq(4) } + it { "xyz\xFFxyz".index('\u{FFFD}', 2).should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}', 4).should be_nil } # Check offset type it { "foobarbaz".index('a', 5_i64).should eq(7) } @@ -1106,6 +1109,7 @@ describe "String" do it { "foobar".rindex('g').should be_nil } it { "日本語日本語".rindex('本').should eq(4) } it { "あいう_えお".rindex('_').should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}').should eq(3) } describe "with offset" do it { "bbbb".rindex('b', 2).should eq(2) } @@ -1118,6 +1122,8 @@ describe "String" do it { "faobar".rindex('a', 3).should eq(1) } it { "faobarbaz".rindex('a', -3).should eq(4) } it { "日本語日本語".rindex('本', 3).should eq(1) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 4).should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 2).should be_nil } # Check offset type it { "bbbb".rindex('b', 2_i64).should eq(2) } diff --git a/src/string.cr b/src/string.cr index cf96401253b8..35c33b903939 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3349,11 +3349,21 @@ class String def index(search : Char, offset = 0) : Int32? # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.fast_index(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.upto(bytesize - 1) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.fast_index(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 @@ -3469,11 +3479,21 @@ class String def rindex(search : Char, offset = size - 1) # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.rindex(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.downto(0) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.rindex(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 From 8878c8b61e85bc4d473178252bbb41d59514e7ce Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 25 Aug 2024 19:15:54 +0800 Subject: [PATCH 066/193] Implement `System::User` on Windows (#14933) This is for the most part a straight port of [Go's implementation](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/os/user/lookup_windows.go), including their interpretation of primary groups on Windows (as opposed to [whatever Cygwin does](https://cygwin.com/cygwin-ug-net/ntsec.html)). --- spec/std/system/user_spec.cr | 38 ++- src/crystal/system/user.cr | 2 + src/crystal/system/win32/path.cr | 20 +- src/crystal/system/win32/user.cr | 273 ++++++++++++++++++ .../x86_64-windows-msvc/c/knownfolders.cr | 1 + src/lib_c/x86_64-windows-msvc/c/lm.cr | 59 ++++ src/lib_c/x86_64-windows-msvc/c/sddl.cr | 6 + src/lib_c/x86_64-windows-msvc/c/security.cr | 21 ++ src/lib_c/x86_64-windows-msvc/c/userenv.cr | 6 + src/lib_c/x86_64-windows-msvc/c/winbase.cr | 7 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 25 ++ 11 files changed, 437 insertions(+), 21 deletions(-) create mode 100644 src/crystal/system/win32/user.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/lm.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/sddl.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/security.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/userenv.cr diff --git a/spec/std/system/user_spec.cr b/spec/std/system/user_spec.cr index 9fea934bc227..f0cb977d014d 100644 --- a/spec/std/system/user_spec.cr +++ b/spec/std/system/user_spec.cr @@ -1,20 +1,36 @@ -{% skip_file if flag?(:win32) %} - require "spec" require "system/user" -USER_NAME = {{ `id -un`.stringify.chomp }} -USER_ID = {{ `id -u`.stringify.chomp }} +{% if flag?(:win32) %} + {% name, id = `whoami /USER /FO TABLE /NH`.stringify.chomp.split(" ") %} + USER_NAME = {{ name }} + USER_ID = {{ id }} +{% else %} + USER_NAME = {{ `id -un`.stringify.chomp }} + USER_ID = {{ `id -u`.stringify.chomp }} +{% end %} + INVALID_USER_NAME = "this_user_does_not_exist" INVALID_USER_ID = {% if flag?(:android) %}"8888"{% else %}"1234567"{% end %} +def normalized_username(username) + # on Windows, domain names are case-insensitive, so we unify the letter case + # from sources like `whoami`, `hostname`, or Win32 APIs + {% if flag?(:win32) %} + domain, _, user = username.partition('\\') + "#{domain.upcase}\\#{user}" + {% else %} + username + {% end %} +end + describe System::User do describe ".find_by(*, name)" do it "returns a user by name" do user = System::User.find_by(name: USER_NAME) user.should be_a(System::User) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) user.id.should eq(USER_ID) end @@ -31,7 +47,7 @@ describe System::User do user.should be_a(System::User) user.id.should eq(USER_ID) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) end it "raises on nonexistent user id" do @@ -46,7 +62,7 @@ describe System::User do user = System::User.find_by?(name: USER_NAME).not_nil! user.should be_a(System::User) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) user.id.should eq(USER_ID) end @@ -62,7 +78,7 @@ describe System::User do user.should be_a(System::User) user.id.should eq(USER_ID) - user.username.should eq(USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) end it "returns nil on nonexistent user id" do @@ -73,7 +89,8 @@ describe System::User do describe "#username" do it "is the same as the source name" do - System::User.find_by(name: USER_NAME).username.should eq(USER_NAME) + user = System::User.find_by(name: USER_NAME) + normalized_username(user.username).should eq(normalized_username(USER_NAME)) end end @@ -109,7 +126,8 @@ describe System::User do describe "#to_s" do it "returns a string representation" do - System::User.find_by(name: USER_NAME).to_s.should eq("#{USER_NAME} (#{USER_ID})") + user = System::User.find_by(name: USER_NAME) + user.to_s.should eq("#{user.username} (#{user.id})") end end end diff --git a/src/crystal/system/user.cr b/src/crystal/system/user.cr index cb3db8cda026..88766496a9d8 100644 --- a/src/crystal/system/user.cr +++ b/src/crystal/system/user.cr @@ -20,6 +20,8 @@ end require "./wasi/user" {% elsif flag?(:unix) %} require "./unix/user" +{% elsif flag?(:win32) %} + require "./win32/user" {% else %} {% raise "No Crystal::System::User implementation available" %} {% end %} diff --git a/src/crystal/system/win32/path.cr b/src/crystal/system/win32/path.cr index 06f9346a2bae..f7bb1d23191b 100644 --- a/src/crystal/system/win32/path.cr +++ b/src/crystal/system/win32/path.cr @@ -4,18 +4,16 @@ require "c/shlobj_core" module Crystal::System::Path def self.home : String - if home_path = ENV["USERPROFILE"]?.presence - home_path + ENV["USERPROFILE"]?.presence || known_folder_path(LibC::FOLDERID_Profile) + end + + def self.known_folder_path(guid : LibC::GUID) : String + if LibC.SHGetKnownFolderPath(pointerof(guid), 0, nil, out path_ptr) == 0 + path, _ = String.from_utf16(path_ptr) + LibC.CoTaskMemFree(path_ptr) + path else - # TODO: interpreter doesn't implement pointerof(Path)` yet - folderid = LibC::FOLDERID_Profile - if LibC.SHGetKnownFolderPath(pointerof(folderid), 0, nil, out path_ptr) == 0 - home_path, _ = String.from_utf16(path_ptr) - LibC.CoTaskMemFree(path_ptr) - home_path - else - raise RuntimeError.from_winerror("SHGetKnownFolderPath") - end + raise RuntimeError.from_winerror("SHGetKnownFolderPath") end end end diff --git a/src/crystal/system/win32/user.cr b/src/crystal/system/win32/user.cr new file mode 100644 index 000000000000..e5fcdbba10aa --- /dev/null +++ b/src/crystal/system/win32/user.cr @@ -0,0 +1,273 @@ +require "c/sddl" +require "c/lm" +require "c/userenv" +require "c/security" + +# This file contains source code derived from the following: +# +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/os/user/lookup_windows.go +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/syscall/security_windows.go +# +# The following is their license: +# +# Copyright 2009 The Go Authors. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Crystal::System::User + def initialize(@username : String, @id : String, @group_id : String, @name : String, @home_directory : String) + end + + def system_username + @username + end + + def system_id + @id + end + + def system_group_id + @group_id + end + + def system_name + @name + end + + def system_home_directory + @home_directory + end + + def system_shell + Crystal::System::User.cmd_path + end + + class_getter(cmd_path : String) do + "#{Crystal::System::Path.known_folder_path(LibC::FOLDERID_System)}\\cmd.exe" + end + + def self.from_username?(username : String) : ::System::User? + if found = name_to_sid(username) + if found.type.sid_type_user? + from_sid(found.sid) + end + end + end + + def self.from_id?(id : String) : ::System::User? + if sid = sid_from_s(id) + begin + from_sid(sid) + ensure + LibC.LocalFree(sid) + end + end + end + + private def self.from_sid(sid : LibC::SID*) : ::System::User? + canonical = sid_to_name(sid) || return + return unless canonical.type.sid_type_user? + + domain_and_user = "#{canonical.domain}\\#{canonical.name}" + full_name = lookup_full_name(canonical.name, canonical.domain, domain_and_user) || return + pgid = lookup_primary_group_id(canonical.name, canonical.domain) || return + uid = sid_to_s(sid) + home_dir = lookup_home_directory(uid, canonical.name) || return + + ::System::User.new(domain_and_user, uid, pgid, full_name, home_dir) + end + + private def self.lookup_full_name(name : String, domain : String, domain_and_user : String) : String? + if domain_joined? + domain_and_user = Crystal::System.to_wstr(domain_and_user) + Crystal::System.retry_wstr_buffer do |buffer, small_buf| + len = LibC::ULong.new(buffer.size) + if LibC.TranslateNameW(domain_and_user, LibC::EXTENDED_NAME_FORMAT::NameSamCompatible, LibC::EXTENDED_NAME_FORMAT::NameDisplay, buffer, pointerof(len)) != 0 + return String.from_utf16(buffer[0, len - 1]) + elsif small_buf && len > 0 + next len + else + break + end + end + end + + info = uninitialized LibC::USER_INFO_10* + if LibC.NetUserGetInfo(Crystal::System.to_wstr(domain), Crystal::System.to_wstr(name), 10, pointerof(info).as(LibC::BYTE**)) == LibC::NERR_Success + begin + str, _ = String.from_utf16(info.value.usri10_full_name) + return str + ensure + LibC.NetApiBufferFree(info) + end + end + + # domain worked neither as a domain nor as a server + # could be domain server unavailable + # pretend username is fullname + name + end + + # obtains the primary group SID for a user using this method: + # https://support.microsoft.com/en-us/help/297951/how-to-use-the-primarygroupid-attribute-to-find-the-primary-group-for + # The method follows this formula: domainRID + "-" + primaryGroupRID + private def self.lookup_primary_group_id(name : String, domain : String) : String? + domain_sid = name_to_sid(domain) || return + return unless domain_sid.type.sid_type_domain? + + domain_sid_str = sid_to_s(domain_sid.sid) + + # If the user has joined a domain use the RID of the default primary group + # called "Domain Users": + # https://support.microsoft.com/en-us/help/243330/well-known-security-identifiers-in-windows-operating-systems + # SID: S-1-5-21domain-513 + # + # The correct way to obtain the primary group of a domain user is + # probing the user primaryGroupID attribute in the server Active Directory: + # https://learn.microsoft.com/en-us/windows/win32/adschema/a-primarygroupid + # + # Note that the primary group of domain users should not be modified + # on Windows for performance reasons, even if it's possible to do that. + # The .NET Developer's Guide to Directory Services Programming - Page 409 + # https://books.google.bg/books?id=kGApqjobEfsC&lpg=PA410&ots=p7oo-eOQL7&dq=primary%20group%20RID&hl=bg&pg=PA409#v=onepage&q&f=false + return "#{domain_sid_str}-513" if domain_joined? + + # For non-domain users call NetUserGetInfo() with level 4, which + # in this case would not have any network overhead. + # The primary group should not change from RID 513 here either + # but the group will be called "None" instead: + # https://www.adampalmer.me/iodigitalsec/2013/08/10/windows-null-session-enumeration/ + # "Group 'None' (RID: 513)" + info = uninitialized LibC::USER_INFO_4* + if LibC.NetUserGetInfo(Crystal::System.to_wstr(domain), Crystal::System.to_wstr(name), 4, pointerof(info).as(LibC::BYTE**)) == LibC::NERR_Success + begin + "#{domain_sid_str}-#{info.value.usri4_primary_group_id}" + ensure + LibC.NetApiBufferFree(info) + end + end + end + + private REGISTRY_PROFILE_LIST = %q(SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList).to_utf16 + private ProfileImagePath = "ProfileImagePath".to_utf16 + + private def self.lookup_home_directory(uid : String, username : String) : String? + # If this user has logged in at least once their home path should be stored + # in the registry under the specified SID. References: + # https://social.technet.microsoft.com/wiki/contents/articles/13895.how-to-remove-a-corrupted-user-profile-from-the-registry.aspx + # https://support.asperasoft.com/hc/en-us/articles/216127438-How-to-delete-Windows-user-profiles + # + # The registry is the most reliable way to find the home path as the user + # might have decided to move it outside of the default location, + # (e.g. C:\users). Reference: + # https://answers.microsoft.com/en-us/windows/forum/windows_7-security/how-do-i-set-a-home-directory-outside-cusers-for-a/aed68262-1bf4-4a4d-93dc-7495193a440f + reg_home_dir = WindowsRegistry.open?(LibC::HKEY_LOCAL_MACHINE, REGISTRY_PROFILE_LIST) do |key_handle| + WindowsRegistry.open?(key_handle, uid.to_utf16) do |sub_handle| + WindowsRegistry.get_string(sub_handle, ProfileImagePath) + end + end + return reg_home_dir if reg_home_dir + + # If the home path does not exist in the registry, the user might + # have not logged in yet; fall back to using getProfilesDirectory(). + # Find the username based on a SID and append that to the result of + # getProfilesDirectory(). The domain is not relevant here. + # NOTE: the user has not logged in so this directory might not exist + profile_dir = Crystal::System.retry_wstr_buffer do |buffer, small_buf| + len = LibC::DWORD.new(buffer.size) + if LibC.GetProfilesDirectoryW(buffer, pointerof(len)) != 0 + break String.from_utf16(buffer[0, len - 1]) + elsif small_buf && len > 0 + next len + else + break nil + end + end + return "#{profile_dir}\\#{username}" if profile_dir + end + + private record SIDLookupResult, sid : LibC::SID*, domain : String, type : LibC::SID_NAME_USE + + private def self.name_to_sid(name : String) : SIDLookupResult? + utf16_name = Crystal::System.to_wstr(name) + + sid_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountNameW(nil, utf16_name, nil, pointerof(sid_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + sid = Pointer(UInt8).malloc(sid_size).as(LibC::SID*) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountNameW(nil, utf16_name, sid, pointerof(sid_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + domain = String.from_utf16(domain_buf[..-2]) + SIDLookupResult.new(sid, domain, sid_type) + end + end + end + + private record NameLookupResult, name : String, domain : String, type : LibC::SID_NAME_USE + + private def self.sid_to_name(sid : LibC::SID*) : NameLookupResult? + name_buf_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountSidW(nil, sid, nil, pointerof(name_buf_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + name_buf = Slice(LibC::WCHAR).new(name_buf_size) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountSidW(nil, sid, name_buf, pointerof(name_buf_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + name = String.from_utf16(name_buf[..-2]) + domain = String.from_utf16(domain_buf[..-2]) + NameLookupResult.new(name, domain, sid_type) + end + end + end + + private def self.domain_joined? : Bool + status = LibC.NetGetJoinInformation(nil, out domain, out type) + if status != LibC::NERR_Success + raise RuntimeError.from_os_error("NetGetJoinInformation", WinError.new(status)) + end + is_domain = type.net_setup_domain_name? + LibC.NetApiBufferFree(domain) + is_domain + end + + private def self.sid_to_s(sid : LibC::SID*) : String + if LibC.ConvertSidToStringSidW(sid, out ptr) == 0 + raise RuntimeError.from_winerror("ConvertSidToStringSidW") + end + str, _ = String.from_utf16(ptr) + LibC.LocalFree(ptr) + str + end + + private def self.sid_from_s(str : String) : LibC::SID* + status = LibC.ConvertStringSidToSidW(Crystal::System.to_wstr(str), out sid) + status != 0 ? sid : Pointer(LibC::SID).null + end +end diff --git a/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr b/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr index 04c16573cc76..6ce1831cb1e5 100644 --- a/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr +++ b/src/lib_c/x86_64-windows-msvc/c/knownfolders.cr @@ -2,4 +2,5 @@ require "c/guiddef" lib LibC FOLDERID_Profile = GUID.new(0x5e6c858f, 0x0e22, 0x4760, UInt8.static_array(0x9a, 0xfe, 0xea, 0x33, 0x17, 0xb6, 0x71, 0x73)) + FOLDERID_System = GUID.new(0x1ac14e77, 0x02e7, 0x4e5d, UInt8.static_array(0xb7, 0x44, 0x2e, 0xb1, 0xae, 0x51, 0x98, 0xb7)) end diff --git a/src/lib_c/x86_64-windows-msvc/c/lm.cr b/src/lib_c/x86_64-windows-msvc/c/lm.cr new file mode 100644 index 000000000000..72f5affc9b55 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/lm.cr @@ -0,0 +1,59 @@ +require "c/winnt" + +@[Link("netapi32")] +lib LibC + alias NET_API_STATUS = DWORD + + NERR_Success = NET_API_STATUS.new!(0) + + enum NETSETUP_JOIN_STATUS + NetSetupUnknownStatus = 0 + NetSetupUnjoined + NetSetupWorkgroupName + NetSetupDomainName + end + + fun NetGetJoinInformation(lpServer : LPWSTR, lpNameBuffer : LPWSTR*, bufferType : NETSETUP_JOIN_STATUS*) : NET_API_STATUS + + struct USER_INFO_4 + usri4_name : LPWSTR + usri4_password : LPWSTR + usri4_password_age : DWORD + usri4_priv : DWORD + usri4_home_dir : LPWSTR + usri4_comment : LPWSTR + usri4_flags : DWORD + usri4_script_path : LPWSTR + usri4_auth_flags : DWORD + usri4_full_name : LPWSTR + usri4_usr_comment : LPWSTR + usri4_parms : LPWSTR + usri4_workstations : LPWSTR + usri4_last_logon : DWORD + usri4_last_logoff : DWORD + usri4_acct_expires : DWORD + usri4_max_storage : DWORD + usri4_units_per_week : DWORD + usri4_logon_hours : BYTE* + usri4_bad_pw_count : DWORD + usri4_num_logons : DWORD + usri4_logon_server : LPWSTR + usri4_country_code : DWORD + usri4_code_page : DWORD + usri4_user_sid : SID* + usri4_primary_group_id : DWORD + usri4_profile : LPWSTR + usri4_home_dir_drive : LPWSTR + usri4_password_expired : DWORD + end + + struct USER_INFO_10 + usri10_name : LPWSTR + usri10_comment : LPWSTR + usri10_usr_comment : LPWSTR + usri10_full_name : LPWSTR + end + + fun NetUserGetInfo(servername : LPWSTR, username : LPWSTR, level : DWORD, bufptr : BYTE**) : NET_API_STATUS + fun NetApiBufferFree(buffer : Void*) : NET_API_STATUS +end diff --git a/src/lib_c/x86_64-windows-msvc/c/sddl.cr b/src/lib_c/x86_64-windows-msvc/c/sddl.cr new file mode 100644 index 000000000000..64e1fa8b25c1 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/sddl.cr @@ -0,0 +1,6 @@ +require "c/winnt" + +lib LibC + fun ConvertSidToStringSidW(sid : SID*, stringSid : LPWSTR*) : BOOL + fun ConvertStringSidToSidW(stringSid : LPWSTR, sid : SID**) : BOOL +end diff --git a/src/lib_c/x86_64-windows-msvc/c/security.cr b/src/lib_c/x86_64-windows-msvc/c/security.cr new file mode 100644 index 000000000000..5a904c51df40 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/security.cr @@ -0,0 +1,21 @@ +require "c/winnt" + +@[Link("secur32")] +lib LibC + enum EXTENDED_NAME_FORMAT + NameUnknown = 0 + NameFullyQualifiedDN = 1 + NameSamCompatible = 2 + NameDisplay = 3 + NameUniqueId = 6 + NameCanonical = 7 + NameUserPrincipal = 8 + NameCanonicalEx = 9 + NameServicePrincipal = 10 + NameDnsDomain = 12 + NameGivenName = 13 + NameSurname = 14 + end + + fun TranslateNameW(lpAccountName : LPWSTR, accountNameFormat : EXTENDED_NAME_FORMAT, desiredNameFormat : EXTENDED_NAME_FORMAT, lpTranslatedName : LPWSTR, nSize : ULong*) : BOOLEAN +end diff --git a/src/lib_c/x86_64-windows-msvc/c/userenv.cr b/src/lib_c/x86_64-windows-msvc/c/userenv.cr new file mode 100644 index 000000000000..bb32977d79f7 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/userenv.cr @@ -0,0 +1,6 @@ +require "c/winnt" + +@[Link("userenv")] +lib LibC + fun GetProfilesDirectoryW(lpProfileDir : LPWSTR, lpcchSize : DWORD*) : BOOL +end diff --git a/src/lib_c/x86_64-windows-msvc/c/winbase.cr b/src/lib_c/x86_64-windows-msvc/c/winbase.cr index 0a736a4fa89c..7b7a8735ddf2 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winbase.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winbase.cr @@ -4,6 +4,10 @@ require "c/int_safe" require "c/minwinbase" lib LibC + alias HLOCAL = Void* + + fun LocalFree(hMem : HLOCAL) + FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100_u32 FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200_u32 FORMAT_MESSAGE_FROM_STRING = 0x00000400_u32 @@ -69,4 +73,7 @@ lib LibC end fun GetFileInformationByHandleEx(hFile : HANDLE, fileInformationClass : FILE_INFO_BY_HANDLE_CLASS, lpFileInformation : Void*, dwBufferSize : DWORD) : BOOL + + fun LookupAccountNameW(lpSystemName : LPWSTR, lpAccountName : LPWSTR, sid : SID*, cbSid : DWORD*, referencedDomainName : LPWSTR, cchReferencedDomainName : DWORD*, peUse : SID_NAME_USE*) : BOOL + fun LookupAccountSidW(lpSystemName : LPWSTR, sid : SID*, name : LPWSTR, cchName : DWORD*, referencedDomainName : LPWSTR, cchReferencedDomainName : DWORD*, peUse : SID_NAME_USE*) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index 535ad835c87a..1db4b2def700 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -95,6 +95,31 @@ lib LibC WRITE = 0x20006 end + struct SID_IDENTIFIER_AUTHORITY + value : BYTE[6] + end + + struct SID + revision : BYTE + subAuthorityCount : BYTE + identifierAuthority : SID_IDENTIFIER_AUTHORITY + subAuthority : DWORD[1] + end + + enum SID_NAME_USE + SidTypeUser = 1 + SidTypeGroup + SidTypeDomain + SidTypeAlias + SidTypeWellKnownGroup + SidTypeDeletedAccount + SidTypeInvalid + SidTypeUnknown + SidTypeComputer + SidTypeLabel + SidTypeLogonSession + end + enum JOBOBJECTINFOCLASS AssociateCompletionPortInformation = 7 ExtendedLimitInformation = 9 From bd49e2e904a1dd0822a8343e362a50a6bc4ac719 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 26 Aug 2024 17:27:50 +0800 Subject: [PATCH 067/193] Optimize arithmetic between `BigFloat` and integers (#14944) --- src/big/big_float.cr | 65 ++++++++++++++++++++++++++++++++++++++++++++ src/big/lib_gmp.cr | 3 ++ src/big/number.cr | 22 --------------- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/src/big/big_float.cr b/src/big/big_float.cr index cadc91282fc1..2c567f21eec9 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -115,18 +115,60 @@ struct BigFloat < Float BigFloat.new { |mpf| LibGMP.mpf_neg(mpf, self) } end + def +(other : Int::Primitive) : BigFloat + Int.primitive_ui_check(other) do |ui, neg_ui, big_i| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_add_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_sub_ui(mpf, self, {{ neg_ui }}) }, + big_i: self + {{ big_i }}, + } + end + end + def +(other : Number) : BigFloat BigFloat.new { |mpf| LibGMP.mpf_add(mpf, self, other.to_big_f) } end + def -(other : Int::Primitive) : BigFloat + Int.primitive_ui_check(other) do |ui, neg_ui, big_i| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_sub_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_add_ui(mpf, self, {{ neg_ui }}) }, + big_i: self - {{ big_i }}, + } + end + end + def -(other : Number) : BigFloat BigFloat.new { |mpf| LibGMP.mpf_sub(mpf, self, other.to_big_f) } end + def *(other : Int::Primitive) : BigFloat + Int.primitive_ui_check(other) do |ui, neg_ui, big_i| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_mul_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_mul_ui(mpf, self, {{ neg_ui }}); LibGMP.mpf_neg(mpf, mpf) }, + big_i: self + {{ big_i }}, + } + end + end + def *(other : Number) : BigFloat BigFloat.new { |mpf| LibGMP.mpf_mul(mpf, self, other.to_big_f) } end + def /(other : Int::Primitive) : BigFloat + # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity + raise DivisionByZeroError.new if other == 0 + Int.primitive_ui_check(other) do |ui, neg_ui, _| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ neg_ui }}); LibGMP.mpf_neg(mpf, mpf) }, + big_i: BigFloat.new { |mpf| LibGMP.mpf_div(mpf, self, BigFloat.new(other)) }, + } + end + end + def /(other : BigFloat) : BigFloat # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity raise DivisionByZeroError.new if other == 0 @@ -448,6 +490,29 @@ struct Int def <=>(other : BigFloat) -(other <=> self) end + + def -(other : BigFloat) : BigFloat + Int.primitive_ui_check(self) do |ui, neg_ui, _| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_neg(mpf, other); LibGMP.mpf_add_ui(mpf, mpf, {{ ui }}) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_neg(mpf, other); LibGMP.mpf_sub_ui(mpf, mpf, {{ neg_ui }}) }, + big_i: BigFloat.new { |mpf| LibGMP.mpf_sub(mpf, BigFloat.new(self), other) }, + } + end + end + + def /(other : BigFloat) : BigFloat + # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity + raise DivisionByZeroError.new if other == 0 + + Int.primitive_ui_check(self) do |ui, neg_ui, _| + { + ui: BigFloat.new { |mpf| LibGMP.mpf_ui_div(mpf, {{ ui }}, other) }, + neg_ui: BigFloat.new { |mpf| LibGMP.mpf_ui_div(mpf, {{ neg_ui }}, other); LibGMP.mpf_neg(mpf, mpf) }, + big_i: BigFloat.new { |mpf| LibGMP.mpf_div(mpf, BigFloat.new(self), other) }, + } + end + end end struct Float diff --git a/src/big/lib_gmp.cr b/src/big/lib_gmp.cr index 00834598d9d2..5ae18b5a4606 100644 --- a/src/big/lib_gmp.cr +++ b/src/big/lib_gmp.cr @@ -233,8 +233,11 @@ lib LibGMP # # Arithmetic fun mpf_add = __gmpf_add(rop : MPF*, op1 : MPF*, op2 : MPF*) + fun mpf_add_ui = __gmpf_add_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_sub = __gmpf_sub(rop : MPF*, op1 : MPF*, op2 : MPF*) + fun mpf_sub_ui = __gmpf_sub_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_mul = __gmpf_mul(rop : MPF*, op1 : MPF*, op2 : MPF*) + fun mpf_mul_ui = __gmpf_mul_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_div = __gmpf_div(rop : MPF*, op1 : MPF*, op2 : MPF*) fun mpf_div_ui = __gmpf_div_ui(rop : MPF*, op1 : MPF*, op2 : UI) fun mpf_ui_div = __gmpf_ui_div(rop : MPF*, op1 : UI, op2 : MPF*) diff --git a/src/big/number.cr b/src/big/number.cr index 1251e8113db3..8761a2aa8b6c 100644 --- a/src/big/number.cr +++ b/src/big/number.cr @@ -8,18 +8,6 @@ struct BigFloat self.class.new(self / other) end - def /(other : Int::Primitive) : BigFloat - # Division by 0 in BigFloat is not allowed, there is no BigFloat::Infinity - raise DivisionByZeroError.new if other == 0 - Int.primitive_ui_check(other) do |ui, neg_ui, _| - { - ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ ui }}) }, - neg_ui: BigFloat.new { |mpf| LibGMP.mpf_div_ui(mpf, self, {{ neg_ui }}); LibGMP.mpf_neg(mpf, mpf) }, - big_i: BigFloat.new { |mpf| LibGMP.mpf_div(mpf, self, BigFloat.new(other)) }, - } - end - end - Number.expand_div [Float32, Float64], BigFloat end @@ -91,70 +79,60 @@ end struct Int8 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int16 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int32 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int64 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct Int128 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt8 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt16 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt32 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt64 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end struct UInt128 Number.expand_div [BigInt], BigFloat - Number.expand_div [BigFloat], BigFloat Number.expand_div [BigDecimal], BigDecimal Number.expand_div [BigRational], BigRational end From 791b0e451766503e4a8d2b63fd1bc726b9948276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Mon, 26 Aug 2024 11:29:00 +0200 Subject: [PATCH 068/193] Add nodoc filter to doc type methods (#14910) The doc generator is creating links to non-documented type. This patch adds filters on `Doc::Type#ancestors`, `Doc::Type#included_modules` and `Doc::Type#extended_modules` as it was already done in `Doc::Type#subclasses`. --- spec/compiler/crystal/tools/doc/type_spec.cr | 136 +++++++++++++++++++ src/compiler/crystal/tools/doc/type.cr | 3 + 2 files changed, 139 insertions(+) diff --git a/spec/compiler/crystal/tools/doc/type_spec.cr b/spec/compiler/crystal/tools/doc/type_spec.cr index 34ab535f6d5e..c5dde7d4b258 100644 --- a/spec/compiler/crystal/tools/doc/type_spec.cr +++ b/spec/compiler/crystal/tools/doc/type_spec.cr @@ -212,4 +212,140 @@ describe Doc::Type do type.macros.map(&.name).should eq ["+", "~", "foo"] end end + + describe "#subclasses" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + class Foo + end + + class Bar < Foo + end + + # :nodoc: + class Baz < Foo + end + + module Mod1 + class Bar < ::Foo + end + end + + # :nodoc: + module Mod2 + class Baz < ::Foo + end + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.subclasses.map(&.full_name).should eq ["Bar", "Mod1::Bar"] + end + end + + describe "#ancestors" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + module Mod3 + class Baz + end + end + + class Mod2::Baz < Mod3::Baz + end + + module Mod1 + # :nodoc: + class Baz < Mod2::Baz + end + end + + class Baz < Mod1::Baz + end + + class Foo < Baz + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.ancestors.map(&.full_name).should eq ["Baz", "Mod2::Baz"] + end + end + + describe "#included_modules" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + module Mod3 + module Baz + end + end + + module Mod2 + # :nodoc: + module Baz + end + end + + module Mod1 + module Baz + end + end + + module Baz + end + + class Foo + include Baz + include Mod1::Baz + include Mod2::Baz + include Mod3::Baz + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.included_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] + end + end + + describe "#included_modules" do + it "only include types with docs" do + program = semantic(<<-CRYSTAL, wants_doc: true).program + # :nodoc: + module Mod3 + module Baz + end + end + + module Mod2 + # :nodoc: + module Baz + end + end + + module Mod1 + module Baz + end + end + + module Baz + end + + class Foo + extend Baz + extend Mod1::Baz + extend Mod2::Baz + extend Mod3::Baz + end + CRYSTAL + + generator = Doc::Generator.new program, [""] + type = generator.type(program.types["Foo"]) + type.extended_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] + end + end end diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 624c8f017fe7..9b1a0a86cf7e 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -117,6 +117,7 @@ class Crystal::Doc::Type unless ast_node? @type.ancestors.each do |ancestor| + next unless @generator.must_include? ancestor doc_type = @generator.type(ancestor) ancestors << doc_type break if ancestor == @generator.program.object || doc_type.ast_node? @@ -258,6 +259,7 @@ class Crystal::Doc::Type included_modules = [] of Type @type.parents.try &.each do |parent| if parent.module? + next unless @generator.must_include? parent included_modules << @generator.type(parent) end end @@ -272,6 +274,7 @@ class Crystal::Doc::Type extended_modules = [] of Type @type.metaclass.parents.try &.each do |parent| if parent.module? + next unless @generator.must_include? parent extended_modules << @generator.type(parent) end end From c015ff6388bb3023e92668baccf2088bf067381b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 2 Sep 2024 20:33:21 +0800 Subject: [PATCH 069/193] Support non-blocking `File#read` and `#write` on Windows (#14940) --- spec/std/file_spec.cr | 16 +++++++++++++ src/crystal/system/win32/event_loop_iocp.cr | 11 ++++----- src/crystal/system/win32/iocp.cr | 26 +++++++++++++++++---- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 96dbacd73cc9..55a7b5d76494 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -122,6 +122,22 @@ describe "File" do end {% end %} {% end %} + + it "reads non-blocking file" do + File.open(datapath("test_file.txt"), "r", blocking: false) do |f| + f.gets_to_end.should eq("Hello World\n" * 20) + end + end + + it "writes and reads large non-blocking file" do + with_tempfile("non-blocking-io.txt") do |path| + File.open(path, "w+", blocking: false) do |f| + f.puts "Hello World\n" * 40000 + f.pos = 0 + f.gets_to_end.should eq("Hello World\n" * 40000) + end + end + end end it "reads entire file" do diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index 25c8db41d9ff..6f9a921ad8d3 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -144,18 +144,15 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop end def read(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - handle = file_descriptor.windows_handle - IOCP.overlapped_operation(file_descriptor, handle, "ReadFile", file_descriptor.read_timeout) do |overlapped| - ret = LibC.ReadFile(handle, slice, slice.size, out byte_count, overlapped) + IOCP.overlapped_operation(file_descriptor, "ReadFile", file_descriptor.read_timeout) do |overlapped| + ret = LibC.ReadFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) {ret, byte_count} end.to_i32 end def write(file_descriptor : Crystal::System::FileDescriptor, slice : Bytes) : Int32 - handle = file_descriptor.windows_handle - - IOCP.overlapped_operation(file_descriptor, handle, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| - ret = LibC.WriteFile(handle, slice, slice.size, out byte_count, overlapped) + IOCP.overlapped_operation(file_descriptor, "WriteFile", file_descriptor.write_timeout, writing: true) do |overlapped| + ret = LibC.WriteFile(file_descriptor.windows_handle, slice, slice.size, out byte_count, overlapped) {ret, byte_count} end.to_i32 end diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index add5a29c2814..af8f778290f3 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -168,8 +168,16 @@ module Crystal::IOCP end end - def self.overlapped_operation(target, handle, method, timeout, *, writing = false, &) + def self.overlapped_operation(file_descriptor, method, timeout, *, writing = false, &) + handle = file_descriptor.windows_handle + seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 + OverlappedOperation.run(handle) do |operation| + overlapped = operation.to_unsafe + if seekable + overlapped.value.union.offset.offset = LibC::DWORD.new!(original_offset) + overlapped.value.union.offset.offsetHigh = LibC::DWORD.new!(original_offset >> 32) + end result, value = yield operation if result == 0 @@ -181,15 +189,19 @@ module Crystal::IOCP when .error_io_pending? # the operation is running asynchronously; do nothing when .error_access_denied? - raise IO::Error.new "File not open for #{writing ? "writing" : "reading"}", target: target + raise IO::Error.new "File not open for #{writing ? "writing" : "reading"}", target: file_descriptor else - raise IO::Error.from_os_error(method, error, target: target) + raise IO::Error.from_os_error(method, error, target: file_descriptor) end else + # operation completed synchronously; seek forward by number of bytes + # read or written if handle is seekable, since overlapped I/O doesn't do + # it automatically + LibC.SetFilePointerEx(handle, value, nil, IO::Seek::Current) if seekable return value end - operation.wait_for_result(timeout) do |error| + byte_count = operation.wait_for_result(timeout) do |error| case error when .error_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") @@ -200,6 +212,12 @@ module Crystal::IOCP return 0_u32 end end + + # operation completed asynchronously; seek to the original file position + # plus the number of bytes read or written (other operations might have + # moved the file pointer so we don't use `IO::Seek::Current` here) + LibC.SetFilePointerEx(handle, original_offset + byte_count, nil, IO::Seek::Set) if seekable + byte_count end end From e6b5b949f2e4bce024fc9d039de167b59d00d75a Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Mon, 2 Sep 2024 07:35:41 -0500 Subject: [PATCH 070/193] Optimize `Hash#transform_{keys,values}` (#14502) --- src/hash.cr | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hash.cr b/src/hash.cr index e2fe7dad186c..9b2936ddd618 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -1747,7 +1747,8 @@ class Hash(K, V) # hash.transform_keys { |key, value| key.to_s * value } # => {"a" => 1, "bb" => 2, "ccc" => 3} # ``` def transform_keys(& : K, V -> K2) : Hash(K2, V) forall K2 - each_with_object({} of K2 => V) do |(key, value), memo| + copy = Hash(K2, V).new(initial_capacity: entries_capacity) + each_with_object(copy) do |(key, value), memo| memo[yield(key, value)] = value end end @@ -1762,7 +1763,8 @@ class Hash(K, V) # hash.transform_values { |value, key| "#{key}#{value}" } # => {:a => "a1", :b => "b2", :c => "c3"} # ``` def transform_values(& : V, K -> V2) : Hash(K, V2) forall V2 - each_with_object({} of K => V2) do |(key, value), memo| + copy = Hash(K, V2).new(initial_capacity: entries_capacity) + each_with_object(copy) do |(key, value), memo| memo[key] = yield(value, key) end end From 281fc3233d06331975e491ebcbbc69521cd4c65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 3 Sep 2024 13:21:44 +0200 Subject: [PATCH 071/193] Add specs for `String#index`, `#rindex` search for `Char::REPLACEMENT` (#14946) Co-authored-by: Quinton Miller --- spec/std/string_spec.cr | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 5b70deda13c3..2ffe5bf3d1fa 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -958,6 +958,7 @@ describe "String" do it { "bar".index('あ').should be_nil } it { "あいう_えお".index('_').should eq(3) } it { "xyz\xFFxyz".index('\u{FFFD}').should eq(3) } + it { "日\xFF語".index('\u{FFFD}').should eq(1) } describe "with offset" do it { "foobarbaz".index('a', 5).should eq(7) } @@ -967,6 +968,8 @@ describe "String" do it { "日本語日本語".index('本', 2).should eq(4) } it { "xyz\xFFxyz".index('\u{FFFD}', 2).should eq(3) } it { "xyz\xFFxyz".index('\u{FFFD}', 4).should be_nil } + it { "日本\xFF語".index('\u{FFFD}', 2).should eq(2) } + it { "日本\xFF語".index('\u{FFFD}', 3).should be_nil } # Check offset type it { "foobarbaz".index('a', 5_i64).should eq(7) } @@ -1110,6 +1113,7 @@ describe "String" do it { "日本語日本語".rindex('本').should eq(4) } it { "あいう_えお".rindex('_').should eq(3) } it { "xyz\xFFxyz".rindex('\u{FFFD}').should eq(3) } + it { "日\xFF語".rindex('\u{FFFD}').should eq(1) } describe "with offset" do it { "bbbb".rindex('b', 2).should eq(2) } @@ -1124,6 +1128,8 @@ describe "String" do it { "日本語日本語".rindex('本', 3).should eq(1) } it { "xyz\xFFxyz".rindex('\u{FFFD}', 4).should eq(3) } it { "xyz\xFFxyz".rindex('\u{FFFD}', 2).should be_nil } + it { "日本\xFF語".rindex('\u{FFFD}', 2).should eq(2) } + it { "日本\xFF語".rindex('\u{FFFD}', 1).should be_nil } # Check offset type it { "bbbb".rindex('b', 2_i64).should eq(2) } From 598931c66700432e8ec1c7b9032791bfeae6364f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 3 Sep 2024 19:21:55 +0800 Subject: [PATCH 072/193] Allow `^` in constant numeric expressions (#14951) --- spec/compiler/codegen/c_enum_spec.cr | 7 +++++++ src/compiler/crystal/semantic/math_interpreter.cr | 1 + 2 files changed, 8 insertions(+) diff --git a/spec/compiler/codegen/c_enum_spec.cr b/spec/compiler/codegen/c_enum_spec.cr index c5197799d2cf..75c9966c6c10 100644 --- a/spec/compiler/codegen/c_enum_spec.cr +++ b/spec/compiler/codegen/c_enum_spec.cr @@ -20,15 +20,22 @@ describe "Code gen: c enum" do end [ + {"+1", 1}, + {"-1", -1}, + {"~1", -2}, {"1 + 2", 3}, {"3 - 2", 1}, {"3 * 2", 6}, + {"1 &+ 2", 3}, + {"3 &- 2", 1}, + {"3 &* 2", 6}, # {"10 / 2", 5}, # MathInterpreter only works with Integer and 10 / 2 : Float {"10 // 2", 5}, {"1 << 3", 8}, {"100 >> 3", 12}, {"10 & 3", 2}, {"10 | 3", 11}, + {"10 ^ 3", 9}, {"(1 + 2) * 3", 9}, {"10 % 3", 1}, ].each do |(code, expected)| diff --git a/src/compiler/crystal/semantic/math_interpreter.cr b/src/compiler/crystal/semantic/math_interpreter.cr index c39d290aa1e9..d6846e420a7b 100644 --- a/src/compiler/crystal/semantic/math_interpreter.cr +++ b/src/compiler/crystal/semantic/math_interpreter.cr @@ -73,6 +73,7 @@ struct Crystal::MathInterpreter when "//" then left // right when "&" then left & right when "|" then left | right + when "^" then left ^ right when "<<" then left << right when ">>" then left >> right when "%" then left % right From 6ee4eb92f35858585b15279a061bcaba284774c5 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 3 Sep 2024 10:04:13 -0300 Subject: [PATCH 073/193] Add `Crystal::Repl::Value#runtime_type` (#14156) Co-authored-by: Sijawusz Pur Rahnama --- spec/compiler/crystal/tools/repl_spec.cr | 49 +++++++++++++++++++ .../crystal/interpreter/primitives.cr | 2 + src/compiler/crystal/interpreter/value.cr | 15 ++++++ 3 files changed, 66 insertions(+) diff --git a/spec/compiler/crystal/tools/repl_spec.cr b/spec/compiler/crystal/tools/repl_spec.cr index 3a1e1275ef12..7a387624f8fa 100644 --- a/spec/compiler/crystal/tools/repl_spec.cr +++ b/spec/compiler/crystal/tools/repl_spec.cr @@ -17,4 +17,53 @@ describe Crystal::Repl do success_value(repl.parse_and_interpret("def foo; 1 + 2; end")).value.should eq(nil) success_value(repl.parse_and_interpret("foo")).value.should eq(3) end + + describe "can return static and runtime type information for" do + it "Non Union" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl_value = success_value(repl.parse_and_interpret("1")) + repl_value.type.to_s.should eq("Int32") + repl_value.runtime_type.to_s.should eq("Int32") + end + + it "MixedUnionType" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl_value = success_value(repl.parse_and_interpret("1 || \"a\"")) + repl_value.type.to_s.should eq("(Int32 | String)") + repl_value.runtime_type.to_s.should eq("Int32") + end + + it "UnionType" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl_value = success_value(repl.parse_and_interpret("true || 1")) + repl_value.type.to_s.should eq("(Bool | Int32)") + repl_value.runtime_type.to_s.should eq("Bool") + end + + it "VirtualType" do + repl = Crystal::Repl.new + repl.prelude = "primitives" + repl.load_prelude + + repl.parse_and_interpret <<-CRYSTAL + class Foo + end + + class Bar < Foo + end + CRYSTAL + repl_value = success_value(repl.parse_and_interpret("Bar.new || Foo.new")) + repl_value.type.to_s.should eq("Foo+") # Maybe should Foo to match typeof + repl_value.runtime_type.to_s.should eq("Bar") + end + end end diff --git a/src/compiler/crystal/interpreter/primitives.cr b/src/compiler/crystal/interpreter/primitives.cr index e411229600f9..7ad508f8d0fc 100644 --- a/src/compiler/crystal/interpreter/primitives.cr +++ b/src/compiler/crystal/interpreter/primitives.cr @@ -87,6 +87,8 @@ class Crystal::Repl::Compiler pointer_add(inner_sizeof_type(element_type), node: node) when "class" + # Should match Crystal::Repl::Value#runtime_type + # in src/compiler/crystal/interpreter/value.cr obj = obj.not_nil! type = obj.type.remove_indirection diff --git a/src/compiler/crystal/interpreter/value.cr b/src/compiler/crystal/interpreter/value.cr index 349dff00f78b..681798bf7a32 100644 --- a/src/compiler/crystal/interpreter/value.cr +++ b/src/compiler/crystal/interpreter/value.cr @@ -67,6 +67,21 @@ struct Crystal::Repl::Value end end + def runtime_type : Crystal::Type + # Should match Crystal::Repl::Compiler#visit_primitive "class" case + # in src/compiler/crystal/interpreter/primitives.cr + case type + when Crystal::UnionType + type_id = @pointer.as(Int32*).value + context.type_from_id(type_id) + when Crystal::VirtualType + type_id = @pointer.as(Void**).value.as(Int32*).value + context.type_from_id(type_id) + else + type + end + end + # Copies the contents of this value to another pointer. def copy_to(pointer : Pointer(UInt8)) @pointer.copy_to(pointer, context.inner_sizeof_type(@type)) From 18c28f9b2f643604d839d7c03508b6420b62f65e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 01:59:18 +0800 Subject: [PATCH 074/193] Support non-blocking `File#read_at` on Windows (#14958) --- spec/std/file_spec.cr | 20 +++++++------ src/crystal/system/unix/file_descriptor.cr | 6 ++-- src/crystal/system/win32/file_descriptor.cr | 33 +++++++++++---------- src/crystal/system/win32/iocp.cr | 14 +++++---- src/file/preader.cr | 2 +- 5 files changed, 42 insertions(+), 33 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 55a7b5d76494..eb740885cd69 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1295,17 +1295,19 @@ describe "File" do it "reads at offset" do filename = datapath("test_file.txt") - File.open(filename) do |file| - file.read_at(6, 100) do |io| - io.gets_to_end.should eq("World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello Worl") - end + {true, false}.each do |blocking| + File.open(filename, blocking: blocking) do |file| + file.read_at(6, 100) do |io| + io.gets_to_end.should eq("World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello World\nHello Worl") + end - file.read_at(0, 240) do |io| - io.gets_to_end.should eq(File.read(filename)) - end + file.read_at(0, 240) do |io| + io.gets_to_end.should eq(File.read(filename)) + end - file.read_at(6_i64, 5_i64) do |io| - io.gets_to_end.should eq("World") + file.read_at(6_i64, 5_i64) do |io| + io.gets_to_end.should eq("World") + end end end end diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index d235114849b4..fc8839ac9e83 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -219,11 +219,11 @@ module Crystal::System::FileDescriptor {r, w} end - def self.pread(fd, buffer, offset) - bytes_read = LibC.pread(fd, buffer, buffer.size, offset).to_i64 + def self.pread(file, buffer, offset) + bytes_read = LibC.pread(file.fd, buffer, buffer.size, offset).to_i64 if bytes_read == -1 - raise IO::Error.from_errno "Error reading file" + raise IO::Error.from_errno("Error reading file", target: file) end bytes_read diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 37813307191f..f4e9200a0488 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -120,10 +120,6 @@ module Crystal::System::FileDescriptor end protected def windows_handle - FileDescriptor.windows_handle(fd) - end - - def self.windows_handle(fd) LibC::HANDLE.new(fd) end @@ -278,19 +274,26 @@ module Crystal::System::FileDescriptor {r, w} end - def self.pread(fd, buffer, offset) - handle = windows_handle(fd) + def self.pread(file, buffer, offset) + handle = file.windows_handle - overlapped = LibC::OVERLAPPED.new - overlapped.union.offset.offset = LibC::DWORD.new!(offset) - overlapped.union.offset.offsetHigh = LibC::DWORD.new!(offset >> 32) - if LibC.ReadFile(handle, buffer, buffer.size, out bytes_read, pointerof(overlapped)) == 0 - error = WinError.value - return 0_i64 if error == WinError::ERROR_HANDLE_EOF - raise IO::Error.from_os_error "Error reading file", error, target: self - end + if file.system_blocking? + overlapped = LibC::OVERLAPPED.new + overlapped.union.offset.offset = LibC::DWORD.new!(offset) + overlapped.union.offset.offsetHigh = LibC::DWORD.new!(offset >> 32) + if LibC.ReadFile(handle, buffer, buffer.size, out bytes_read, pointerof(overlapped)) == 0 + error = WinError.value + return 0_i64 if error == WinError::ERROR_HANDLE_EOF + raise IO::Error.from_os_error "Error reading file", error, target: file + end - bytes_read.to_i64 + bytes_read.to_i64 + else + IOCP.overlapped_operation(file, "ReadFile", file.read_timeout, offset: offset) do |overlapped| + ret = LibC.ReadFile(handle, buffer, buffer.size, out byte_count, overlapped) + {ret, byte_count} + end.to_i64 + end end def self.from_stdio(fd) diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index af8f778290f3..6f5746954277 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -168,15 +168,16 @@ module Crystal::IOCP end end - def self.overlapped_operation(file_descriptor, method, timeout, *, writing = false, &) + def self.overlapped_operation(file_descriptor, method, timeout, *, offset = nil, writing = false, &) handle = file_descriptor.windows_handle seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 OverlappedOperation.run(handle) do |operation| overlapped = operation.to_unsafe if seekable - overlapped.value.union.offset.offset = LibC::DWORD.new!(original_offset) - overlapped.value.union.offset.offsetHigh = LibC::DWORD.new!(original_offset >> 32) + start_offset = offset || original_offset + overlapped.value.union.offset.offset = LibC::DWORD.new!(start_offset) + overlapped.value.union.offset.offsetHigh = LibC::DWORD.new!(start_offset >> 32) end result, value = yield operation @@ -215,8 +216,11 @@ module Crystal::IOCP # operation completed asynchronously; seek to the original file position # plus the number of bytes read or written (other operations might have - # moved the file pointer so we don't use `IO::Seek::Current` here) - LibC.SetFilePointerEx(handle, original_offset + byte_count, nil, IO::Seek::Set) if seekable + # moved the file pointer so we don't use `IO::Seek::Current` here), unless + # we are calling `Crystal::System::FileDescriptor.pread` + if seekable && !offset + LibC.SetFilePointerEx(handle, original_offset + byte_count, nil, IO::Seek::Set) + end byte_count end end diff --git a/src/file/preader.cr b/src/file/preader.cr index d366457314ce..9f7d09643305 100644 --- a/src/file/preader.cr +++ b/src/file/preader.cr @@ -20,7 +20,7 @@ class File::PReader < IO count = slice.size count = Math.min(count, @bytesize - @pos) - bytes_read = Crystal::System::FileDescriptor.pread(@file.fd, slice[0, count], @offset + @pos) + bytes_read = Crystal::System::FileDescriptor.pread(@file, slice[0, count], @offset + @pos) @pos += bytes_read From e9b86d00f54dcf1fc5c92e37217fc298ee225f8f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 01:59:29 +0800 Subject: [PATCH 075/193] Update REPLy version (#14950) --- lib/.shards.info | 2 +- lib/reply/shard.yml | 2 +- lib/reply/spec/reader_spec.cr | 33 ++++++++++++++++++- lib/reply/spec/spec_helper.cr | 21 ++++++++++++ lib/reply/src/char_reader.cr | 25 +-------------- lib/reply/src/reader.cr | 60 ++++++++++++++++++++++++----------- lib/reply/src/term_size.cr | 3 -- shard.lock | 2 +- shard.yml | 2 +- 9 files changed, 99 insertions(+), 51 deletions(-) diff --git a/lib/.shards.info b/lib/.shards.info index 7f03bb906410..b6371e9397c4 100644 --- a/lib/.shards.info +++ b/lib/.shards.info @@ -6,4 +6,4 @@ shards: version: 0.5.0 reply: git: https://github.com/i3oris/reply.git - version: 0.3.1+git.commit.90a7eb5a76048884d5d56bf6b9369f1e67fdbcd7 + version: 0.3.1+git.commit.db423dae3dd34c6ba5e36174653a0c109117a167 diff --git a/lib/reply/shard.yml b/lib/reply/shard.yml index e6cd9dab283a..02a0d3490923 100644 --- a/lib/reply/shard.yml +++ b/lib/reply/shard.yml @@ -5,7 +5,7 @@ description: "Shard to create a REPL interface" authors: - I3oris -crystal: 1.5.0 +crystal: 1.13.0 license: MIT diff --git a/lib/reply/spec/reader_spec.cr b/lib/reply/spec/reader_spec.cr index 4e9f446f3de0..4dbc53cbb51b 100644 --- a/lib/reply/spec/reader_spec.cr +++ b/lib/reply/spec/reader_spec.cr @@ -254,7 +254,7 @@ module Reply reader.auto_completion.verify(open: true, entries: %w(hello hey), name_filter: "h", selection_pos: 0) reader.editor.verify("42.hello") - SpecHelper.send(pipe_in, "\e\t") # shit_tab + SpecHelper.send(pipe_in, "\e\t") # shift_tab reader.auto_completion.verify(open: true, entries: %w(hello hey), name_filter: "h", selection_pos: 1) reader.editor.verify("42.hey") @@ -298,6 +298,37 @@ module Reply SpecHelper.send(pipe_in, '\0') end + it "retriggers auto-completion when current word ends with ':'" do + reader = SpecHelper.reader(SpecReaderWithAutoCompletionRetrigger) + pipe_out, pipe_in = IO.pipe + + spawn do + reader.read_next(from: pipe_out) + end + + SpecHelper.send(pipe_in, "fo") + SpecHelper.send(pipe_in, '\t') + reader.auto_completion.verify(open: true, entries: %w(foo foobar), name_filter: "fo") + reader.editor.verify("foo") + + SpecHelper.send(pipe_in, ':') + SpecHelper.send(pipe_in, ':') + reader.auto_completion.verify(open: true, entries: %w(foo::foo foo::foobar foo::bar), name_filter: "foo::") + reader.editor.verify("foo::") + + SpecHelper.send(pipe_in, 'b') + SpecHelper.send(pipe_in, '\t') + reader.auto_completion.verify(open: true, entries: %w(foo::bar), name_filter: "foo::b", selection_pos: 0) + reader.editor.verify("foo::bar") + + SpecHelper.send(pipe_in, ':') + SpecHelper.send(pipe_in, ':') + reader.auto_completion.verify(open: true, entries: %w(foo::bar::foo foo::bar::foobar foo::bar::bar), name_filter: "foo::bar::") + reader.editor.verify("foo::bar::") + + SpecHelper.send(pipe_in, '\0') + end + it "uses escape" do reader = SpecHelper.reader pipe_out, pipe_in = IO.pipe diff --git a/lib/reply/spec/spec_helper.cr b/lib/reply/spec/spec_helper.cr index 432220b98f98..7e0a93052320 100644 --- a/lib/reply/spec/spec_helper.cr +++ b/lib/reply/spec/spec_helper.cr @@ -94,6 +94,27 @@ module Reply getter auto_completion end + class SpecReaderWithAutoCompletionRetrigger < Reader + def initialize + super + self.word_delimiters.delete(':') + end + + def auto_complete(current_word : String, expression_before : String) + if current_word.ends_with? "::" + return "title", ["#{current_word}foo", "#{current_word}foobar", "#{current_word}bar"] + else + return "title", %w(foo foobar bar) + end + end + + def auto_completion_retrigger_when(current_word : String) : Bool + current_word.ends_with? ':' + end + + getter auto_completion + end + module SpecHelper def self.auto_completion(returning results) results = results.clone diff --git a/lib/reply/src/char_reader.cr b/lib/reply/src/char_reader.cr index 3da5ca06d804..c4ab01ca802e 100644 --- a/lib/reply/src/char_reader.cr +++ b/lib/reply/src/char_reader.cr @@ -43,20 +43,9 @@ module Reply @slice_buffer = Bytes.new(buffer_size) end - def read_char(from io : T = STDIN) forall T - {% if flag?(:win32) && T <= IO::FileDescriptor %} - handle = LibC._get_osfhandle(io.fd) - raise RuntimeError.from_errno("_get_osfhandle") if handle == -1 - - raw(io) do - LibC.ReadConsoleA(LibC::HANDLE.new(handle), @slice_buffer, @slice_buffer.size, out nb_read, nil) - - parse_escape_sequence(@slice_buffer[0...nb_read]) - end - {% else %} + def read_char(from io : IO = STDIN) nb_read = raw(io, &.read(@slice_buffer)) parse_escape_sequence(@slice_buffer[0...nb_read]) - {% end %} end private def parse_escape_sequence(chars : Bytes) : Char | Sequence | String? @@ -184,15 +173,3 @@ module Reply end end end - -{% if flag?(:win32) %} - lib LibC - STD_INPUT_HANDLE = -10 - - fun ReadConsoleA(hConsoleInput : Void*, - lpBuffer : Void*, - nNumberOfCharsToRead : UInt32, - lpNumberOfCharsRead : UInt32*, - pInputControl : Void*) : UInt8 - end -{% end %} diff --git a/lib/reply/src/reader.cr b/lib/reply/src/reader.cr index f8bb5bbb03fd..01228cf7027a 100644 --- a/lib/reply/src/reader.cr +++ b/lib/reply/src/reader.cr @@ -168,6 +168,13 @@ module Reply @auto_completion.default_display_selected_entry(io, entry) end + # Override to retrigger auto completion when condition is met. + # + # default: `false` + def auto_completion_retrigger_when(current_word : String) : Bool + false + end + # Override to enable line re-indenting. # # This methods is called each time a character is entered. @@ -240,8 +247,11 @@ module Reply if read.is_a?(CharReader::Sequence) && (read.tab? || read.enter? || read.alt_enter? || read.shift_tab? || read.escape? || read.backspace? || read.ctrl_c?) else if @auto_completion.open? - auto_complete_insert_char(read) - @editor.update + replacement = auto_complete_insert_char(read) + # Replace the current_word by the replacement word + @editor.update do + @editor.current_word = replacement if replacement + end end end end @@ -362,12 +372,6 @@ module Reply end private def on_tab(shift_tab = false) - line = @editor.current_line - - # Retrieve the word under the cursor - word_begin, word_end = @editor.current_word_begin_end - current_word = line[word_begin..word_end] - if @auto_completion.open? if shift_tab replacement = @auto_completion.selection_previous @@ -375,15 +379,7 @@ module Reply replacement = @auto_completion.selection_next end else - # Get whole expression before cursor, allow auto-completion to deduce the receiver type - expr = @editor.expression_before_cursor(x: word_begin) - - # Compute auto-completion, return `replacement` (`nil` if no entry, full name if only one entry, or the begin match of entries otherwise) - replacement = @auto_completion.complete_on(current_word, expr) - - if replacement && @auto_completion.entries.size >= 2 - @auto_completion.open - end + replacement = compute_completions end # Replace the current_word by the replacement word @@ -405,14 +401,40 @@ module Reply @editor.move_cursor_to_end end - private def auto_complete_insert_char(char) + private def compute_completions : String? + line = @editor.current_line + + # Retrieve the word under the cursor + word_begin, word_end = @editor.current_word_begin_end + current_word = line[word_begin..word_end] + + expr = @editor.expression_before_cursor(x: word_begin) + + # Compute auto-completion, return `replacement` (`nil` if no entry, full name if only one entry, or the begin match of entries otherwise) + replacement = @auto_completion.complete_on(current_word, expr) + + if replacement + if @auto_completion.entries.size >= 2 + @auto_completion.open + else + @auto_completion.name_filter = replacement + end + end + + replacement + end + + private def auto_complete_insert_char(char) : String? if char.is_a? Char && !char.in?(@editor.word_delimiters) - @auto_completion.name_filter = @editor.current_word + @auto_completion.name_filter = current_word = @editor.current_word + + return compute_completions if auto_completion_retrigger_when(current_word + char) elsif @editor.expression_scrolled? || char.is_a?(String) @auto_completion.close else @auto_completion.clear end + nil end private def auto_complete_remove_char diff --git a/lib/reply/src/term_size.cr b/lib/reply/src/term_size.cr index fd0c60421c4f..3af381101543 100644 --- a/lib/reply/src/term_size.cr +++ b/lib/reply/src/term_size.cr @@ -120,10 +120,7 @@ end dwMaximumWindowSize : COORD end - STD_OUTPUT_HANDLE = -11 - fun GetConsoleScreenBufferInfo(hConsoleOutput : Void*, lpConsoleScreenBufferInfo : CONSOLE_SCREEN_BUFFER_INFO*) : Void - fun GetStdHandle(nStdHandle : UInt32) : Void* end {% else %} lib LibC diff --git a/shard.lock b/shard.lock index e7f2ddc86d10..697bfe23b3c3 100644 --- a/shard.lock +++ b/shard.lock @@ -6,5 +6,5 @@ shards: reply: git: https://github.com/i3oris/reply.git - version: 0.3.1+git.commit.90a7eb5a76048884d5d56bf6b9369f1e67fdbcd7 + version: 0.3.1+git.commit.db423dae3dd34c6ba5e36174653a0c109117a167 diff --git a/shard.yml b/shard.yml index 85b76f49c8d8..1b2835281466 100644 --- a/shard.yml +++ b/shard.yml @@ -14,7 +14,7 @@ dependencies: github: icyleaf/markd reply: github: I3oris/reply - commit: 90a7eb5a76048884d5d56bf6b9369f1e67fdbcd7 + commit: db423dae3dd34c6ba5e36174653a0c109117a167 license: Apache-2.0 From d2e87322c045bd792cd7853d837136e89cc3aa3a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 01:59:44 +0800 Subject: [PATCH 076/193] Simplify `Socket::Addrinfo.getaddrinfo(&)` (#14956) This private method is now directly responsible for iterating over all `LibC::Addrinfo` objects, so there is no need to store this information in the `Socket::Addrinfo` struct itself. --- src/socket/addrinfo.cr | 62 +++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index 83ef561c88ac..c7a8ada00d86 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -10,7 +10,6 @@ class Socket getter size : Int32 @addr : LibC::SockaddrIn6 - @next : LibC::Addrinfo* # Resolves a domain that best matches the given options. # @@ -34,13 +33,10 @@ class Socket addrinfos = [] of Addrinfo getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| - loop do - addrinfos << addrinfo.not_nil! - unless addrinfo = addrinfo.next? - return addrinfos - end - end + addrinfos << addrinfo end + + addrinfos end # Resolves a domain that best matches the given options. @@ -57,28 +53,29 @@ class Socket # The iteration will be stopped once the block returns something that isn't # an `Exception` (e.g. a `Socket` or `nil`). def self.resolve(domain : String, service, family : Family? = nil, type : Type = nil, protocol : Protocol = Protocol::IP, timeout = nil, &) + exception = nil + getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| - loop do - value = yield addrinfo.not_nil! - - if value.is_a?(Exception) - unless addrinfo = addrinfo.try(&.next?) - if value.is_a?(Socket::ConnectError) - raise Socket::ConnectError.from_os_error("Error connecting to '#{domain}:#{service}'", value.os_error) - else - {% if flag?(:win32) && compare_versions(Crystal::LLVM_VERSION, "13.0.0") < 0 %} - # FIXME: Workaround for https://github.com/crystal-lang/crystal/issues/11047 - array = StaticArray(UInt8, 0).new(0) - {% end %} - - raise value - end - end - else - return value - end + value = yield addrinfo + + if value.is_a?(Exception) + exception = value + else + return value end end + + case exception + when Socket::ConnectError + raise Socket::ConnectError.from_os_error("Error connecting to '#{domain}:#{service}'", exception.os_error) + when Exception + {% if flag?(:win32) && compare_versions(Crystal::LLVM_VERSION, "13.0.0") < 0 %} + # FIXME: Workaround for https://github.com/crystal-lang/crystal/issues/11047 + array = StaticArray(UInt8, 0).new(0) + {% end %} + + raise exception + end end class Error < Socket::Error @@ -179,8 +176,12 @@ class Socket raise Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) end + addrinfo = ptr begin - yield new(ptr) + while addrinfo + yield new(addrinfo) + addrinfo = addrinfo.value.ai_next + end ensure LibC.freeaddrinfo(ptr) end @@ -232,7 +233,6 @@ class Socket @size = addrinfo.value.ai_addrlen.to_i @addr = uninitialized LibC::SockaddrIn6 - @next = addrinfo.value.ai_next case @family when Family::INET6 @@ -263,11 +263,5 @@ class Socket def to_unsafe pointerof(@addr).as(LibC::Sockaddr*) end - - protected def next? - if addrinfo = @next - Addrinfo.new(addrinfo) - end - end end end From 73263a8f1ab0f5662dd4974420e8e8d5ab7ae989 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 16:00:52 +0800 Subject: [PATCH 077/193] Support non-blocking `Process.run` standard streams on Windows (#14941) --- spec/std/process_spec.cr | 14 +++++++++ src/process.cr | 67 +++++++++++++++++++++++----------------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index f067d2f5c775..57f90121c26b 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -189,6 +189,20 @@ pending_interpreted describe: Process do Process.run(*stdin_to_stdout_command, error: closed_io) end + it "forwards non-blocking file" do + with_tempfile("non-blocking-process-input.txt", "non-blocking-process-output.txt") do |in_path, out_path| + File.open(in_path, "w+", blocking: false) do |input| + File.open(out_path, "w+", blocking: false) do |output| + input.puts "hello" + input.rewind + Process.run(*stdin_to_stdout_command, input: input, output: output) + output.rewind + output.gets_to_end.chomp.should eq("hello") + end + end + end + end + it "sets working directory with string" do parent = File.dirname(Dir.current) command = {% if flag?(:win32) %} diff --git a/src/process.cr b/src/process.cr index c8364196373f..63b78bf0f716 100644 --- a/src/process.cr +++ b/src/process.cr @@ -291,33 +291,20 @@ class Process private def stdio_to_fd(stdio : Stdio, for dst_io : IO::FileDescriptor) : IO::FileDescriptor case stdio - when IO::FileDescriptor - stdio - when IO - if stdio.closed? - if dst_io == STDIN - return File.open(File::NULL, "r").tap(&.close) - else - return File.open(File::NULL, "w").tap(&.close) + in IO::FileDescriptor + # on Windows, only async pipes can be passed to child processes, async + # regular files will report an error and those require a separate pipe + # (https://github.com/crystal-lang/crystal/pull/13362#issuecomment-1519082712) + {% if flag?(:win32) %} + unless stdio.blocking || stdio.info.type.pipe? + return io_to_fd(stdio, for: dst_io) end - end - - if dst_io == STDIN - fork_io, process_io = IO.pipe(read_blocking: true) - - @wait_count += 1 - ensure_channel - spawn { copy_io(stdio, process_io, channel, close_dst: true) } - else - process_io, fork_io = IO.pipe(write_blocking: true) + {% end %} - @wait_count += 1 - ensure_channel - spawn { copy_io(process_io, stdio, channel, close_src: true) } - end - - fork_io - when Redirect::Pipe + stdio + in IO + io_to_fd(stdio, for: dst_io) + in Redirect::Pipe case dst_io when STDIN fork_io, @input = IO.pipe(read_blocking: true) @@ -330,17 +317,41 @@ class Process end fork_io - when Redirect::Inherit + in Redirect::Inherit dst_io - when Redirect::Close + in Redirect::Close if dst_io == STDIN File.open(File::NULL, "r") else File.open(File::NULL, "w") end + end + end + + private def io_to_fd(stdio : Stdio, for dst_io : IO::FileDescriptor) : IO::FileDescriptor + if stdio.closed? + if dst_io == STDIN + return File.open(File::NULL, "r").tap(&.close) + else + return File.open(File::NULL, "w").tap(&.close) + end + end + + if dst_io == STDIN + fork_io, process_io = IO.pipe(read_blocking: true) + + @wait_count += 1 + ensure_channel + spawn { copy_io(stdio, process_io, channel, close_dst: true) } else - raise "BUG: Impossible type in stdio #{stdio.class}" + process_io, fork_io = IO.pipe(write_blocking: true) + + @wait_count += 1 + ensure_channel + spawn { copy_io(process_io, stdio, channel, close_src: true) } end + + fork_io end # :nodoc: From a798ae62dfe09d9a0698a1c64fb0ab221dc354e3 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 4 Sep 2024 16:01:12 +0800 Subject: [PATCH 078/193] Support `IO::FileDescriptor#flock_*` on non-blocking files on Windows (#14943) --- spec/std/file_spec.cr | 66 +++++++++++---------- src/crystal/system/win32/file_descriptor.cr | 57 ++++++++++++------ 2 files changed, 74 insertions(+), 49 deletions(-) diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index eb740885cd69..07b919bd4a6e 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -1248,46 +1248,50 @@ describe "File" do end end - it "#flock_shared" do - File.open(datapath("test_file.txt")) do |file1| - File.open(datapath("test_file.txt")) do |file2| - file1.flock_shared do - file2.flock_shared(blocking: false) { } + {true, false}.each do |blocking| + context "blocking: #{blocking}" do + it "#flock_shared" do + File.open(datapath("test_file.txt"), blocking: blocking) do |file1| + File.open(datapath("test_file.txt"), blocking: blocking) do |file2| + file1.flock_shared do + file2.flock_shared(blocking: false) { } + end + end end end - end - end - it "#flock_shared soft blocking fiber" do - File.open(datapath("test_file.txt")) do |file1| - File.open(datapath("test_file.txt")) do |file2| - done = Channel(Nil).new - file1.flock_exclusive + it "#flock_shared soft blocking fiber" do + File.open(datapath("test_file.txt"), blocking: blocking) do |file1| + File.open(datapath("test_file.txt"), blocking: blocking) do |file2| + done = Channel(Nil).new + file1.flock_exclusive - spawn do - file1.flock_unlock - done.send nil - end + spawn do + file1.flock_unlock + done.send nil + end - file2.flock_shared - done.receive + file2.flock_shared + done.receive + end + end end - end - end - it "#flock_exclusive soft blocking fiber" do - File.open(datapath("test_file.txt")) do |file1| - File.open(datapath("test_file.txt")) do |file2| - done = Channel(Nil).new - file1.flock_exclusive + it "#flock_exclusive soft blocking fiber" do + File.open(datapath("test_file.txt"), blocking: blocking) do |file1| + File.open(datapath("test_file.txt"), blocking: blocking) do |file2| + done = Channel(Nil).new + file1.flock_exclusive - spawn do - file1.flock_unlock - done.send nil - end + spawn do + file1.flock_unlock + done.send nil + end - file2.flock_exclusive - done.receive + file2.flock_exclusive + done.receive + end + end end end end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index f4e9200a0488..3c7823e62d3e 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -207,41 +207,62 @@ module Crystal::System::FileDescriptor end private def flock(exclusive, retry) - flags = LibC::LOCKFILE_FAIL_IMMEDIATELY + flags = 0_u32 + flags |= LibC::LOCKFILE_FAIL_IMMEDIATELY if !retry || system_blocking? flags |= LibC::LOCKFILE_EXCLUSIVE_LOCK if exclusive handle = windows_handle - if retry + if retry && system_blocking? until lock_file(handle, flags) sleep 0.1 end else - lock_file(handle, flags) || raise IO::Error.from_winerror("Error applying file lock: file is already locked") + lock_file(handle, flags) || raise IO::Error.from_winerror("Error applying file lock: file is already locked", target: self) end end private def lock_file(handle, flags) - # lpOverlapped must be provided despite the synchronous use of this method. - overlapped = LibC::OVERLAPPED.new - # lock the entire file with offset 0 in overlapped and number of bytes set to max value - if 0 != LibC.LockFileEx(handle, flags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, pointerof(overlapped)) - true - else - winerror = WinError.value - if winerror == WinError::ERROR_LOCK_VIOLATION - false + IOCP::OverlappedOperation.run(handle) do |operation| + result = LibC.LockFileEx(handle, flags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) + + if result == 0 + case error = WinError.value + when .error_io_pending? + # the operation is running asynchronously; do nothing + when .error_lock_violation? + # synchronous failure + return false + else + raise IO::Error.from_os_error("LockFileEx", error, target: self) + end else - raise IO::Error.from_os_error("LockFileEx", winerror, target: self) + return true end + + operation.wait_for_result(nil) do |error| + raise IO::Error.from_os_error("LockFileEx", error, target: self) + end + + true end end private def unlock_file(handle) - # lpOverlapped must be provided despite the synchronous use of this method. - overlapped = LibC::OVERLAPPED.new - # unlock the entire file with offset 0 in overlapped and number of bytes set to max value - if 0 == LibC.UnlockFileEx(handle, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, pointerof(overlapped)) - raise IO::Error.from_winerror("UnLockFileEx") + IOCP::OverlappedOperation.run(handle) do |operation| + result = LibC.UnlockFileEx(handle, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) + + if result == 0 + error = WinError.value + unless error.error_io_pending? + raise IO::Error.from_os_error("UnlockFileEx", error, target: self) + end + else + return + end + + operation.wait_for_result(nil) do |error| + raise IO::Error.from_os_error("UnlockFileEx", error, target: self) + end end end From 24b243bbf5af51c8fb5f1d45fc83f0ec56e4d493 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 00:23:48 +0800 Subject: [PATCH 079/193] Use correct timeout for `Socket#connect` on Windows (#14961) It does not appear the use of `read_timeout` here was intended. This applies to connection-oriented sockets only. Connectionless sockets like `UDPSocket` call `Crystal::System::Socket#system_connect_connectionless` instead which ignores the timeout parameter. --- src/crystal/system/win32/event_loop_iocp.cr | 2 +- src/crystal/system/win32/socket.cr | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index 6f9a921ad8d3..d1aae09b680a 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -228,7 +228,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop end def connect(socket : ::Socket, address : ::Socket::Addrinfo | ::Socket::Address, timeout : ::Time::Span?) : IO::Error? - socket.overlapped_connect(socket.fd, "ConnectEx") do |overlapped| + socket.overlapped_connect(socket.fd, "ConnectEx", timeout) do |overlapped| # This is: LibC.ConnectEx(fd, address, address.size, nil, 0, nil, overlapped) Crystal::System::Socket.connect_ex.call(socket.fd, address.to_unsafe, address.size, Pointer(Void).null, 0_u32, Pointer(UInt32).null, overlapped.to_unsafe) end diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 78645d51f320..3172be467836 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -128,7 +128,7 @@ module Crystal::System::Socket end # :nodoc: - def overlapped_connect(socket, method, &) + def overlapped_connect(socket, method, timeout, &) IOCP::OverlappedOperation.run(socket) do |operation| result = yield operation @@ -145,7 +145,7 @@ module Crystal::System::Socket return nil end - operation.wait_for_wsa_result(read_timeout) do |error| + operation.wait_for_wsa_result(timeout) do |error| case error when .wsa_io_incomplete?, .wsaeconnrefused? return ::Socket::ConnectError.from_os_error(method, error) From 1055af25d7acd7ce4f7dea6f933549d71873095b Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Wed, 4 Sep 2024 11:24:10 -0500 Subject: [PATCH 080/193] Make `IO::Buffered#buffer_size=` idempotent (#14855) The purpose of raising an exception here is to prevent the caller from doing something unsafe. _Changing_ the value is unsafe, but setting `buffer_size` to the same value is a safe operation. --- spec/std/io/buffered_spec.cr | 9 +++++++++ src/io/buffered.cr | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/spec/std/io/buffered_spec.cr b/spec/std/io/buffered_spec.cr index fbf6ac638ab8..faf684da0e25 100644 --- a/spec/std/io/buffered_spec.cr +++ b/spec/std/io/buffered_spec.cr @@ -72,6 +72,15 @@ describe "IO::Buffered" do end end + it "can set buffer_size to the same value after first use" do + io = BufferedWrapper.new(IO::Memory.new("hello\r\nworld\n")) + io.buffer_size = 16_384 + io.gets + + io.buffer_size = 16_384 + io.buffer_size.should eq(16_384) + end + it "does gets" do io = BufferedWrapper.new(IO::Memory.new("hello\r\nworld\n")) io.gets.should eq("hello") diff --git a/src/io/buffered.cr b/src/io/buffered.cr index 0e69872a638f..8bd65210aef2 100644 --- a/src/io/buffered.cr +++ b/src/io/buffered.cr @@ -49,7 +49,7 @@ module IO::Buffered # Set the buffer size of both the read and write buffer # Cannot be changed after any of the buffers have been allocated def buffer_size=(value) - if @in_buffer || @out_buffer + if (@in_buffer || @out_buffer) && (buffer_size != value) raise ArgumentError.new("Cannot change buffer_size after buffers have been allocated") end @buffer_size = value From 95af602c46b5ed68999492c401e7f56a020038b4 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 15:34:21 +0800 Subject: [PATCH 081/193] Emulate non-blocking `STDIN` console on Windows (#14947) --- src/crystal/system/file_descriptor.cr | 9 +++ src/crystal/system/win32/file_descriptor.cr | 64 ++++++++++++++++++++- src/crystal/system/win32/iocp.cr | 35 ++++++++--- src/crystal/system/win32/process.cr | 2 +- src/io/file_descriptor.cr | 8 +++ src/lib_c/x86_64-windows-msvc/c/ioapiset.cr | 8 +++ 6 files changed, 115 insertions(+), 11 deletions(-) diff --git a/src/crystal/system/file_descriptor.cr b/src/crystal/system/file_descriptor.cr index 0652ed56e52a..481e00982e25 100644 --- a/src/crystal/system/file_descriptor.cr +++ b/src/crystal/system/file_descriptor.cr @@ -22,6 +22,15 @@ module Crystal::System::FileDescriptor # Also used in `IO::FileDescriptor#finalize`. # def file_descriptor_close + # Returns `true` or `false` if this file descriptor pretends to block or not + # to block the caller thread regardless of the underlying internal file + # descriptor's implementation. Returns `nil` if nothing needs to be done, i.e. + # `#blocking` is identical to `#system_blocking?`. + # + # Currently used by console STDIN on Windows. + private def emulated_blocking? : Bool? + end + private def system_read(slice : Bytes) : Int32 event_loop.read(self, slice) end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 3c7823e62d3e..4fdc319a8b6c 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -3,6 +3,7 @@ require "c/consoleapi" require "c/consoleapi2" require "c/winnls" require "crystal/system/win32/iocp" +require "crystal/system/thread" module Crystal::System::FileDescriptor # Platform-specific type to represent a file descriptor handle to the operating @@ -76,13 +77,24 @@ module Crystal::System::FileDescriptor bytes_written end + def emulated_blocking? : Bool? + # reading from STDIN is done via a separate thread (see + # `ConsoleUtils.read_console` below) + handle = windows_handle + if LibC.GetConsoleMode(handle, out _) != 0 + if handle == LibC.GetStdHandle(LibC::STD_INPUT_HANDLE) + return false + end + end + end + # :nodoc: def system_blocking? @system_blocking end private def system_blocking=(blocking) - unless blocking == @system_blocking + unless blocking == self.blocking raise IO::Error.new("Cannot reconfigure `IO::FileDescriptor#blocking` after creation") end end @@ -339,7 +351,11 @@ module Crystal::System::FileDescriptor end end + # `blocking` must be set to `true` because the underlying handles never + # support overlapped I/O; instead, `#emulated_blocking?` should return + # `false` for `STDIN` as it uses a separate thread io = IO::FileDescriptor.new(handle.address, blocking: true) + # Set sync or flush_on_newline as described in STDOUT and STDERR docs. # See https://crystal-lang.org/api/toplevel.html#STDERR if console_handle @@ -465,11 +481,57 @@ private module ConsoleUtils end private def self.read_console(handle : LibC::HANDLE, slice : Slice(UInt16)) : Int32 + @@mtx.synchronize do + @@read_requests << ReadRequest.new( + handle: handle, + slice: slice, + iocp: Crystal::EventLoop.current.iocp, + completion_key: Crystal::IOCP::CompletionKey.new(:stdin_read, ::Fiber.current), + ) + @@read_cv.signal + end + + ::Fiber.suspend + + @@mtx.synchronize do + @@bytes_read.shift + end + end + + private def self.read_console_blocking(handle : LibC::HANDLE, slice : Slice(UInt16)) : Int32 if 0 == LibC.ReadConsoleW(handle, slice, slice.size, out units_read, nil) raise IO::Error.from_winerror("ReadConsoleW") end units_read.to_i32 end + + record ReadRequest, handle : LibC::HANDLE, slice : Slice(UInt16), iocp : LibC::HANDLE, completion_key : Crystal::IOCP::CompletionKey + + @@read_cv = ::Thread::ConditionVariable.new + @@read_requests = Deque(ReadRequest).new + @@bytes_read = Deque(Int32).new + @@mtx = ::Thread::Mutex.new + @@reader_thread = ::Thread.new { reader_loop } + + private def self.reader_loop + while true + request = @@mtx.synchronize do + loop do + if entry = @@read_requests.shift? + break entry + end + @@read_cv.wait(@@mtx) + end + end + + bytes = read_console_blocking(request.handle, request.slice) + + @@mtx.synchronize do + @@bytes_read << bytes + LibC.PostQueuedCompletionStatus(request.iocp, LibC::JOB_OBJECT_MSG_EXIT_PROCESS, request.completion_key.object_id, nil) + end + end + end end # Enable UTF-8 console I/O for the duration of program execution diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index 6f5746954277..ba87ed123f22 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -6,7 +6,16 @@ require "crystal/system/thread_linked_list" module Crystal::IOCP # :nodoc: class CompletionKey + enum Tag + ProcessRun + StdinRead + end + property fiber : Fiber? + getter tag : Tag + + def initialize(@tag : Tag, @fiber : Fiber? = nil) + end end def self.wait_queued_completions(timeout, alertable = false, &) @@ -39,20 +48,19 @@ module Crystal::IOCP # at the moment only `::Process#wait` uses a non-nil completion key; all # I/O operations, including socket ones, do not set this field case completion_key = Pointer(Void).new(entry.lpCompletionKey).as(CompletionKey?) - when Nil + in Nil operation = OverlappedOperation.unbox(entry.lpOverlapped) operation.schedule { |fiber| yield fiber } - else - case entry.dwNumberOfBytesTransferred - when LibC::JOB_OBJECT_MSG_EXIT_PROCESS, LibC::JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS + in CompletionKey + if completion_key_valid?(completion_key, entry.dwNumberOfBytesTransferred) + # if `Process` exits before a call to `#wait`, this fiber will be + # reset already if fiber = completion_key.fiber - # this ensures the `::Process` doesn't keep an indirect reference to - # `::Thread.current`, as that leads to a finalization cycle + # this ensures existing references to `completion_key` do not keep + # an indirect reference to `::Thread.current`, as that leads to a + # finalization cycle completion_key.fiber = nil - yield fiber - else - # the `Process` exits before a call to `#wait`; do nothing end end end @@ -61,6 +69,15 @@ module Crystal::IOCP false end + private def self.completion_key_valid?(completion_key, number_of_bytes_transferred) + case completion_key.tag + in .process_run? + number_of_bytes_transferred.in?(LibC::JOB_OBJECT_MSG_EXIT_PROCESS, LibC::JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS) + in .stdin_read? + true + end + end + class OverlappedOperation enum State STARTED diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 05b2ea36584e..2c6d81720636 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -17,7 +17,7 @@ struct Crystal::System::Process @thread_id : LibC::DWORD @process_handle : LibC::HANDLE @job_object : LibC::HANDLE - @completion_key = IOCP::CompletionKey.new + @completion_key = IOCP::CompletionKey.new(:process_run) @@interrupt_handler : Proc(::Process::ExitReason, Nil)? @@interrupt_count = Crystal::AtomicSemaphore.new diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index 8940a118041f..622229e43e00 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -66,7 +66,15 @@ class IO::FileDescriptor < IO Crystal::System::FileDescriptor.from_stdio(fd) end + # Returns whether I/O operations on this file descriptor block the current + # thread. If false, operations might opt to suspend the current fiber instead. + # + # This might be different from the internal file descriptor. For example, when + # `STDIN` is a terminal on Windows, this returns `false` since the underlying + # blocking reads are done on a completely separate thread. def blocking + emulated = emulated_blocking? + return emulated unless emulated.nil? system_blocking? end diff --git a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr index 1c94b66db4c8..f6d56ef5a0e6 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr @@ -21,6 +21,14 @@ lib LibC dwMilliseconds : DWORD, fAlertable : BOOL ) : BOOL + + fun PostQueuedCompletionStatus( + completionPort : HANDLE, + dwNumberOfBytesTransferred : DWORD, + dwCompletionKey : ULONG_PTR, + lpOverlapped : OVERLAPPED* + ) : BOOL + fun CancelIoEx( hFile : HANDLE, lpOverlapped : OVERLAPPED* From f6e2ab33f272167b68a64ac3ab2ca877fa714e2a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 15:37:47 +0800 Subject: [PATCH 082/193] Deprecate `::sleep(Number)` (#14962) This is in line with other places in the standard library that favor `Time::Span` over number types, such as `IO` timeouts (#14368) and `Benchmark.ips` (#14805). --- samples/channel_select.cr | 2 +- samples/conway.cr | 4 ++-- samples/tcp_client.cr | 2 +- spec/std/benchmark_spec.cr | 4 ++-- spec/std/channel_spec.cr | 12 ++++++------ spec/std/http/client/client_spec.cr | 14 +++++++------- spec/std/http/server/server_spec.cr | 4 ++-- spec/std/http/spec_helper.cr | 2 +- spec/std/openssl/ssl/server_spec.cr | 2 +- spec/std/signal_spec.cr | 4 ++-- spec/support/channel.cr | 4 ++-- spec/support/retry.cr | 2 +- src/benchmark.cr | 6 +++--- src/concurrent.cr | 5 +++-- src/crystal/system/unix/file_descriptor.cr | 2 +- src/crystal/system/win32/file_descriptor.cr | 2 +- src/signal.cr | 6 +++--- 17 files changed, 39 insertions(+), 38 deletions(-) diff --git a/samples/channel_select.cr b/samples/channel_select.cr index 1ad24e1ff779..25ef96c7db16 100644 --- a/samples/channel_select.cr +++ b/samples/channel_select.cr @@ -2,7 +2,7 @@ def generator(n : T) forall T channel = Channel(T).new spawn do loop do - sleep n + sleep n.seconds channel.send n end end diff --git a/samples/conway.cr b/samples/conway.cr index b1d9d9089bb0..5178d48f9bd0 100644 --- a/samples/conway.cr +++ b/samples/conway.cr @@ -78,7 +78,7 @@ struct ConwayMap end end -PAUSE_MILLIS = 20 +PAUSE = 20.milliseconds DEFAULT_COUNT = 300 INITIAL_MAP = [ " 1 ", @@ -99,6 +99,6 @@ spawn { gets; exit } 1.upto(DEFAULT_COUNT) do |i| puts map puts "n = #{i}\tPress ENTER to exit" - sleep PAUSE_MILLIS * 0.001 + sleep PAUSE map.next end diff --git a/samples/tcp_client.cr b/samples/tcp_client.cr index 95392dc72601..f4f02d5bdf05 100644 --- a/samples/tcp_client.cr +++ b/samples/tcp_client.cr @@ -6,5 +6,5 @@ socket = TCPSocket.new "127.0.0.1", 9000 10.times do |i| socket.puts i puts "Server response: #{socket.gets}" - sleep 0.5 + sleep 0.5.seconds end diff --git a/spec/std/benchmark_spec.cr b/spec/std/benchmark_spec.cr index 2f3c1fb06fd5..8113f5f03a4c 100644 --- a/spec/std/benchmark_spec.cr +++ b/spec/std/benchmark_spec.cr @@ -13,8 +13,8 @@ describe Benchmark::IPS::Job do # test several things to avoid running a benchmark over and over again in # the specs j = Benchmark::IPS::Job.new(0.001, 0.001, interactive: false) - a = j.report("a") { sleep 0.001 } - b = j.report("b") { sleep 0.002 } + a = j.report("a") { sleep 1.milliseconds } + b = j.report("b") { sleep 2.milliseconds } j.execute diff --git a/spec/std/channel_spec.cr b/spec/std/channel_spec.cr index 9d121f9d9827..69161dd96e01 100644 --- a/spec/std/channel_spec.cr +++ b/spec/std/channel_spec.cr @@ -110,7 +110,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action) end @@ -129,7 +129,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end @@ -178,7 +178,7 @@ describe Channel do it "returns nil channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do i, m = Channel.select(ch.receive_select_action?) m.should be_nil end @@ -191,7 +191,7 @@ describe Channel do Channel.select(ch.receive_select_action?) } - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({ {0, nil}, {0, nil}, {0, nil}, {0, nil} }) end @@ -273,7 +273,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo")) end @@ -292,7 +292,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2; ch.close }) do + spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 4c9da8db7ad7..451960a8c79f 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -6,7 +6,7 @@ require "http/server" require "http/log" require "log/spec" -private def test_server(host, port, read_time = 0, content_type = "text/plain", write_response = true, &) +private def test_server(host, port, read_time = 0.seconds, content_type = "text/plain", write_response = true, &) server = TCPServer.new(host, port) begin spawn do @@ -312,12 +312,12 @@ module HTTP end it "doesn't read the body if request was HEAD" do - resp_get = test_server("localhost", 0, 0) do |server| + resp_get = test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) break client.get("/") end - test_server("localhost", 0, 0) do |server| + test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) resp_head = client.head("/") resp_head.headers.should eq(resp_get.headers) @@ -338,7 +338,7 @@ module HTTP end it "tests read_timeout" do - test_server("localhost", 0, 0) do |server| + test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) client.read_timeout = 1.second client.get("/") @@ -348,7 +348,7 @@ module HTTP # it doesn't make sense to try to write because the client will already # timeout on read. Writing a response could lead on an exception in # the server if the socket is closed. - test_server("localhost", 0, 0.5, write_response: false) do |server| + test_server("localhost", 0, 0.5.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSARecv timed out" {% else %} "Read timed out" {% end %}) do client.read_timeout = 0.001 @@ -362,7 +362,7 @@ module HTTP # it doesn't make sense to try to write because the client will already # timeout on read. Writing a response could lead on an exception in # the server if the socket is closed. - test_server("localhost", 0, 0, write_response: false) do |server| + test_server("localhost", 0, 0.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSASend timed out" {% else %} "Write timed out" {% end %}) do client.write_timeout = 0.001 @@ -372,7 +372,7 @@ module HTTP end it "tests connect_timeout" do - test_server("localhost", 0, 0) do |server| + test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) client.connect_timeout = 0.5 client.get("/") diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index c8b39c9e7e42..5e1e5dab76f6 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -65,14 +65,14 @@ describe HTTP::Server do while !server.listening? Fiber.yield end - sleep 0.1 + sleep 0.1.seconds schedule_timeout ch TCPSocket.open(address.address, address.port) { } # wait before closing the server - sleep 0.1 + sleep 0.1.seconds server.close ch.receive.should eq SpecChannelStatus::End diff --git a/spec/std/http/spec_helper.cr b/spec/std/http/spec_helper.cr index 18ec9e0bab46..82b4f12d6774 100644 --- a/spec/std/http/spec_helper.cr +++ b/spec/std/http/spec_helper.cr @@ -49,7 +49,7 @@ def run_server(server, &) {% if flag?(:preview_mt) %} # avoids fiber synchronization issues in specs, like closing the server # before we properly listen, ... - sleep 0.001 + sleep 1.millisecond {% end %} yield server_done ensure diff --git a/spec/std/openssl/ssl/server_spec.cr b/spec/std/openssl/ssl/server_spec.cr index ff5e578a8ed0..2e0e413a618d 100644 --- a/spec/std/openssl/ssl/server_spec.cr +++ b/spec/std/openssl/ssl/server_spec.cr @@ -130,7 +130,7 @@ describe OpenSSL::SSL::Server do OpenSSL::SSL::Server.open tcp_server, server_context do |server| spawn do - sleep 1 + sleep 1.second OpenSSL::SSL::Socket::Client.open(TCPSocket.new(tcp_server.local_address.address, tcp_server.local_address.port), client_context, hostname: "example.com") do |socket| end end diff --git a/spec/std/signal_spec.cr b/spec/std/signal_spec.cr index cae1c5e83834..969e4dc3d742 100644 --- a/spec/std/signal_spec.cr +++ b/spec/std/signal_spec.cr @@ -27,7 +27,7 @@ pending_interpreted describe: "Signal" do Process.signal Signal::USR1, Process.pid 10.times do |i| break if ran - sleep 0.1 + sleep 0.1.seconds end ran.should be_true ensure @@ -52,7 +52,7 @@ pending_interpreted describe: "Signal" do end Process.signal Signal::USR1, Process.pid - sleep 0.1 + sleep 0.1.seconds ran_first.should be_true ran_second.should be_true ensure diff --git a/spec/support/channel.cr b/spec/support/channel.cr index 7ca8d0668797..5ec3511c89c8 100644 --- a/spec/support/channel.cr +++ b/spec/support/channel.cr @@ -10,9 +10,9 @@ def schedule_timeout(c : Channel(SpecChannelStatus)) # TODO: it's not clear why some interpreter specs # take more than 1 second in some cases. # See #12429. - sleep 5 + sleep 5.seconds {% else %} - sleep 1 + sleep 1.second {% end %} c.send(SpecChannelStatus::Timeout) end diff --git a/spec/support/retry.cr b/spec/support/retry.cr index 638804c4be81..76fca476db95 100644 --- a/spec/support/retry.cr +++ b/spec/support/retry.cr @@ -7,7 +7,7 @@ def retry(n = 5, &) if i == 0 Fiber.yield else - sleep 0.01 * (2**i) + sleep 10.milliseconds * (2**i) end else return diff --git a/src/benchmark.cr b/src/benchmark.cr index a0f4933ddf2a..14bc12ae069a 100644 --- a/src/benchmark.cr +++ b/src/benchmark.cr @@ -11,8 +11,8 @@ require "./benchmark/**" # require "benchmark" # # Benchmark.ips do |x| -# x.report("short sleep") { sleep 0.01 } -# x.report("shorter sleep") { sleep 0.001 } +# x.report("short sleep") { sleep 10.milliseconds } +# x.report("shorter sleep") { sleep 1.millisecond } # end # ``` # @@ -31,7 +31,7 @@ require "./benchmark/**" # require "benchmark" # # Benchmark.ips(warmup: 4, calculation: 10) do |x| -# x.report("sleep") { sleep 0.01 } +# x.report("sleep") { sleep 10.milliseconds } # end # ``` # diff --git a/src/concurrent.cr b/src/concurrent.cr index 6f3a58291a22..0f8805857720 100644 --- a/src/concurrent.cr +++ b/src/concurrent.cr @@ -7,6 +7,7 @@ require "crystal/tracing" # # While this fiber is waiting this time, other ready-to-execute # fibers might start their execution. +@[Deprecated("Use `::sleep(Time::Span)` instead")] def sleep(seconds : Number) : Nil if seconds < 0 raise ArgumentError.new "Sleep seconds must be positive" @@ -42,7 +43,7 @@ end # # spawn do # 6.times do -# sleep 1 +# sleep 1.second # puts 1 # end # ch.send(nil) @@ -50,7 +51,7 @@ end # # spawn do # 3.times do -# sleep 2 +# sleep 2.seconds # puts 2 # end # ch.send(nil) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index fc8839ac9e83..56a9eee80dd5 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -158,7 +158,7 @@ module Crystal::System::FileDescriptor if retry until flock(op) - sleep 0.1 + sleep 0.1.seconds end else flock(op) || raise IO::Error.from_errno("Error applying file lock: file is already locked", target: self) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 4fdc319a8b6c..d4831d9528cb 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -226,7 +226,7 @@ module Crystal::System::FileDescriptor handle = windows_handle if retry && system_blocking? until lock_file(handle, flags) - sleep 0.1 + sleep 0.1.seconds end else lock_file(handle, flags) || raise IO::Error.from_winerror("Error applying file lock: file is already locked", target: self) diff --git a/src/signal.cr b/src/signal.cr index e0f59a9f57d3..37999c76b9e1 100644 --- a/src/signal.cr +++ b/src/signal.cr @@ -8,17 +8,17 @@ require "crystal/system/signal" # # ``` # puts "Ctrl+C still has the OS default action (stops the program)" -# sleep 3 +# sleep 3.seconds # # Signal::INT.trap do # puts "Gotcha!" # end # puts "Ctrl+C will be caught from now on" -# sleep 3 +# sleep 3.seconds # # Signal::INT.reset # puts "Ctrl+C is back to the OS default action" -# sleep 3 +# sleep 3.seconds # ``` # # WARNING: An uncaught exception in a signal handler is a fatal error. From 256c555b92db30b88742b0f30144c08fd07b5ce3 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 23:30:31 +0800 Subject: [PATCH 083/193] Implement `System::Group` on Windows (#14945) More examples of valid group IDs can be obtained using `whoami.exe /groups`. --- spec/std/system/group_spec.cr | 12 +++-- src/crystal/system/group.cr | 2 + src/crystal/system/win32/group.cr | 82 +++++++++++++++++++++++++++++++ src/crystal/system/win32/user.cr | 65 +++--------------------- src/crystal/system/windows.cr | 53 ++++++++++++++++++++ src/docs_main.cr | 4 +- 6 files changed, 153 insertions(+), 65 deletions(-) create mode 100644 src/crystal/system/win32/group.cr diff --git a/spec/std/system/group_spec.cr b/spec/std/system/group_spec.cr index 5c55611e4d28..ba511d03a05c 100644 --- a/spec/std/system/group_spec.cr +++ b/spec/std/system/group_spec.cr @@ -1,10 +1,14 @@ -{% skip_file if flag?(:win32) %} - require "spec" require "system/group" -GROUP_NAME = {{ `id -gn`.stringify.chomp }} -GROUP_ID = {{ `id -g`.stringify.chomp }} +{% if flag?(:win32) %} + GROUP_NAME = "BUILTIN\\Administrators" + GROUP_ID = "S-1-5-32-544" +{% else %} + GROUP_NAME = {{ `id -gn`.stringify.chomp }} + GROUP_ID = {{ `id -g`.stringify.chomp }} +{% end %} + INVALID_GROUP_NAME = "this_group_does_not_exist" INVALID_GROUP_ID = {% if flag?(:android) %}"8888"{% else %}"1234567"{% end %} diff --git a/src/crystal/system/group.cr b/src/crystal/system/group.cr index 8a542e2cc63c..6cb93739a900 100644 --- a/src/crystal/system/group.cr +++ b/src/crystal/system/group.cr @@ -12,6 +12,8 @@ end require "./wasi/group" {% elsif flag?(:unix) %} require "./unix/group" +{% elsif flag?(:win32) %} + require "./win32/group" {% else %} {% raise "No Crystal::System::Group implementation available" %} {% end %} diff --git a/src/crystal/system/win32/group.cr b/src/crystal/system/win32/group.cr new file mode 100644 index 000000000000..3b40774ac2d8 --- /dev/null +++ b/src/crystal/system/win32/group.cr @@ -0,0 +1,82 @@ +require "crystal/system/windows" + +# This file contains source code derived from the following: +# +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/os/user/lookup_windows.go +# * https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/syscall/security_windows.go +# +# The following is their license: +# +# Copyright 2009 The Go Authors. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Crystal::System::Group + def initialize(@name : String, @id : String) + end + + def system_name : String + @name + end + + def system_id : String + @id + end + + def self.from_name?(groupname : String) : ::System::Group? + if found = Crystal::System.name_to_sid(groupname) + from_sid(found.sid) + end + end + + def self.from_id?(groupid : String) : ::System::Group? + if sid = Crystal::System.sid_from_s(groupid) + begin + from_sid(sid) + ensure + LibC.LocalFree(sid) + end + end + end + + private def self.from_sid(sid : LibC::SID*) : ::System::Group? + canonical = Crystal::System.sid_to_name(sid) || return + + # https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/7b2aeb27-92fc-41f6-8437-deb65d950921#gt_0387e636-5654-4910-9519-1f8326cf5ec0 + # SidTypeAlias should also be treated as a group type next to SidTypeGroup + # and SidTypeWellKnownGroup: + # "alias object -> resource group: A group object..." + # + # Tests show that "Administrators" can be considered of type SidTypeAlias. + case canonical.type + when .sid_type_group?, .sid_type_well_known_group?, .sid_type_alias? + domain_and_group = canonical.domain.empty? ? canonical.name : "#{canonical.domain}\\#{canonical.name}" + gid = Crystal::System.sid_to_s(sid) + ::System::Group.new(domain_and_group, gid) + end + end +end diff --git a/src/crystal/system/win32/user.cr b/src/crystal/system/win32/user.cr index e5fcdbba10aa..4a06570c72b8 100644 --- a/src/crystal/system/win32/user.cr +++ b/src/crystal/system/win32/user.cr @@ -1,4 +1,4 @@ -require "c/sddl" +require "crystal/system/windows" require "c/lm" require "c/userenv" require "c/security" @@ -71,7 +71,7 @@ module Crystal::System::User end def self.from_username?(username : String) : ::System::User? - if found = name_to_sid(username) + if found = Crystal::System.name_to_sid(username) if found.type.sid_type_user? from_sid(found.sid) end @@ -79,7 +79,7 @@ module Crystal::System::User end def self.from_id?(id : String) : ::System::User? - if sid = sid_from_s(id) + if sid = Crystal::System.sid_from_s(id) begin from_sid(sid) ensure @@ -89,13 +89,13 @@ module Crystal::System::User end private def self.from_sid(sid : LibC::SID*) : ::System::User? - canonical = sid_to_name(sid) || return + canonical = Crystal::System.sid_to_name(sid) || return return unless canonical.type.sid_type_user? domain_and_user = "#{canonical.domain}\\#{canonical.name}" full_name = lookup_full_name(canonical.name, canonical.domain, domain_and_user) || return pgid = lookup_primary_group_id(canonical.name, canonical.domain) || return - uid = sid_to_s(sid) + uid = Crystal::System.sid_to_s(sid) home_dir = lookup_home_directory(uid, canonical.name) || return ::System::User.new(domain_and_user, uid, pgid, full_name, home_dir) @@ -136,10 +136,10 @@ module Crystal::System::User # https://support.microsoft.com/en-us/help/297951/how-to-use-the-primarygroupid-attribute-to-find-the-primary-group-for # The method follows this formula: domainRID + "-" + primaryGroupRID private def self.lookup_primary_group_id(name : String, domain : String) : String? - domain_sid = name_to_sid(domain) || return + domain_sid = Crystal::System.name_to_sid(domain) || return return unless domain_sid.type.sid_type_domain? - domain_sid_str = sid_to_s(domain_sid.sid) + domain_sid_str = Crystal::System.sid_to_s(domain_sid.sid) # If the user has joined a domain use the RID of the default primary group # called "Domain Users": @@ -210,43 +210,6 @@ module Crystal::System::User return "#{profile_dir}\\#{username}" if profile_dir end - private record SIDLookupResult, sid : LibC::SID*, domain : String, type : LibC::SID_NAME_USE - - private def self.name_to_sid(name : String) : SIDLookupResult? - utf16_name = Crystal::System.to_wstr(name) - - sid_size = LibC::DWORD.zero - domain_buf_size = LibC::DWORD.zero - LibC.LookupAccountNameW(nil, utf16_name, nil, pointerof(sid_size), nil, pointerof(domain_buf_size), out _) - - unless WinError.value.error_none_mapped? - sid = Pointer(UInt8).malloc(sid_size).as(LibC::SID*) - domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) - if LibC.LookupAccountNameW(nil, utf16_name, sid, pointerof(sid_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 - domain = String.from_utf16(domain_buf[..-2]) - SIDLookupResult.new(sid, domain, sid_type) - end - end - end - - private record NameLookupResult, name : String, domain : String, type : LibC::SID_NAME_USE - - private def self.sid_to_name(sid : LibC::SID*) : NameLookupResult? - name_buf_size = LibC::DWORD.zero - domain_buf_size = LibC::DWORD.zero - LibC.LookupAccountSidW(nil, sid, nil, pointerof(name_buf_size), nil, pointerof(domain_buf_size), out _) - - unless WinError.value.error_none_mapped? - name_buf = Slice(LibC::WCHAR).new(name_buf_size) - domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) - if LibC.LookupAccountSidW(nil, sid, name_buf, pointerof(name_buf_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 - name = String.from_utf16(name_buf[..-2]) - domain = String.from_utf16(domain_buf[..-2]) - NameLookupResult.new(name, domain, sid_type) - end - end - end - private def self.domain_joined? : Bool status = LibC.NetGetJoinInformation(nil, out domain, out type) if status != LibC::NERR_Success @@ -256,18 +219,4 @@ module Crystal::System::User LibC.NetApiBufferFree(domain) is_domain end - - private def self.sid_to_s(sid : LibC::SID*) : String - if LibC.ConvertSidToStringSidW(sid, out ptr) == 0 - raise RuntimeError.from_winerror("ConvertSidToStringSidW") - end - str, _ = String.from_utf16(ptr) - LibC.LocalFree(ptr) - str - end - - private def self.sid_from_s(str : String) : LibC::SID* - status = LibC.ConvertStringSidToSidW(Crystal::System.to_wstr(str), out sid) - status != 0 ? sid : Pointer(LibC::SID).null - end end diff --git a/src/crystal/system/windows.cr b/src/crystal/system/windows.cr index b303d4d61f6d..90b38396cf8f 100644 --- a/src/crystal/system/windows.cr +++ b/src/crystal/system/windows.cr @@ -1,3 +1,5 @@ +require "c/sddl" + # :nodoc: module Crystal::System def self.retry_wstr_buffer(&) @@ -13,4 +15,55 @@ module Crystal::System def self.to_wstr(str : String, name : String? = nil) : LibC::LPWSTR str.check_no_null_byte(name).to_utf16.to_unsafe end + + def self.sid_to_s(sid : LibC::SID*) : String + if LibC.ConvertSidToStringSidW(sid, out ptr) == 0 + raise RuntimeError.from_winerror("ConvertSidToStringSidW") + end + str, _ = String.from_utf16(ptr) + LibC.LocalFree(ptr) + str + end + + def self.sid_from_s(str : String) : LibC::SID* + status = LibC.ConvertStringSidToSidW(to_wstr(str), out sid) + status != 0 ? sid : Pointer(LibC::SID).null + end + + record SIDLookupResult, sid : LibC::SID*, domain : String, type : LibC::SID_NAME_USE + + def self.name_to_sid(name : String) : SIDLookupResult? + utf16_name = to_wstr(name) + + sid_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountNameW(nil, utf16_name, nil, pointerof(sid_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + sid = Pointer(UInt8).malloc(sid_size).as(LibC::SID*) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountNameW(nil, utf16_name, sid, pointerof(sid_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + domain = String.from_utf16(domain_buf[..-2]) + SIDLookupResult.new(sid, domain, sid_type) + end + end + end + + record NameLookupResult, name : String, domain : String, type : LibC::SID_NAME_USE + + def self.sid_to_name(sid : LibC::SID*) : NameLookupResult? + name_buf_size = LibC::DWORD.zero + domain_buf_size = LibC::DWORD.zero + LibC.LookupAccountSidW(nil, sid, nil, pointerof(name_buf_size), nil, pointerof(domain_buf_size), out _) + + unless WinError.value.error_none_mapped? + name_buf = Slice(LibC::WCHAR).new(name_buf_size) + domain_buf = Slice(LibC::WCHAR).new(domain_buf_size) + if LibC.LookupAccountSidW(nil, sid, name_buf, pointerof(name_buf_size), domain_buf, pointerof(domain_buf_size), out sid_type) != 0 + name = String.from_utf16(name_buf[..-2]) + domain = String.from_utf16(domain_buf[..-2]) + NameLookupResult.new(name, domain, sid_type) + end + end + end end diff --git a/src/docs_main.cr b/src/docs_main.cr index e670d6d3fa83..ab3ee2affdbc 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -56,8 +56,6 @@ require "./uri/params/serializable" require "./uuid" require "./uuid/json" require "./syscall" -{% unless flag?(:win32) %} - require "./system/*" -{% end %} +require "./system/*" require "./wait_group" require "./docs_pseudo_methods" From 05c5eaa17ce17c60917b30cebaca022d59721e2e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 5 Sep 2024 23:31:33 +0800 Subject: [PATCH 084/193] Implement `Reference.pre_initialize` in the interpreter (#14968) --- spec/primitives/reference_spec.cr | 12 ++++++---- .../crystal/interpreter/instructions.cr | 10 ++++++++ .../crystal/interpreter/primitives.cr | 24 +++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/spec/primitives/reference_spec.cr b/spec/primitives/reference_spec.cr index 13bb024f1ba9..497b49155b5a 100644 --- a/spec/primitives/reference_spec.cr +++ b/spec/primitives/reference_spec.cr @@ -37,8 +37,7 @@ describe "Primitives: reference" do end end - # TODO: implement in the interpreter - pending_interpreted describe: ".pre_initialize" do + describe ".pre_initialize" do it "doesn't fail on complex ivar initializer if value is discarded (#14325)" do bar_buffer = GC.malloc(instance_sizeof(Outer)) Outer.pre_initialize(bar_buffer) @@ -55,7 +54,12 @@ describe "Primitives: reference" do it "sets type ID" do foo_buffer = GC.malloc(instance_sizeof(Foo)) base = Foo.pre_initialize(foo_buffer).as(Base) - base.crystal_type_id.should eq(Foo.crystal_instance_type_id) + base.should be_a(Foo) + base.as(typeof(Foo.crystal_instance_type_id)*).value.should eq(Foo.crystal_instance_type_id) + {% unless flag?(:interpreted) %} + # FIXME: `Object#crystal_type_id` is incorrect for virtual types in the interpreter (#14967) + base.crystal_type_id.should eq(Foo.crystal_instance_type_id) + {% end %} end it "runs inline instance initializers" do @@ -89,7 +93,7 @@ describe "Primitives: reference" do end end - pending_interpreted describe: ".unsafe_construct" do + describe ".unsafe_construct" do it "constructs an object in-place" do foo_buffer = GC.malloc(instance_sizeof(Foo)) foo = Foo.unsafe_construct(foo_buffer, 123_i64) diff --git a/src/compiler/crystal/interpreter/instructions.cr b/src/compiler/crystal/interpreter/instructions.cr index 8fae94f5ee62..6a38afd888d3 100644 --- a/src/compiler/crystal/interpreter/instructions.cr +++ b/src/compiler/crystal/interpreter/instructions.cr @@ -1276,6 +1276,16 @@ require "./repl" ptr end, }, + reset_class: { + operands: [size : Int32, type_id : Int32], + pop_values: [pointer : Pointer(UInt8)], + push: true, + code: begin + pointer.clear(size) + pointer.as(Int32*).value = type_id + pointer + end, + }, put_metaclass: { operands: [size : Int32, union_type : Bool], push: true, diff --git a/src/compiler/crystal/interpreter/primitives.cr b/src/compiler/crystal/interpreter/primitives.cr index 7ad508f8d0fc..ca436947370e 100644 --- a/src/compiler/crystal/interpreter/primitives.cr +++ b/src/compiler/crystal/interpreter/primitives.cr @@ -178,6 +178,30 @@ class Crystal::Repl::Compiler pop(sizeof(Pointer(Void)), node: nil) end end + when "pre_initialize" + type = + if obj + discard_value(obj) + obj.type.instance_type + else + scope.instance_type + end + + accept_call_members(node) + + dup sizeof(Pointer(Void)), node: nil + reset_class(aligned_instance_sizeof_type(type), type_id(type), node: node) + + initializer_compiled_defs = @context.type_instance_var_initializers(type) + unless initializer_compiled_defs.empty? + initializer_compiled_defs.size.times do + dup sizeof(Pointer(Void)), node: nil + end + + initializer_compiled_defs.each do |compiled_def| + call compiled_def, node: nil + end + end when "tuple_indexer_known_index" unless @wants_value accept_call_members(node) From 777643886c3e7e549ce295e920b95d8d426e544c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 6 Sep 2024 06:36:14 +0800 Subject: [PATCH 085/193] Add `Crystal::System::Addrinfo` (#14957) Moves the platform-specific code into a separate module, so that implementations other than `LibC.getaddrinfo` can be added without cluttering the same file (e.g. Win32's `GetAddrInfoExW` from #13619). --- src/crystal/system/addrinfo.cr | 36 ++++++++++ src/crystal/system/unix/addrinfo.cr | 71 +++++++++++++++++++ src/crystal/system/wasi/addrinfo.cr | 27 +++++++ src/crystal/system/win32/addrinfo.cr | 61 ++++++++++++++++ src/socket/addrinfo.cr | 102 ++++----------------------- 5 files changed, 209 insertions(+), 88 deletions(-) create mode 100644 src/crystal/system/addrinfo.cr create mode 100644 src/crystal/system/unix/addrinfo.cr create mode 100644 src/crystal/system/wasi/addrinfo.cr create mode 100644 src/crystal/system/win32/addrinfo.cr diff --git a/src/crystal/system/addrinfo.cr b/src/crystal/system/addrinfo.cr new file mode 100644 index 000000000000..23513e6f763e --- /dev/null +++ b/src/crystal/system/addrinfo.cr @@ -0,0 +1,36 @@ +module Crystal::System::Addrinfo + # alias Handle + + # protected def initialize(addrinfo : Handle) + + # def system_ip_address : ::Socket::IPAddress + + # def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + + # def self.next_addrinfo(addrinfo : Handle) : Handle + + # def self.free_addrinfo(addrinfo : Handle) + + def self.getaddrinfo(domain, service, family, type, protocol, timeout, & : ::Socket::Addrinfo ->) + addrinfo = root = getaddrinfo(domain, service, family, type, protocol, timeout) + + begin + while addrinfo + yield ::Socket::Addrinfo.new(addrinfo) + addrinfo = next_addrinfo(addrinfo) + end + ensure + free_addrinfo(root) + end + end +end + +{% if flag?(:wasi) %} + require "./wasi/addrinfo" +{% elsif flag?(:unix) %} + require "./unix/addrinfo" +{% elsif flag?(:win32) %} + require "./win32/addrinfo" +{% else %} + {% raise "No Crystal::System::Addrinfo implementation available" %} +{% end %} diff --git a/src/crystal/system/unix/addrinfo.cr b/src/crystal/system/unix/addrinfo.cr new file mode 100644 index 000000000000..7f1e51558397 --- /dev/null +++ b/src/crystal/system/unix/addrinfo.cr @@ -0,0 +1,71 @@ +module Crystal::System::Addrinfo + alias Handle = LibC::Addrinfo* + + @addr : LibC::SockaddrIn6 + + protected def initialize(addrinfo : Handle) + @family = ::Socket::Family.from_value(addrinfo.value.ai_family) + @type = ::Socket::Type.from_value(addrinfo.value.ai_socktype) + @protocol = ::Socket::Protocol.from_value(addrinfo.value.ai_protocol) + @size = addrinfo.value.ai_addrlen.to_i + + @addr = uninitialized LibC::SockaddrIn6 + + case @family + when ::Socket::Family::INET6 + addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) + when ::Socket::Family::INET + addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) + else + # TODO: (asterite) UNSPEC and UNIX unsupported? + end + end + + def system_ip_address : ::Socket::IPAddress + ::Socket::IPAddress.from(to_unsafe, size) + end + + def to_unsafe + pointerof(@addr).as(LibC::Sockaddr*) + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + hints = LibC::Addrinfo.new + hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 + hints.ai_socktype = type + hints.ai_protocol = protocol + hints.ai_flags = 0 + + if service.is_a?(Int) + hints.ai_flags |= LibC::AI_NUMERICSERV + end + + # On OS X < 10.12, the libsystem implementation of getaddrinfo segfaults + # if AI_NUMERICSERV is set, and servname is NULL or 0. + {% if flag?(:darwin) %} + if service.in?(0, nil) && (hints.ai_flags & LibC::AI_NUMERICSERV) + hints.ai_flags |= LibC::AI_NUMERICSERV + service = "00" + end + {% end %} + + ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) + unless ret.zero? + if ret == LibC::EAI_SYSTEM + raise ::Socket::Addrinfo::Error.from_os_error nil, Errno.value, domain: domain + end + + error = Errno.new(ret) + raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + end + ptr + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + addrinfo.value.ai_next + end + + def self.free_addrinfo(addrinfo : Handle) + LibC.freeaddrinfo(addrinfo) + end +end diff --git a/src/crystal/system/wasi/addrinfo.cr b/src/crystal/system/wasi/addrinfo.cr new file mode 100644 index 000000000000..29ba8e0b3cfc --- /dev/null +++ b/src/crystal/system/wasi/addrinfo.cr @@ -0,0 +1,27 @@ +module Crystal::System::Addrinfo + alias Handle = NoReturn + + protected def initialize(addrinfo : Handle) + raise NotImplementedError.new("Crystal::System::Addrinfo#initialize") + end + + def system_ip_address : ::Socket::IPAddress + raise NotImplementedError.new("Crystal::System::Addrinfo#system_ip_address") + end + + def to_unsafe + raise NotImplementedError.new("Crystal::System::Addrinfo#to_unsafe") + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + raise NotImplementedError.new("Crystal::System::Addrinfo.getaddrinfo") + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + raise NotImplementedError.new("Crystal::System::Addrinfo.next_addrinfo") + end + + def self.free_addrinfo(addrinfo : Handle) + raise NotImplementedError.new("Crystal::System::Addrinfo.free_addrinfo") + end +end diff --git a/src/crystal/system/win32/addrinfo.cr b/src/crystal/system/win32/addrinfo.cr new file mode 100644 index 000000000000..b033d61f16e7 --- /dev/null +++ b/src/crystal/system/win32/addrinfo.cr @@ -0,0 +1,61 @@ +module Crystal::System::Addrinfo + alias Handle = LibC::Addrinfo* + + @addr : LibC::SockaddrIn6 + + protected def initialize(addrinfo : Handle) + @family = ::Socket::Family.from_value(addrinfo.value.ai_family) + @type = ::Socket::Type.from_value(addrinfo.value.ai_socktype) + @protocol = ::Socket::Protocol.from_value(addrinfo.value.ai_protocol) + @size = addrinfo.value.ai_addrlen.to_i + + @addr = uninitialized LibC::SockaddrIn6 + + case @family + when ::Socket::Family::INET6 + addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) + when ::Socket::Family::INET + addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) + else + # TODO: (asterite) UNSPEC and UNIX unsupported? + end + end + + def system_ip_address : ::Socket::IPAddress + ::Socket::IPAddress.from(to_unsafe, size) + end + + def to_unsafe + pointerof(@addr).as(LibC::Sockaddr*) + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + hints = LibC::Addrinfo.new + hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 + hints.ai_socktype = type + hints.ai_protocol = protocol + hints.ai_flags = 0 + + if service.is_a?(Int) + hints.ai_flags |= LibC::AI_NUMERICSERV + if service < 0 + raise ::Socket::Addrinfo::Error.from_os_error(nil, WinError::WSATYPE_NOT_FOUND, domain: domain, type: type, protocol: protocol, service: service) + end + end + + ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) + unless ret.zero? + error = WinError.new(ret.to_u32!) + raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + end + ptr + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + addrinfo.value.ai_next + end + + def self.free_addrinfo(addrinfo : Handle) + LibC.freeaddrinfo(addrinfo) + end +end diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index c7a8ada00d86..cdf55c912601 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -1,16 +1,17 @@ require "uri/punycode" require "./address" +require "crystal/system/addrinfo" class Socket # Domain name resolver. struct Addrinfo + include Crystal::System::Addrinfo + getter family : Family getter type : Type getter protocol : Protocol getter size : Int32 - @addr : LibC::SockaddrIn6 - # Resolves a domain that best matches the given options. # # - *domain* may be an IP address or a domain name. @@ -126,66 +127,15 @@ class Socket end private def self.getaddrinfo(domain, service, family, type, protocol, timeout, &) - {% if flag?(:wasm32) %} - raise NotImplementedError.new "Socket::Addrinfo.getaddrinfo" - {% else %} - # RFC 3986 says: - # > When a non-ASCII registered name represents an internationalized domain name - # > intended for resolution via the DNS, the name must be transformed to the IDNA - # > encoding [RFC3490] prior to name lookup. - domain = URI::Punycode.to_ascii domain - - hints = LibC::Addrinfo.new - hints.ai_family = (family || Family::UNSPEC).to_i32 - hints.ai_socktype = type - hints.ai_protocol = protocol - hints.ai_flags = 0 - - if service.is_a?(Int) - hints.ai_flags |= LibC::AI_NUMERICSERV - end - - # On OS X < 10.12, the libsystem implementation of getaddrinfo segfaults - # if AI_NUMERICSERV is set, and servname is NULL or 0. - {% if flag?(:darwin) %} - if service.in?(0, nil) && (hints.ai_flags & LibC::AI_NUMERICSERV) - hints.ai_flags |= LibC::AI_NUMERICSERV - service = "00" - end - {% end %} - {% if flag?(:win32) %} - if service.is_a?(Int) && service < 0 - raise Error.from_os_error(nil, WinError::WSATYPE_NOT_FOUND, domain: domain, type: type, protocol: protocol, service: service) - end - {% end %} - - ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) - unless ret.zero? - {% if flag?(:unix) %} - # EAI_SYSTEM is not defined on win32 - if ret == LibC::EAI_SYSTEM - raise Error.from_os_error nil, Errno.value, domain: domain - end - {% end %} - - error = {% if flag?(:win32) %} - WinError.new(ret.to_u32!) - {% else %} - Errno.new(ret) - {% end %} - raise Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) - end - - addrinfo = ptr - begin - while addrinfo - yield new(addrinfo) - addrinfo = addrinfo.value.ai_next - end - ensure - LibC.freeaddrinfo(ptr) - end - {% end %} + # RFC 3986 says: + # > When a non-ASCII registered name represents an internationalized domain name + # > intended for resolution via the DNS, the name must be transformed to the IDNA + # > encoding [RFC3490] prior to name lookup. + domain = URI::Punycode.to_ascii domain + + Crystal::System::Addrinfo.getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| + yield addrinfo + end end # Resolves *domain* for the TCP protocol and returns an `Array` of possible @@ -226,29 +176,9 @@ class Socket resolve(domain, service, family, Type::DGRAM, Protocol::UDP) { |addrinfo| yield addrinfo } end - protected def initialize(addrinfo : LibC::Addrinfo*) - @family = Family.from_value(addrinfo.value.ai_family) - @type = Type.from_value(addrinfo.value.ai_socktype) - @protocol = Protocol.from_value(addrinfo.value.ai_protocol) - @size = addrinfo.value.ai_addrlen.to_i - - @addr = uninitialized LibC::SockaddrIn6 - - case @family - when Family::INET6 - addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) - when Family::INET - addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) - else - # TODO: (asterite) UNSPEC and UNIX unsupported? - end - end - - @ip_address : IPAddress? - # Returns an `IPAddress` matching this addrinfo. - def ip_address : Socket::IPAddress - @ip_address ||= IPAddress.from(to_unsafe, size) + getter(ip_address : Socket::IPAddress) do + system_ip_address end def inspect(io : IO) @@ -259,9 +189,5 @@ class Socket io << protocol io << ")" end - - def to_unsafe - pointerof(@addr).as(LibC::Sockaddr*) - end end end From db2ecd781422d5b4cd615d5b10874072c1310b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 6 Sep 2024 00:36:42 +0200 Subject: [PATCH 086/193] Fix `Range#size` return type to `Int32` (#14588) The `super` implementation `Enumerable#size` has the same type restriction already. --- src/range.cr | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/range.cr b/src/range.cr index 39d8119dff6e..e8ee24b190cb 100644 --- a/src/range.cr +++ b/src/range.cr @@ -480,7 +480,10 @@ struct Range(B, E) # (3..8).size # => 6 # (3...8).size # => 5 # ``` - def size + # + # Raises `OverflowError` if the difference is bigger than `Int32`. + # Raises `ArgumentError` if either `begin` or `end` are `nil`. + def size : Int32 b = self.begin e = self.end @@ -488,7 +491,7 @@ struct Range(B, E) if b.is_a?(Int) && e.is_a?(Int) e -= 1 if @exclusive n = e - b + 1 - n < 0 ? 0 : n + n < 0 ? 0 : n.to_i32 else if b.nil? || e.nil? raise ArgumentError.new("Can't calculate size of an open range") From 214d39ad112374910640c87e21695c9e8eb9d213 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 6 Sep 2024 14:15:19 +0800 Subject: [PATCH 087/193] Fix CRT static-dynamic linking conflict in specs with C sources (#14970) This fixes the `LINK : warning LNK4098: defaultlib 'LIBCMT' conflicts with use of other libs; use /NODEFAULTLIB:library` message that shows up on Windows CI while running compiler specs. --- spec/support/tempfile.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/tempfile.cr b/spec/support/tempfile.cr index a77070d90e40..ef4468040955 100644 --- a/spec/support/tempfile.cr +++ b/spec/support/tempfile.cr @@ -67,7 +67,7 @@ def with_temp_c_object_file(c_code, *, filename = "temp_c", file = __FILE__, &) end end - `#{cl} /nologo /c #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_filename}")}`.should be_truthy + `#{cl} /nologo /c /MD #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_filename}")}`.should be_truthy {% else %} `#{ENV["CC"]? || "cc"} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy {% end %} From a310dee1bbf30839964e798d7cd5653c5149ba3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 6 Sep 2024 08:19:38 +0200 Subject: [PATCH 088/193] Fix use global paths in macro bodies (#14965) Macros inject code into other scopes. Paths are resolved in the expanded scope and there can be namespace conflicts. This fixes non-global paths in macro bodies that expand into uncontrolled scopes where namespaces could clash. This is a fixup for #14282 (released in 1.12.0). --- src/crystal/pointer_linked_list.cr | 4 ++-- src/ecr/macros.cr | 2 +- src/intrinsics.cr | 34 +++++++++++++++--------------- src/json/serialization.cr | 6 +++--- src/number.cr | 8 +++---- src/object.cr | 12 +++++------ src/slice.cr | 6 +++--- src/spec/dsl.cr | 4 ++-- src/spec/helpers/iterate.cr | 8 +++---- src/static_array.cr | 2 +- src/syscall/aarch64-linux.cr | 2 +- src/syscall/arm-linux.cr | 2 +- src/syscall/i386-linux.cr | 2 +- src/syscall/x86_64-linux.cr | 2 +- src/uri/params/serializable.cr | 14 ++++++------ src/yaml/serialization.cr | 14 ++++++------ 16 files changed, 61 insertions(+), 61 deletions(-) diff --git a/src/crystal/pointer_linked_list.cr b/src/crystal/pointer_linked_list.cr index 03109979d662..cde9b0b79ddc 100644 --- a/src/crystal/pointer_linked_list.cr +++ b/src/crystal/pointer_linked_list.cr @@ -7,8 +7,8 @@ struct Crystal::PointerLinkedList(T) module Node macro included - property previous : Pointer(self) = Pointer(self).null - property next : Pointer(self) = Pointer(self).null + property previous : ::Pointer(self) = ::Pointer(self).null + property next : ::Pointer(self) = ::Pointer(self).null end end diff --git a/src/ecr/macros.cr b/src/ecr/macros.cr index 92c02cc4284a..5e051232271b 100644 --- a/src/ecr/macros.cr +++ b/src/ecr/macros.cr @@ -34,7 +34,7 @@ module ECR # ``` macro def_to_s(filename) def to_s(__io__ : IO) : Nil - ECR.embed {{filename}}, "__io__" + ::ECR.embed {{filename}}, "__io__" end end diff --git a/src/intrinsics.cr b/src/intrinsics.cr index c5ae837d8931..7cdc462ce543 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -179,7 +179,7 @@ end module Intrinsics macro debugtrap - LibIntrinsics.debugtrap + ::LibIntrinsics.debugtrap end def self.pause @@ -191,15 +191,15 @@ module Intrinsics end macro memcpy(dest, src, len, is_volatile) - LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memmove(dest, src, len, is_volatile) - LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memset(dest, val, len, is_volatile) - LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) end def self.read_cycle_counter @@ -263,43 +263,43 @@ module Intrinsics end macro countleading8(src, zero_is_undef) - LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) end macro countleading16(src, zero_is_undef) - LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) end macro countleading32(src, zero_is_undef) - LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) end macro countleading64(src, zero_is_undef) - LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) end macro countleading128(src, zero_is_undef) - LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) end macro counttrailing8(src, zero_is_undef) - LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) end macro counttrailing16(src, zero_is_undef) - LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) end macro counttrailing32(src, zero_is_undef) - LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) end macro counttrailing64(src, zero_is_undef) - LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) end macro counttrailing128(src, zero_is_undef) - LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) end def self.fshl8(a, b, count) : UInt8 @@ -343,14 +343,14 @@ module Intrinsics end macro va_start(ap) - LibIntrinsics.va_start({{ap}}) + ::LibIntrinsics.va_start({{ap}}) end macro va_end(ap) - LibIntrinsics.va_end({{ap}}) + ::LibIntrinsics.va_end({{ap}}) end end macro debugger - Intrinsics.debugtrap + ::Intrinsics.debugtrap end diff --git a/src/json/serialization.cr b/src/json/serialization.cr index b1eb86d15082..15d948f02f40 100644 --- a/src/json/serialization.cr +++ b/src/json/serialization.cr @@ -164,7 +164,7 @@ module JSON private def self.new_from_json_pull_parser(pull : ::JSON::PullParser) instance = allocate instance.initialize(__pull_for_json_serializable: pull) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -422,8 +422,8 @@ module JSON # Try to find the discriminator while also getting the raw # string value of the parsed JSON, so then we can pass it # to the final type. - json = String.build do |io| - JSON.build(io) do |builder| + json = ::String.build do |io| + ::JSON.build(io) do |builder| builder.start_object pull.read_object do |key| if key == {{field.id.stringify}} diff --git a/src/number.cr b/src/number.cr index f7c82aa4cded..9d955c065df3 100644 --- a/src/number.cr +++ b/src/number.cr @@ -59,7 +59,7 @@ struct Number # :nodoc: macro expand_div(rhs_types, result_type) {% for rhs in rhs_types %} - @[AlwaysInline] + @[::AlwaysInline] def /(other : {{rhs}}) : {{result_type}} {{result_type}}.new(self) / {{result_type}}.new(other) end @@ -84,7 +84,7 @@ struct Number # [1, 2, 3, 4] of Int64 # : Array(Int64) # ``` macro [](*nums) - Array({{@type}}).build({{nums.size}}) do |%buffer| + ::Array({{@type}}).build({{nums.size}}) do |%buffer| {% for num, i in nums %} %buffer[{{i}}] = {{@type}}.new({{num}}) {% end %} @@ -113,7 +113,7 @@ struct Number # Slice[1_i64, 2_i64, 3_i64, 4_i64] # : Slice(Int64) # ``` macro slice(*nums, read_only = false) - %slice = Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) + %slice = ::Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) {% for num, i in nums %} %slice.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} @@ -139,7 +139,7 @@ struct Number # StaticArray[1_i64, 2_i64, 3_i64, 4_i64] # : StaticArray(Int64) # ``` macro static_array(*nums) - %array = uninitialized StaticArray({{@type}}, {{nums.size}}) + %array = uninitialized ::StaticArray({{@type}}, {{nums.size}}) {% for num, i in nums %} %array.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} diff --git a/src/object.cr b/src/object.cr index ba818ac2979e..800736687788 100644 --- a/src/object.cr +++ b/src/object.cr @@ -562,7 +562,7 @@ class Object def {{method_prefix}}\{{name.var.id}} : \{{name.type}} if (value = {{var_prefix}}\{{name.var.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") else value end @@ -574,7 +574,7 @@ class Object def {{method_prefix}}\{{name.id}} if (value = {{var_prefix}}\{{name.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") else value end @@ -1293,7 +1293,7 @@ class Object # wrapper.capitalize # => "Hello" # ``` macro delegate(*methods, to object) - {% if compare_versions(Crystal::VERSION, "1.12.0-dev") >= 0 %} + {% if compare_versions(::Crystal::VERSION, "1.12.0-dev") >= 0 %} {% eq_operators = %w(<= >= == != []= ===) %} {% for method in methods %} {% if method.id.ends_with?('=') && !eq_operators.includes?(method.id.stringify) %} @@ -1427,18 +1427,18 @@ class Object macro def_clone # Returns a copy of `self` with all instance variables cloned. def clone - \{% if @type < Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} + \{% if @type < ::Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} exec_recursive_clone do |hash| clone = \{{@type}}.allocate hash[object_id] = clone.object_id clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone end \{% else %} clone = \{{@type}}.allocate clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone \{% end %} end diff --git a/src/slice.cr b/src/slice.cr index c87816f315d9..ace008e53e05 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -34,14 +34,14 @@ struct Slice(T) macro [](*args, read_only = false) # TODO: there should be a better way to check this, probably # asking if @type was instantiated or if T is defined - {% if @type.name != "Slice(T)" && T < Number %} + {% if @type.name != "Slice(T)" && T < ::Number %} {{T}}.slice({{args.splat(", ")}}read_only: {{read_only}}) {% else %} - %ptr = Pointer(typeof({{args.splat}})).malloc({{args.size}}) + %ptr = ::Pointer(typeof({{args.splat}})).malloc({{args.size}}) {% for arg, i in args %} %ptr[{{i}}] = {{arg}} {% end %} - Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) + ::Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) {% end %} end diff --git a/src/spec/dsl.cr b/src/spec/dsl.cr index 578076b86d69..d712aa59da4f 100644 --- a/src/spec/dsl.cr +++ b/src/spec/dsl.cr @@ -298,8 +298,8 @@ module Spec # If the "log" module is required it is configured to emit no entries by default. def log_setup defined?(::Log) do - if Log.responds_to?(:setup) - Log.setup_from_env(default_level: :none) + if ::Log.responds_to?(:setup) + ::Log.setup_from_env(default_level: :none) end end end diff --git a/src/spec/helpers/iterate.cr b/src/spec/helpers/iterate.cr index be302ebb49c2..7a70f83408ca 100644 --- a/src/spec/helpers/iterate.cr +++ b/src/spec/helpers/iterate.cr @@ -47,7 +47,7 @@ module Spec::Methods # See `.it_iterates` for details. macro assert_iterates_yielding(expected, method, *, infinite = false, tuple = false) %remaining = ({{expected}}).size - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) {{ method.id }} do |{% if tuple %}*{% end %}x| if %remaining == 0 if {{ infinite }} @@ -73,11 +73,11 @@ module Spec::Methods # # See `.it_iterates` for details. macro assert_iterates_iterator(expected, method, *, infinite = false) - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) %iter = {{ method.id }} ({{ expected }}).size.times do %v = %iter.next - if %v.is_a?(Iterator::Stop) + if %v.is_a?(::Iterator::Stop) # Compare the actual value directly. Since there are less # then expected values, the expectation will fail and raise. %ary.should eq({{ expected }}) @@ -86,7 +86,7 @@ module Spec::Methods %ary << %v end unless {{ infinite }} - %iter.next.should be_a(Iterator::Stop) + %iter.next.should be_a(::Iterator::Stop) end %ary.should eq({{ expected }}) diff --git a/src/static_array.cr b/src/static_array.cr index 2c09e21df166..3d00705bc21a 100644 --- a/src/static_array.cr +++ b/src/static_array.cr @@ -50,7 +50,7 @@ struct StaticArray(T, N) # * `Number.static_array` is a convenient alternative for designating a # specific numerical item type. macro [](*args) - %array = uninitialized StaticArray(typeof({{args.splat}}), {{args.size}}) + %array = uninitialized ::StaticArray(typeof({{args.splat}}), {{args.size}}) {% for arg, i in args %} %array.to_unsafe[{{i}}] = {{arg}} {% end %} diff --git a/src/syscall/aarch64-linux.cr b/src/syscall/aarch64-linux.cr index 5a61e8e7eed8..77b891fe2a7c 100644 --- a/src/syscall/aarch64-linux.cr +++ b/src/syscall/aarch64-linux.cr @@ -334,7 +334,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/arm-linux.cr b/src/syscall/arm-linux.cr index 97119fc4b3f3..da349dd45301 100644 --- a/src/syscall/arm-linux.cr +++ b/src/syscall/arm-linux.cr @@ -409,7 +409,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/i386-linux.cr b/src/syscall/i386-linux.cr index 843b2d1fd856..a0f94a51160a 100644 --- a/src/syscall/i386-linux.cr +++ b/src/syscall/i386-linux.cr @@ -445,7 +445,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/x86_64-linux.cr b/src/syscall/x86_64-linux.cr index 1f01c9226658..5a63b6ee2e1a 100644 --- a/src/syscall/x86_64-linux.cr +++ b/src/syscall/x86_64-linux.cr @@ -368,7 +368,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/uri/params/serializable.cr b/src/uri/params/serializable.cr index c0d766e85242..54d3b970e53c 100644 --- a/src/uri/params/serializable.cr +++ b/src/uri/params/serializable.cr @@ -59,19 +59,19 @@ struct URI::Params # ``` module Serializable macro included - def self.from_www_form(params : String) - new_from_www_form URI::Params.parse params + def self.from_www_form(params : ::String) + new_from_www_form ::URI::Params.parse params end # :nodoc: # # This is needed so that nested types can pass the name thru internally. # Has to be public so the generated code can call it, but should be considered an implementation detail. - def self.from_www_form(params : ::URI::Params, name : String) + def self.from_www_form(params : ::URI::Params, name : ::String) new_from_www_form(params, name) end - protected def self.new_from_www_form(params : ::URI::Params, name : String? = nil) + protected def self.new_from_www_form(params : ::URI::Params, name : ::String? = nil) instance = allocate instance.initialize(__uri_params: params, name: name) GC.add_finalizer(instance) if instance.responds_to?(:finalize) @@ -79,12 +79,12 @@ struct URI::Params end macro inherited - def self.from_www_form(params : String) - new_from_www_form URI::Params.parse params + def self.from_www_form(params : ::String) + new_from_www_form ::URI::Params.parse params end # :nodoc: - def self.from_www_form(params : ::URI::Params, name : String) + def self.from_www_form(params : ::URI::Params, name : ::String) new_from_www_form(params, name) end end diff --git a/src/yaml/serialization.cr b/src/yaml/serialization.cr index d5fae8dfe9c0..4a1521469dea 100644 --- a/src/yaml/serialization.cr +++ b/src/yaml/serialization.cr @@ -156,11 +156,11 @@ module YAML # Define a `new` directly in the included type, # so it overloads well with other possible initializes - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end - private def self.new_from_yaml_node(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + private def self.new_from_yaml_node(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, self) do |obj| return obj end @@ -170,7 +170,7 @@ module YAML ctx.record_anchor(node, instance) instance.initialize(__context_for_yaml_serializable: ctx, __node_for_yaml_serializable: node) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -178,7 +178,7 @@ module YAML # so it can compete with other possible initializes macro inherited - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end end @@ -409,17 +409,17 @@ module YAML {% mapping.raise "Mapping argument must be a HashLiteral or a NamedTupleLiteral, not #{mapping.class_name.id}" %} {% end %} - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, \{{@type}}) do |obj| return obj end - unless node.is_a?(YAML::Nodes::Mapping) + unless node.is_a?(::YAML::Nodes::Mapping) node.raise "Expected YAML mapping, not #{node.class}" end node.each do |key, value| - next unless key.is_a?(YAML::Nodes::Scalar) && value.is_a?(YAML::Nodes::Scalar) + next unless key.is_a?(::YAML::Nodes::Scalar) && value.is_a?(::YAML::Nodes::Scalar) next unless key.value == {{field.id.stringify}} discriminator_value = value.value From 9240f50795ebfc3f52d85a07d546acf173dc6379 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 6 Sep 2024 18:37:06 +0800 Subject: [PATCH 089/193] Fix exponent wrapping in `Math.frexp(BigFloat)` for very large values (#14971) `BigFloat`s represent their base-`256 ** sizeof(LibGMP::MpLimb)` exponent with a `LibGMP::MpExp` field, but `LibGMP.mpf_get_d_2exp` only returns the base-2 exponent as a `LibC::Long`, so values outside `(2.0.to_big_f ** -0x80000001)...(2.0.to_big_f ** 0x7FFFFFFF)` lead to an exponent overflow on Windows or 32-bit platforms: ```crystal require "big" Math.frexp(2.0.to_big_f ** 0xFFFFFFF5) # => {1.55164027193164307015e+1292913986, -10} Math.frexp(2.0.to_big_f ** -0xFFFFFFF4) # => {1.61119819150333097422e-1292913987, 13} Math.frexp(2.0.to_big_f ** 0x7FFFFFFF) # raises OverflowError ``` This patch fixes it by computing the exponent ourselves. --- spec/std/big/big_float_spec.cr | 16 +++++++++++++ src/big/big_float.cr | 41 +++++++++++++--------------------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 08d7e93bfb0b..73c6bcf06de8 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -548,7 +548,23 @@ end describe "BigFloat Math" do it ".frexp" do + Math.frexp(0.to_big_f).should eq({0.0, 0}) + Math.frexp(1.to_big_f).should eq({0.5, 1}) Math.frexp(0.2.to_big_f).should eq({0.8, -2}) + Math.frexp(2.to_big_f ** 63).should eq({0.5, 64}) + Math.frexp(2.to_big_f ** 64).should eq({0.5, 65}) + Math.frexp(2.to_big_f ** 200).should eq({0.5, 201}) + Math.frexp(2.to_big_f ** -200).should eq({0.5, -199}) + Math.frexp(2.to_big_f ** 0x7FFFFFFF).should eq({0.5, 0x80000000}) + Math.frexp(2.to_big_f ** 0x80000000).should eq({0.5, 0x80000001}) + Math.frexp(2.to_big_f ** 0xFFFFFFFF).should eq({0.5, 0x100000000}) + Math.frexp(1.75 * 2.to_big_f ** 0x123456789).should eq({0.875, 0x12345678A}) + Math.frexp(2.to_big_f ** -0x80000000).should eq({0.5, -0x7FFFFFFF}) + Math.frexp(2.to_big_f ** -0x80000001).should eq({0.5, -0x80000000}) + Math.frexp(2.to_big_f ** -0x100000000).should eq({0.5, -0xFFFFFFFF}) + Math.frexp(1.75 * 2.to_big_f ** -0x123456789).should eq({0.875, -0x123456788}) + Math.frexp(-(2.to_big_f ** 0x7FFFFFFF)).should eq({-0.5, 0x80000000}) + Math.frexp(-(2.to_big_f ** -0x100000000)).should eq({-0.5, -0xFFFFFFFF}) end it ".sqrt" do diff --git a/src/big/big_float.cr b/src/big/big_float.cr index 2c567f21eec9..ff78b7de8290 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -537,15 +537,24 @@ end module Math # Decomposes the given floating-point *value* into a normalized fraction and an integral power of two. def frexp(value : BigFloat) : {BigFloat, Int64} - LibGMP.mpf_get_d_2exp(out exp, value) # we need BigFloat frac, so will skip Float64 one. + return {BigFloat.zero, 0_i64} if value.zero? + + # We compute this ourselves since `LibGMP.mpf_get_d_2exp` only returns a + # `LibC::Long` exponent, which is not sufficient for 32-bit `LibC::Long` and + # 32-bit `LibGMP::MpExp`, e.g. on 64-bit Windows. + # Since `0.5 <= frac.abs < 1.0`, the radix point should be just above the + # most significant limb, and there should be no leading zeros in that limb. + leading_zeros = value.@mpf._mp_d[value.@mpf._mp_size.abs - 1].leading_zeros_count + exp = 8_i64 * sizeof(LibGMP::MpLimb) * value.@mpf._mp_exp - leading_zeros + frac = BigFloat.new do |mpf| - if exp >= 0 - LibGMP.mpf_div_2exp(mpf, value, exp) - else - LibGMP.mpf_mul_2exp(mpf, value, -exp) - end + # remove leading zeros in the most significant limb + LibGMP.mpf_mul_2exp(mpf, value, leading_zeros) + # reset the exponent manually + mpf.value._mp_exp = 0 end - {frac, exp.to_i64} + + {frac, exp} end # Calculates the square root of *value*. @@ -559,21 +568,3 @@ module Math BigFloat.new { |mpf| LibGMP.mpf_sqrt(mpf, value) } end end - -# :nodoc: -struct Crystal::Hasher - def self.reduce_num(value : BigFloat) - float_normalize_wrap(value) do |value| - # more exact version of `Math.frexp` - LibGMP.mpf_get_d_2exp(out exp, value) - frac = BigFloat.new do |mpf| - if exp >= 0 - LibGMP.mpf_div_2exp(mpf, value, exp) - else - LibGMP.mpf_mul_2exp(mpf, value, -exp) - end - end - float_normalize_reference(value, frac, exp) - end - end -end From 85d1ae78670b756e47c544ce34d6022ea0b6b4d1 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 12:37:53 +0200 Subject: [PATCH 090/193] Fix: `Crystal::SpinLock` doesn't need to be allocated on the HEAP (#14972) The abstraction is a mere abstraction over an atomic integer and the object itself are only ever used internally of other objects, with the exception of Channel where the code explicitely accesses the ivar directly (thus not making copies). We can avoid a HEAP allocation everywhere we use them (i.e. in lots of places). --- src/crystal/spin_lock.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/spin_lock.cr b/src/crystal/spin_lock.cr index 4255fcae7bbd..105c235e0c66 100644 --- a/src/crystal/spin_lock.cr +++ b/src/crystal/spin_lock.cr @@ -1,5 +1,5 @@ # :nodoc: -class Crystal::SpinLock +struct Crystal::SpinLock private UNLOCKED = 0 private LOCKED = 1 From 025f3e041693882790d8941a8ecbd989715645cb Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 12:39:25 +0200 Subject: [PATCH 091/193] Fix: `#file_descriptor_close` should set `@closed` (UNIX) (#14973) Prevents the GC from trying to cleanup resources that had already been closed in signal/process pipes. --- src/crystal/system/unix/file_descriptor.cr | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 56a9eee80dd5..759802f4323e 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -121,6 +121,13 @@ module Crystal::System::FileDescriptor end def file_descriptor_close(&) : Nil + # It would usually be set by IO::Buffered#unbuffered_close but we sometimes + # close file descriptors directly (i.e. signal/process pipes) and the IO + # object wouldn't be marked as closed, leading IO::FileDescriptor#finalize + # to try to close the fd again (pointless) and lead to other issues if we + # try to do more cleanup in the finalizer (error) + @closed = true + # Clear the @volatile_fd before actually closing it in order to # reduce the chance of reading an outdated fd value _fd = @volatile_fd.swap(-1) From cf15fb2dfbd9cf92aff2f191a136c1a1c12013ca Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 6 Sep 2024 08:59:02 -0300 Subject: [PATCH 092/193] Adds initial support for external commands (#14953) --- spec/primitives/external_command_spec.cr | 34 ++++++++++++++++++++++++ src/compiler/crystal/command.cr | 3 +++ 2 files changed, 37 insertions(+) create mode 100644 spec/primitives/external_command_spec.cr diff --git a/spec/primitives/external_command_spec.cr b/spec/primitives/external_command_spec.cr new file mode 100644 index 000000000000..91687f7c2d21 --- /dev/null +++ b/spec/primitives/external_command_spec.cr @@ -0,0 +1,34 @@ +{% skip_file if flag?(:interpreted) %} + +require "../spec_helper" + +describe Crystal::Command do + it "exec external commands", tags: %w[slow] do + with_temp_executable "crystal-external" do |path| + with_tempfile "crystal-external.cr" do |source_file| + File.write source_file, <<-CRYSTAL + puts ENV["CRYSTAL"]? + puts PROGRAM_NAME + puts ARGV + CRYSTAL + + Process.run(ENV["CRYSTAL_SPEC_COMPILER_BIN"]? || "bin/crystal", ["build", source_file, "-o", path]) + end + + File.exists?(path).should be_true + + process = Process.new(ENV["CRYSTAL_SPEC_COMPILER_BIN"]? || "bin/crystal", + ["external", "foo", "bar"], + output: :pipe, + env: {"PATH" => {ENV["PATH"], File.dirname(path)}.join(Process::PATH_DELIMITER)} + ) + output = process.output.gets_to_end + status = process.wait + status.success?.should be_true + lines = output.lines + lines[0].should match /crystal/ + lines[1].should match /crystal-external/ + lines[2].should eq %(["foo", "bar"]) + end + end +end diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index f8ece87e3d4b..1354594706fb 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -130,6 +130,9 @@ class Crystal::Command else if command.ends_with?(".cr") error "file '#{command}' does not exist" + elsif external_command = Process.find_executable("crystal-#{command}") + options.shift + Process.exec(external_command, options, env: {"CRYSTAL" => Process.executable_path}) else error "unknown command: #{command}" end From ef306fb5aba4a03c497317ae7bf9f5da0c7a3549 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 18:13:08 +0200 Subject: [PATCH 093/193] Fix: reinit event loop first after fork (UNIX) (#14975) Signal handling manipulate pipes (file descriptors) on UNIX, and messing with the evloop after fork can affect the parent process evloop in some cases. --- src/kernel.cr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/kernel.cr b/src/kernel.cr index 8c84a197b78f..14e66bd4fade 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -584,14 +584,14 @@ end # Hooks are defined here due to load order problems. def self.after_fork_child_callbacks @@after_fork_child_callbacks ||= [ - # clean ups (don't depend on event loop): + # reinit event loop first: + ->{ Crystal::EventLoop.current.after_fork }, + + # reinit signal handling: ->Crystal::System::Signal.after_fork, ->Crystal::System::SignalChildHandler.after_fork, - # reinit event loop: - ->{ Crystal::EventLoop.current.after_fork }, - - # more clean ups (may depend on event loop): + # additional reinitialization ->Random::DEFAULT.new_seed, ] of -> Nil end From 136f85ede8c7a3985eff2e4a0871f3645876e36f Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 18:13:49 +0200 Subject: [PATCH 094/193] Don't involve evloop after fork in System::Process.spawn (UNIX) (#14974) Refactors spawning child processes on UNIX that relies on fork/exec to not involve the event loop after fork and before exec. We still continue to rely on the eventloop in the parent process, of course. * Extract Crystal::System::FileDescriptor.system_pipe (UNIX) * Fix: avoid evloop after fork to report failures in Process.spawn (UNIX) * Fix: don't involve evloop in System::Process.reopen_io (UNIX) This is called after fork before exec to reopen the stdio. We can leverage some abstractions (set blocking, unset cloexec) but musn't call to methods that involve the evloop and would mess with the parent evloop. --- src/crystal/system/unix/file_descriptor.cr | 28 ++++++++++-- src/crystal/system/unix/process.cr | 53 ++++++++++++---------- 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 759802f4323e..60515b701136 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -203,6 +203,14 @@ module Crystal::System::FileDescriptor end def self.pipe(read_blocking, write_blocking) + pipe_fds = system_pipe + r = IO::FileDescriptor.new(pipe_fds[0], read_blocking) + w = IO::FileDescriptor.new(pipe_fds[1], write_blocking) + w.sync = true + {r, w} + end + + def self.system_pipe : StaticArray(LibC::Int, 2) pipe_fds = uninitialized StaticArray(LibC::Int, 2) {% if LibC.has_method?(:pipe2) %} @@ -219,11 +227,7 @@ module Crystal::System::FileDescriptor end {% end %} - r = IO::FileDescriptor.new(pipe_fds[0], read_blocking) - w = IO::FileDescriptor.new(pipe_fds[1], write_blocking) - w.sync = true - - {r, w} + pipe_fds end def self.pread(file, buffer, offset) @@ -255,6 +259,20 @@ module Crystal::System::FileDescriptor io end + # Helper to write *size* values at *pointer* to a given *fd*. + def self.write_fully(fd : LibC::Int, pointer : Pointer, size : Int32 = 1) : Nil + write_fully(fd, Slice.new(pointer, size).unsafe_slice_of(UInt8)) + end + + # Helper to fully write a slice to a given *fd*. + def self.write_fully(fd : LibC::Int, slice : Slice(UInt8)) : Nil + until slice.size == 0 + size = LibC.write(fd, slice, slice.size) + break if size == -1 + slice += size + end + end + private def system_echo(enable : Bool, mode = nil) new_mode = mode || FileDescriptor.tcgetattr(fd) flags = LibC::ECHO | LibC::ECHOE | LibC::ECHOK | LibC::ECHONL diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 4a540fa53a3d..06b18aea7b1d 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -231,44 +231,47 @@ struct Crystal::System::Process end def self.spawn(command_args, env, clear_env, input, output, error, chdir) - reader_pipe, writer_pipe = IO.pipe + r, w = FileDescriptor.system_pipe pid = self.fork(will_exec: true) if !pid + LibC.close(r) begin - reader_pipe.close - writer_pipe.close_on_exec = true self.try_replace(command_args, env, clear_env, input, output, error, chdir) - writer_pipe.write_byte(1) - writer_pipe.write_bytes(Errno.value.to_i) + byte = 1_u8 + errno = Errno.value.to_i32 + FileDescriptor.write_fully(w, pointerof(byte)) + FileDescriptor.write_fully(w, pointerof(errno)) rescue ex - writer_pipe.write_byte(0) + byte = 0_u8 message = ex.inspect_with_backtrace - writer_pipe.write_bytes(message.bytesize) - writer_pipe << message - writer_pipe.close + FileDescriptor.write_fully(w, pointerof(byte)) + FileDescriptor.write_fully(w, message.to_slice) ensure + LibC.close(w) LibC._exit 127 end end - writer_pipe.close + LibC.close(w) + reader_pipe = IO::FileDescriptor.new(r, blocking: false) + begin case reader_pipe.read_byte when nil # Pipe was closed, no error when 0 # Error message coming - message_size = reader_pipe.read_bytes(Int32) - if message_size > 0 - message = String.build(message_size) { |io| IO.copy(reader_pipe, io, message_size) } - end - reader_pipe.close + message = reader_pipe.gets_to_end raise RuntimeError.new("Error executing process: '#{command_args[0]}': #{message}") when 1 # Errno coming - errno = Errno.new(reader_pipe.read_bytes(Int32)) - self.raise_exception_from_errno(command_args[0], errno) + # can't use IO#read_bytes(Int32) because we skipped system/network + # endianness check when writing the integer while read_bytes would; + # we thus read it in the same as order as written + buf = uninitialized StaticArray(UInt8, 4) + reader_pipe.read_fully(buf.to_slice) + raise_exception_from_errno(command_args[0], Errno.new(buf.unsafe_as(Int32))) else raise RuntimeError.new("BUG: Invalid error response received from subprocess") end @@ -339,15 +342,17 @@ struct Crystal::System::Process private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) if src_io.closed? - dst_io.close - return - end + dst_io.file_descriptor_close + else + src_io = to_real_fd(src_io) - src_io = to_real_fd(src_io) + # dst_io.reopen(src_io) + ret = LibC.dup2(src_io.fd, dst_io.fd) + raise IO::Error.from_errno("dup2") if ret == -1 - dst_io.reopen(src_io) - dst_io.blocking = true - dst_io.close_on_exec = false + dst_io.blocking = true + dst_io.close_on_exec = false + end end private def self.to_real_fd(fd : IO::FileDescriptor) From cdd9ccf460641ee17a603c67ff9d23aa1199f14f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 7 Sep 2024 17:43:48 +0800 Subject: [PATCH 095/193] Enable the interpreter on Windows (#14964) --- .github/workflows/win.yml | 37 +++++++++++++++++++++++- .github/workflows/win_build_portable.yml | 2 +- spec/std/http/client/client_spec.cr | 6 ++++ spec/std/http/server/server_spec.cr | 6 ++++ spec/std/http/web_socket_spec.cr | 6 ++++ spec/std/io/io_spec.cr | 33 +++++++++++---------- spec/std/oauth2/client_spec.cr | 6 ++++ spec/std/openssl/ssl/server_spec.cr | 6 ++++ spec/std/openssl/ssl/socket_spec.cr | 6 ++++ spec/std/process_spec.cr | 7 ++++- spec/std/socket/socket_spec.cr | 6 ++++ spec/std/socket/tcp_socket_spec.cr | 6 ++++ spec/std/socket/unix_server_spec.cr | 6 ++++ spec/std/socket/unix_socket_spec.cr | 6 ++++ spec/std/uuid_spec.cr | 1 + src/compiler/crystal/loader/msvc.cr | 37 ++++++++++++++++++------ src/kernel.cr | 13 +++++++++ 17 files changed, 163 insertions(+), 27 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 05f74b6378c6..568828b17bee 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -7,6 +7,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} env: + SPEC_SPLIT_DOTS: 160 CI_LLVM_VERSION: "18.1.1" jobs: @@ -266,13 +267,47 @@ jobs: run: make -f Makefile.win samples x86_64-windows-release: - if: github.repository_owner == 'crystal-lang' && (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/ci/')) needs: [x86_64-windows-libs, x86_64-windows-dlls, x86_64-windows-llvm-libs, x86_64-windows-llvm-dlls] uses: ./.github/workflows/win_build_portable.yml with: release: true llvm_version: "18.1.1" + x86_64-windows-test-interpreter: + runs-on: windows-2022 + needs: [x86_64-windows-release] + steps: + - name: Disable CRLF line ending substitution + run: | + git config --global core.autocrlf false + + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Download Crystal executable + uses: actions/download-artifact@v4 + with: + name: crystal-release + path: build + + - name: Restore LLVM + uses: actions/cache/restore@v4 + with: + path: llvm + key: llvm-libs-${{ env.CI_LLVM_VERSION }}-msvc + fail-on-cache-miss: true + + - name: Set up environment + run: | + Add-Content $env:GITHUB_PATH "$(pwd)\build" + Add-Content $env:GITHUB_ENV "CRYSTAL_SPEC_COMPILER_BIN=$(pwd)\build\crystal.exe" + + - name: Run stdlib specs with interpreter + run: bin\crystal i spec\std_spec.cr + + - name: Run primitives specs with interpreter + run: bin\crystal i spec\primitives_spec.cr + x86_64-windows-installer: if: github.repository_owner == 'crystal-lang' && (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/ci/')) runs-on: windows-2022 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index d2ed6469d264..12ee17da9e68 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -114,7 +114,7 @@ jobs: - name: Build Crystal run: | bin/crystal.bat env - make -f Makefile.win -B ${{ inputs.release && 'release=1' || '' }} + make -f Makefile.win -B ${{ inputs.release && 'release=1' || '' }} interpreter=1 - name: Download shards release uses: actions/checkout@v4 diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 451960a8c79f..6bd04ab3e2f2 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -6,6 +6,12 @@ require "http/server" require "http/log" require "log/spec" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending HTTP::Client + {% skip_file %} +{% end %} + private def test_server(host, port, read_time = 0.seconds, content_type = "text/plain", write_response = true, &) server = TCPServer.new(host, port) begin diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index 5e1e5dab76f6..3980084ea414 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -4,6 +4,12 @@ require "http/client" require "../../../support/ssl" require "../../../support/channel" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending HTTP::Server + {% skip_file %} +{% end %} + # TODO: replace with `HTTP::Client.get` once it supports connecting to Unix socket (#2735) private def unix_request(path) UNIXSocket.open(path) do |io| diff --git a/spec/std/http/web_socket_spec.cr b/spec/std/http/web_socket_spec.cr index 75a54e91fb2e..164a1d067df5 100644 --- a/spec/std/http/web_socket_spec.cr +++ b/spec/std/http/web_socket_spec.cr @@ -7,6 +7,12 @@ require "../../support/fibers" require "../../support/ssl" require "../socket/spec_helper.cr" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending HTTP::WebSocket + {% skip_file %} +{% end %} + private def assert_text_packet(packet, size, final = false) assert_packet packet, HTTP::WebSocket::Protocol::Opcode::TEXT, size, final: final end diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 6974a9fe3466..c584ec81a1e8 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -816,23 +816,26 @@ describe IO do io.gets_to_end.should eq("\r\nFoo\nBar") end - it "gets ascii from socket (#9056)" do - server = TCPServer.new "localhost", 0 - sock = TCPSocket.new "localhost", server.local_address.port - begin - sock.set_encoding("ascii") - spawn do - client = server.accept - message = client.gets - client << "#{message}\n" + # TODO: Windows networking in the interpreter requires #12495 + {% unless flag?(:interpreted) || flag?(:win32) %} + it "gets ascii from socket (#9056)" do + server = TCPServer.new "localhost", 0 + sock = TCPSocket.new "localhost", server.local_address.port + begin + sock.set_encoding("ascii") + spawn do + client = server.accept + message = client.gets + client << "#{message}\n" + end + sock << "K\n" + sock.gets.should eq("K") + ensure + server.close + sock.close end - sock << "K\n" - sock.gets.should eq("K") - ensure - server.close - sock.close end - end + {% end %} end describe "encode" do diff --git a/spec/std/oauth2/client_spec.cr b/spec/std/oauth2/client_spec.cr index 3ee66e29ab49..ee445f3426e7 100644 --- a/spec/std/oauth2/client_spec.cr +++ b/spec/std/oauth2/client_spec.cr @@ -3,6 +3,12 @@ require "oauth2" require "http/server" require "../http/spec_helper" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending OAuth2::Client + {% skip_file %} +{% end %} + describe OAuth2::Client do describe "authorization uri" do it "gets with default endpoint" do diff --git a/spec/std/openssl/ssl/server_spec.cr b/spec/std/openssl/ssl/server_spec.cr index 2e0e413a618d..8618ed780a50 100644 --- a/spec/std/openssl/ssl/server_spec.cr +++ b/spec/std/openssl/ssl/server_spec.cr @@ -3,6 +3,12 @@ require "socket" require "../../spec_helper" require "../../../support/ssl" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending OpenSSL::SSL::Server + {% skip_file %} +{% end %} + describe OpenSSL::SSL::Server do it "sync_close" do TCPServer.open(0) do |tcp_server| diff --git a/spec/std/openssl/ssl/socket_spec.cr b/spec/std/openssl/ssl/socket_spec.cr index bbc5b11e4b9b..47374ce28cca 100644 --- a/spec/std/openssl/ssl/socket_spec.cr +++ b/spec/std/openssl/ssl/socket_spec.cr @@ -4,6 +4,12 @@ require "../../spec_helper" require "../../socket/spec_helper" require "../../../support/ssl" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending OpenSSL::SSL::Socket + {% skip_file %} +{% end %} + describe OpenSSL::SSL::Socket do describe OpenSSL::SSL::Socket::Server do it "auto accept client by default" do diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 57f90121c26b..d41ee0bed242 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -55,7 +55,12 @@ private def newline end # interpreted code doesn't receive SIGCHLD for `#wait` to work (#12241) -pending_interpreted describe: Process do +{% if flag?(:interpreted) && !flag?(:win32) %} + pending Process + {% skip_file %} +{% end %} + +describe Process do describe ".new" do it "raises if command doesn't exist" do expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index 2127e196b746..98555937dea3 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -2,6 +2,12 @@ require "./spec_helper" require "../../support/tempfile" require "../../support/win32" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending Socket + {% skip_file %} +{% end %} + describe Socket, tags: "network" do describe ".unix" do it "creates a unix socket" do diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index 68c00ccd2e79..f3d460f92401 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -3,6 +3,12 @@ require "./spec_helper" require "../../support/win32" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending TCPSocket + {% skip_file %} +{% end %} + describe TCPSocket, tags: "network" do describe "#connect" do each_ip_family do |family, address| diff --git a/spec/std/socket/unix_server_spec.cr b/spec/std/socket/unix_server_spec.cr index ca364f08667c..60f0279b4091 100644 --- a/spec/std/socket/unix_server_spec.cr +++ b/spec/std/socket/unix_server_spec.cr @@ -4,6 +4,12 @@ require "../../support/fibers" require "../../support/channel" require "../../support/tempfile" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending UNIXServer + {% skip_file %} +{% end %} + describe UNIXServer do describe ".new" do it "raises when path is too long" do diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index 24777bada67f..c51f37193c0e 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -2,6 +2,12 @@ require "spec" require "socket" require "../../support/tempfile" +# TODO: Windows networking in the interpreter requires #12495 +{% if flag?(:interpreted) && flag?(:win32) %} + pending UNIXSocket + {% skip_file %} +{% end %} + describe UNIXSocket do it "raises when path is too long" do with_tempfile("unix_socket-too_long-#{("a" * 2048)}.sock") do |path| diff --git a/spec/std/uuid_spec.cr b/spec/std/uuid_spec.cr index 48cc3351a3c6..5d7e627031f0 100644 --- a/spec/std/uuid_spec.cr +++ b/spec/std/uuid_spec.cr @@ -1,6 +1,7 @@ require "spec" require "uuid" require "spec/helpers/string" +require "../support/wasm32" describe "UUID" do describe "#==" do diff --git a/src/compiler/crystal/loader/msvc.cr b/src/compiler/crystal/loader/msvc.cr index 05bf988c9218..772e4c5c232f 100644 --- a/src/compiler/crystal/loader/msvc.cr +++ b/src/compiler/crystal/loader/msvc.cr @@ -133,15 +133,25 @@ class Crystal::Loader end def load_file?(path : String | ::Path) : Bool + # API sets shouldn't be linked directly from linker flags, but just in case + if api_set?(path) + return load_dll?(path.to_s) + end + return false unless File.file?(path) # On Windows, each `.lib` import library may reference any number of `.dll` # files, whose base names may not match the library's. Thus it is necessary # to extract this information from the library archive itself. - System::LibraryArchive.imported_dlls(path).each do |dll| - dll_full_path = @dll_search_paths.try &.each do |search_path| - full_path = File.join(search_path, dll) - break full_path if File.file?(full_path) + System::LibraryArchive.imported_dlls(path).all? do |dll| + # API set names do not refer to physical filenames despite ending with + # `.dll`, and therefore should not use a path search: + # https://learn.microsoft.com/en-us/cpp/windows/universal-crt-deployment?view=msvc-170#local-deployment + unless api_set?(dll) + dll_full_path = @dll_search_paths.try &.each do |search_path| + full_path = File.join(search_path, dll) + break full_path if File.file?(full_path) + end end dll = dll_full_path || dll @@ -152,13 +162,16 @@ class Crystal::Loader # # Note that the compiler's directory and PATH are effectively searched # twice when coming from the interpreter - handle = open_library(dll) - return false unless handle - - @handles << handle - @loaded_libraries << (module_filename(handle) || dll) + load_dll?(dll) end + end + + private def load_dll?(dll) + handle = open_library(dll) + return false unless handle + @handles << handle + @loaded_libraries << (module_filename(handle) || dll) true end @@ -190,6 +203,12 @@ class Crystal::Loader @handles.clear end + # Returns whether *dll* names an API set according to: + # https://learn.microsoft.com/en-us/windows/win32/apiindex/windows-apisets#api-set-contract-names + private def api_set?(dll) + dll.to_s.matches?(/^(?:api-|ext-)[a-zA-Z0-9-]*l\d+-\d+-\d+\.dll$/) + end + private def module_filename(handle) Crystal::System.retry_wstr_buffer do |buffer, small_buf| len = LibC.GetModuleFileNameW(handle, buffer, buffer.size) diff --git a/src/kernel.cr b/src/kernel.cr index 14e66bd4fade..16c4a770309a 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -616,3 +616,16 @@ end Crystal::System::Signal.setup_default_handlers {% end %} {% end %} + +# This is a temporary workaround to ensure there is always something in the IOCP +# event loop being awaited, since both the interrupt loop and the fiber stack +# pool collector are disabled in interpreted code. Without this, asynchronous +# code that bypasses `Crystal::IOCP::OverlappedOperation` does not currently +# work, see https://github.com/crystal-lang/crystal/pull/14949#issuecomment-2328314463 +{% if flag?(:interpreted) && flag?(:win32) %} + spawn(name: "Interpreter idle loop") do + while true + sleep 1.day + end + end +{% end %} From bdddae759a2e1f77306e1a54523a136a0b64c078 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 9 Sep 2024 22:12:05 +0200 Subject: [PATCH 096/193] Add methods to Crystal::EventLoop (#14977) Add `#after_fork_before_exec` to allow an evloop to do some cleanup before exec (UNIX only). Add `#remove(io)` to allow an evloop to free resources when the IO is closed in a GC finalizer. --- src/crystal/scheduler.cr | 6 ++++++ src/crystal/system/event_loop.cr | 5 +++++ src/crystal/system/event_loop/file_descriptor.cr | 8 ++++++++ src/crystal/system/event_loop/socket.cr | 8 ++++++++ src/crystal/system/file_descriptor.cr | 4 ++++ src/crystal/system/socket.cr | 4 ++++ src/crystal/system/unix/event_loop_libevent.cr | 9 +++++++++ src/crystal/system/unix/process.cr | 3 +++ src/crystal/system/wasi/event_loop.cr | 6 ++++++ src/crystal/system/win32/event_loop_iocp.cr | 6 ++++++ src/io/file_descriptor.cr | 1 + src/socket.cr | 1 + 12 files changed, 61 insertions(+) diff --git a/src/crystal/scheduler.cr b/src/crystal/scheduler.cr index d3634e9aea6a..bed98ef4d05b 100644 --- a/src/crystal/scheduler.cr +++ b/src/crystal/scheduler.cr @@ -24,6 +24,12 @@ class Crystal::Scheduler Thread.current.scheduler.@event_loop end + def self.event_loop? + if scheduler = Thread.current?.try(&.scheduler?) + scheduler.@event_loop + end + end + def self.enqueue(fiber : Fiber) : Nil Crystal.trace :sched, "enqueue", fiber: fiber do thread = Thread.current diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index 46954e6034ff..fb1042b21f96 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -17,6 +17,11 @@ abstract class Crystal::EventLoop Crystal::Scheduler.event_loop end + @[AlwaysInline] + def self.current? : self? + Crystal::Scheduler.event_loop? + end + # Runs the loop. # # Returns immediately if events are activable. Set `blocking` to false to diff --git a/src/crystal/system/event_loop/file_descriptor.cr b/src/crystal/system/event_loop/file_descriptor.cr index a041263609d9..5fb6cbb95cb0 100644 --- a/src/crystal/system/event_loop/file_descriptor.cr +++ b/src/crystal/system/event_loop/file_descriptor.cr @@ -19,5 +19,13 @@ abstract class Crystal::EventLoop # Closes the file descriptor resource. abstract def close(file_descriptor : Crystal::System::FileDescriptor) : Nil + + # Removes the file descriptor from the event loop. Can be used to free up + # memory resources associated with the file descriptor, as well as removing + # the file descriptor from kernel data structures. + # + # Called by `::IO::FileDescriptor#finalize` before closing the file + # descriptor. Errors shall be silently ignored. + abstract def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil end end diff --git a/src/crystal/system/event_loop/socket.cr b/src/crystal/system/event_loop/socket.cr index e6f35478b487..6309aed391e0 100644 --- a/src/crystal/system/event_loop/socket.cr +++ b/src/crystal/system/event_loop/socket.cr @@ -62,5 +62,13 @@ abstract class Crystal::EventLoop # Closes the socket. abstract def close(socket : ::Socket) : Nil + + # Removes the socket from the event loop. Can be used to free up memory + # resources associated with the socket, as well as removing the socket from + # kernel data structures. + # + # Called by `::Socket#finalize` before closing the socket. Errors shall be + # silently ignored. + abstract def remove(socket : ::Socket) : Nil end end diff --git a/src/crystal/system/file_descriptor.cr b/src/crystal/system/file_descriptor.cr index 481e00982e25..03868bc07034 100644 --- a/src/crystal/system/file_descriptor.cr +++ b/src/crystal/system/file_descriptor.cr @@ -39,6 +39,10 @@ module Crystal::System::FileDescriptor event_loop.write(self, slice) end + private def event_loop? : Crystal::EventLoop::FileDescriptor? + Crystal::EventLoop.current? + end + private def event_loop : Crystal::EventLoop::FileDescriptor Crystal::EventLoop.current end diff --git a/src/crystal/system/socket.cr b/src/crystal/system/socket.cr index 10f902e9f0c1..8d5e8c9afaf0 100644 --- a/src/crystal/system/socket.cr +++ b/src/crystal/system/socket.cr @@ -99,6 +99,10 @@ module Crystal::System::Socket # Also used in `Socket#finalize` # def socket_close + private def event_loop? : Crystal::EventLoop::Socket? + Crystal::EventLoop.current? + end + private def event_loop : Crystal::EventLoop::Socket Crystal::EventLoop.current end diff --git a/src/crystal/system/unix/event_loop_libevent.cr b/src/crystal/system/unix/event_loop_libevent.cr index b67bad63ff2f..4594f07ffe66 100644 --- a/src/crystal/system/unix/event_loop_libevent.cr +++ b/src/crystal/system/unix/event_loop_libevent.cr @@ -4,6 +4,9 @@ require "./event_libevent" class Crystal::LibEvent::EventLoop < Crystal::EventLoop private getter(event_base) { Crystal::LibEvent::Event::Base.new } + def after_fork_before_exec : Nil + end + {% unless flag?(:preview_mt) %} # Reinitializes the event loop after a fork. def after_fork : Nil @@ -93,6 +96,9 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop file_descriptor.evented_close end + def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + end + def read(socket : ::Socket, slice : Bytes) : Int32 evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 @@ -186,6 +192,9 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop socket.evented_close end + def remove(socket : ::Socket) : Nil + end + def evented_read(target, errno_msg : String, &) : Int32 loop do bytes_read = yield diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 06b18aea7b1d..420030f8ba53 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -185,6 +185,9 @@ struct Crystal::System::Process # child: pid = nil if will_exec + # notify event loop + Crystal::EventLoop.current.after_fork_before_exec + # reset signal handlers, then sigmask (inherited on exec): Crystal::System::Signal.after_fork_before_exec LibC.sigemptyset(pointerof(newmask)) diff --git a/src/crystal/system/wasi/event_loop.cr b/src/crystal/system/wasi/event_loop.cr index ba657b917154..c804c4be27aa 100644 --- a/src/crystal/system/wasi/event_loop.cr +++ b/src/crystal/system/wasi/event_loop.cr @@ -53,6 +53,9 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop file_descriptor.evented_close end + def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + end + def read(socket : ::Socket, slice : Bytes) : Int32 evented_read(socket, "Error reading socket") do LibC.recv(socket.fd, slice, slice.size, 0).to_i32 @@ -85,6 +88,9 @@ class Crystal::Wasi::EventLoop < Crystal::EventLoop socket.evented_close end + def remove(socket : ::Socket) : Nil + end + def evented_read(target, errno_msg : String, &) : Int32 loop do bytes_read = yield diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index d1aae09b680a..d3cfaf98d853 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -161,6 +161,9 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop LibC.CancelIoEx(file_descriptor.windows_handle, nil) unless file_descriptor.system_blocking? end + def remove(file_descriptor : Crystal::System::FileDescriptor) : Nil + end + private def wsa_buffer(bytes) wsabuf = LibC::WSABUF.new wsabuf.len = bytes.size @@ -271,6 +274,9 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop def close(socket : ::Socket) : Nil end + + def remove(socket : ::Socket) : Nil + end end class Crystal::IOCP::Event diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index 622229e43e00..a9b303b4b58c 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -255,6 +255,7 @@ class IO::FileDescriptor < IO def finalize return if closed? || !close_on_finalize? + event_loop?.try(&.remove(self)) file_descriptor_close { } # ignore error end diff --git a/src/socket.cr b/src/socket.cr index 1d367f805343..e97deea9eb04 100644 --- a/src/socket.cr +++ b/src/socket.cr @@ -430,6 +430,7 @@ class Socket < IO def finalize return if closed? + event_loop?.try(&.remove(self)) socket_close { } # ignore error end From 849e0d7ad61448eb5b9c9cbd7e1296f41d0f88f9 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 10 Sep 2024 04:12:58 +0800 Subject: [PATCH 097/193] Make `Crystal::IOCP::OverlappedOperation` abstract (#14987) This allows different overlapped operations to provide their own closure data, instead of putting everything in one big class, such as in https://github.com/crystal-lang/crystal/pull/14979#discussion_r1746549086. --- src/crystal/system/win32/file_descriptor.cr | 4 +- src/crystal/system/win32/iocp.cr | 121 ++++++++++++-------- src/crystal/system/win32/socket.cr | 8 +- 3 files changed, 79 insertions(+), 54 deletions(-) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index d4831d9528cb..cdd23e3ed54d 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -234,7 +234,7 @@ module Crystal::System::FileDescriptor end private def lock_file(handle, flags) - IOCP::OverlappedOperation.run(handle) do |operation| + IOCP::IOOverlappedOperation.run(handle) do |operation| result = LibC.LockFileEx(handle, flags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) if result == 0 @@ -260,7 +260,7 @@ module Crystal::System::FileDescriptor end private def unlock_file(handle) - IOCP::OverlappedOperation.run(handle) do |operation| + IOCP::IOOverlappedOperation.run(handle) do |operation| result = LibC.UnlockFileEx(handle, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, operation) if result == 0 diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index ba87ed123f22..384784a193db 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -78,39 +78,66 @@ module Crystal::IOCP end end - class OverlappedOperation + abstract class OverlappedOperation enum State STARTED DONE end + abstract def wait_for_result(timeout, & : WinError ->) + private abstract def try_cancel : Bool + @overlapped = LibC::OVERLAPPED.new @fiber = Fiber.current @state : State = :started - def initialize(@handle : LibC::HANDLE) + def self.run(*args, **opts, &) + operation_storage = uninitialized ReferenceStorage(self) + operation = unsafe_construct(pointerof(operation_storage), *args, **opts) + yield operation end - def initialize(handle : LibC::SOCKET) - @handle = LibC::HANDLE.new(handle) + def self.unbox(overlapped : LibC::OVERLAPPED*) : self + start = overlapped.as(Pointer(UInt8)) - offsetof(self, @overlapped) + Box(self).unbox(start.as(Pointer(Void))) end - def self.run(handle, &) - operation_storage = uninitialized ReferenceStorage(OverlappedOperation) - operation = OverlappedOperation.unsafe_construct(pointerof(operation_storage), handle) - yield operation + def to_unsafe + pointerof(@overlapped) end - def self.unbox(overlapped : LibC::OVERLAPPED*) - start = overlapped.as(Pointer(UInt8)) - offsetof(OverlappedOperation, @overlapped) - Box(OverlappedOperation).unbox(start.as(Pointer(Void))) + protected def schedule(&) + done! + yield @fiber end - def to_unsafe - pointerof(@overlapped) + private def done! + @fiber.cancel_timeout + @state = :done end - def wait_for_result(timeout, &) + private def wait_for_completion(timeout) + if timeout + sleep timeout + else + Fiber.suspend + end + + unless @state.done? + if try_cancel + # Wait for cancellation to complete. We must not free the operation + # until it's completed. + Fiber.suspend + end + end + end + end + + class IOOverlappedOperation < OverlappedOperation + def initialize(@handle : LibC::HANDLE) + end + + def wait_for_result(timeout, & : WinError ->) wait_for_completion(timeout) result = LibC.GetOverlappedResult(@handle, self, out bytes, 0) @@ -124,11 +151,35 @@ module Crystal::IOCP bytes end - def wait_for_wsa_result(timeout, &) + private def try_cancel : Bool + # Microsoft documentation: + # The application must not free or reuse the OVERLAPPED structure + # associated with the canceled I/O operations until they have completed + # (this does not apply to asynchronous operations that finished + # synchronously, as nothing would be queued to the IOCP) + ret = LibC.CancelIoEx(@handle, self) + if ret.zero? + case error = WinError.value + when .error_not_found? + # Operation has already completed, do nothing + return false + else + raise RuntimeError.from_os_error("CancelIoEx", os_error: error) + end + end + true + end + end + + class WSAOverlappedOperation < OverlappedOperation + def initialize(@handle : LibC::SOCKET) + end + + def wait_for_result(timeout, & : WinError ->) wait_for_completion(timeout) flags = 0_u32 - result = LibC.WSAGetOverlappedResult(LibC::SOCKET.new(@handle.address), self, out bytes, false, pointerof(flags)) + result = LibC.WSAGetOverlappedResult(@handle, self, out bytes, false, pointerof(flags)) if result.zero? error = WinError.wsa_value yield error @@ -139,57 +190,31 @@ module Crystal::IOCP bytes end - protected def schedule(&) - done! - yield @fiber - end - - def done! - @fiber.cancel_timeout - @state = :done - end - - def try_cancel : Bool + private def try_cancel : Bool # Microsoft documentation: # The application must not free or reuse the OVERLAPPED structure # associated with the canceled I/O operations until they have completed # (this does not apply to asynchronous operations that finished # synchronously, as nothing would be queued to the IOCP) - ret = LibC.CancelIoEx(@handle, self) + ret = LibC.CancelIoEx(Pointer(Void).new(@handle), self) if ret.zero? case error = WinError.value when .error_not_found? # Operation has already completed, do nothing return false else - raise RuntimeError.from_os_error("CancelIOEx", os_error: error) + raise RuntimeError.from_os_error("CancelIoEx", os_error: error) end end true end - - def wait_for_completion(timeout) - if timeout - sleep timeout - else - Fiber.suspend - end - - unless @state.done? - if try_cancel - # Wait for cancellation to complete. We must not free the operation - # until it's completed. - Fiber.suspend - end - end - end end def self.overlapped_operation(file_descriptor, method, timeout, *, offset = nil, writing = false, &) handle = file_descriptor.windows_handle seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 - OverlappedOperation.run(handle) do |operation| + IOOverlappedOperation.run(handle) do |operation| overlapped = operation.to_unsafe if seekable start_offset = offset || original_offset @@ -243,7 +268,7 @@ module Crystal::IOCP end def self.wsa_overlapped_operation(target, socket, method, timeout, connreset_is_error = true, &) - OverlappedOperation.run(socket) do |operation| + WSAOverlappedOperation.run(socket) do |operation| result, value = yield operation if result == LibC::SOCKET_ERROR @@ -257,7 +282,7 @@ module Crystal::IOCP return value end - operation.wait_for_wsa_result(timeout) do |error| + operation.wait_for_result(timeout) do |error| case error when .wsa_io_incomplete?, .error_operation_aborted? raise IO::TimeoutError.new("#{method} timed out") diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 3172be467836..5ed235e24574 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -129,7 +129,7 @@ module Crystal::System::Socket # :nodoc: def overlapped_connect(socket, method, timeout, &) - IOCP::OverlappedOperation.run(socket) do |operation| + IOCP::WSAOverlappedOperation.run(socket) do |operation| result = yield operation if result == 0 @@ -145,7 +145,7 @@ module Crystal::System::Socket return nil end - operation.wait_for_wsa_result(timeout) do |error| + operation.wait_for_result(timeout) do |error| case error when .wsa_io_incomplete?, .wsaeconnrefused? return ::Socket::ConnectError.from_os_error(method, error) @@ -192,7 +192,7 @@ module Crystal::System::Socket end def overlapped_accept(socket, method, &) - IOCP::OverlappedOperation.run(socket) do |operation| + IOCP::WSAOverlappedOperation.run(socket) do |operation| result = yield operation if result == 0 @@ -206,7 +206,7 @@ module Crystal::System::Socket return true end - operation.wait_for_wsa_result(read_timeout) do |error| + operation.wait_for_result(read_timeout) do |error| case error when .wsa_io_incomplete?, .wsaenotsock? return false From c8ecd9339f64a72c83f76f7c65975856ba96e3b5 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 9 Sep 2024 22:14:43 +0200 Subject: [PATCH 098/193] Refactor `EventLoop` interface for sleeps & select timeouts (#14980) A couple refactors related to select timeout and the signature of `Crystal::EventLoop::Event#add` that only needs to handle a nilable for `libevent` (because it caches events); other loop don't need that. Sleep and registering a select action are always setting a `Time::Span` and don't need to deal with a nilable. The exception is `IO::Evented` that keeps a cache of `libevent` events and must be able to remove the timeout in case `@read_timeout` would have been set to nil before a second wait on read. --- src/channel/select/timeout_action.cr | 8 +++++--- src/crystal/system/event_loop.cr | 2 +- src/crystal/system/unix/event_libevent.cr | 20 ++++++++++---------- src/crystal/system/wasi/event_loop.cr | 5 ++++- src/crystal/system/win32/event_loop_iocp.cr | 4 ++-- src/fiber.cr | 5 +++-- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/channel/select/timeout_action.cr b/src/channel/select/timeout_action.cr index 9240b480db1a..39986197bbdc 100644 --- a/src/channel/select/timeout_action.cr +++ b/src/channel/select/timeout_action.cr @@ -58,9 +58,11 @@ class Channel(T) end def time_expired(fiber : Fiber) : Nil - if @select_context.try &.try_trigger - fiber.enqueue - end + fiber.enqueue if time_expired? + end + + def time_expired? : Bool + @select_context.try &.try_trigger || false end end end diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index fb1042b21f96..fe973ec8c99e 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -56,7 +56,7 @@ abstract class Crystal::EventLoop abstract def free : Nil # Adds a new timeout to this event. - abstract def add(timeout : Time::Span?) : Nil + abstract def add(timeout : Time::Span) : Nil end end diff --git a/src/crystal/system/unix/event_libevent.cr b/src/crystal/system/unix/event_libevent.cr index 21d6765646d1..32578e5aba9a 100644 --- a/src/crystal/system/unix/event_libevent.cr +++ b/src/crystal/system/unix/event_libevent.cr @@ -19,16 +19,16 @@ module Crystal::LibEvent @freed = false end - def add(timeout : Time::Span?) : Nil - if timeout - timeval = LibC::Timeval.new( - tv_sec: LibC::TimeT.new(timeout.total_seconds), - tv_usec: timeout.nanoseconds // 1_000 - ) - LibEvent2.event_add(@event, pointerof(timeval)) - else - LibEvent2.event_add(@event, nil) - end + def add(timeout : Time::Span) : Nil + timeval = LibC::Timeval.new( + tv_sec: LibC::TimeT.new(timeout.total_seconds), + tv_usec: timeout.nanoseconds // 1_000 + ) + LibEvent2.event_add(@event, pointerof(timeval)) + end + + def add(timeout : Nil) : Nil + LibEvent2.event_add(@event, nil) end def free : Nil diff --git a/src/crystal/system/wasi/event_loop.cr b/src/crystal/system/wasi/event_loop.cr index c804c4be27aa..3cce9ba8361c 100644 --- a/src/crystal/system/wasi/event_loop.cr +++ b/src/crystal/system/wasi/event_loop.cr @@ -132,7 +132,10 @@ end struct Crystal::Wasi::Event include Crystal::EventLoop::Event - def add(timeout : Time::Span?) : Nil + def add(timeout : Time::Span) : Nil + end + + def add(timeout : Nil) : Nil end def free : Nil diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index d3cfaf98d853..d3655fdb5861 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -298,8 +298,8 @@ class Crystal::IOCP::Event free end - def add(timeout : Time::Span?) : Nil - @wake_at = timeout ? Time.monotonic + timeout : Time.monotonic + def add(timeout : Time::Span) : Nil + @wake_at = Time.monotonic + timeout Crystal::EventLoop.current.enqueue(self) end end diff --git a/src/fiber.cr b/src/fiber.cr index 0d471e5a96e4..1086ebdd3669 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -234,20 +234,21 @@ class Fiber end # :nodoc: - def timeout(timeout : Time::Span?, select_action : Channel::TimeoutAction? = nil) : Nil + def timeout(timeout : Time::Span, select_action : Channel::TimeoutAction) : Nil @timeout_select_action = select_action timeout_event.add(timeout) end # :nodoc: def cancel_timeout : Nil + return unless @timeout_select_action @timeout_select_action = nil @timeout_event.try &.delete end # The current fiber will resume after a period of time. # The timeout can be cancelled with `cancel_timeout` - def self.timeout(timeout : Time::Span?, select_action : Channel::TimeoutAction? = nil) : Nil + def self.timeout(timeout : Time::Span, select_action : Channel::TimeoutAction) : Nil Fiber.current.timeout(timeout, select_action) end From 4023f522935743ba6966b627c0976fa653739975 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 10 Sep 2024 14:25:48 +0800 Subject: [PATCH 099/193] Remove some uses of deprecated overloads in standard library specs (#14963) * `File.real_path` * `Benchmark::IPS::Job.new` * `File.executable?`, `.readable?`, `.writable?` * `#read_timeout=`, `#write_timeout=`, `#connect_timeout=` --- spec/compiler/semantic/warnings_spec.cr | 4 +- spec/std/benchmark_spec.cr | 2 +- spec/std/dir_spec.cr | 2 +- spec/std/file_spec.cr | 269 ++++++++++++------------ spec/std/http/client/client_spec.cr | 6 +- spec/std/http/server/server_spec.cr | 2 +- spec/std/io/io_spec.cr | 4 +- spec/std/socket/socket_spec.cr | 2 +- spec/std/socket/unix_socket_spec.cr | 4 +- 9 files changed, 149 insertions(+), 146 deletions(-) diff --git a/spec/compiler/semantic/warnings_spec.cr b/spec/compiler/semantic/warnings_spec.cr index 6c6914c60fe5..e8bbad7b7c29 100644 --- a/spec/compiler/semantic/warnings_spec.cr +++ b/spec/compiler/semantic/warnings_spec.cr @@ -234,7 +234,7 @@ describe "Semantic: warnings" do # NOTE tempfile might be created in symlinked folder # which affects how to match current dir /var/folders/... # with the real path /private/var/folders/... - path = File.real_path(path) + path = File.realpath(path) main_filename = File.join(path, "main.cr") output_filename = File.join(path, "main") @@ -416,7 +416,7 @@ describe "Semantic: warnings" do # NOTE tempfile might be created in symlinked folder # which affects how to match current dir /var/folders/... # with the real path /private/var/folders/... - path = File.real_path(path) + path = File.realpath(path) main_filename = File.join(path, "main.cr") output_filename = File.join(path, "main") diff --git a/spec/std/benchmark_spec.cr b/spec/std/benchmark_spec.cr index 8113f5f03a4c..4a46798b2436 100644 --- a/spec/std/benchmark_spec.cr +++ b/spec/std/benchmark_spec.cr @@ -12,7 +12,7 @@ describe Benchmark::IPS::Job do it "works in general / integration test" do # test several things to avoid running a benchmark over and over again in # the specs - j = Benchmark::IPS::Job.new(0.001, 0.001, interactive: false) + j = Benchmark::IPS::Job.new(1.millisecond, 1.millisecond, interactive: false) a = j.report("a") { sleep 1.milliseconds } b = j.report("b") { sleep 2.milliseconds } diff --git a/spec/std/dir_spec.cr b/spec/std/dir_spec.cr index 439da15becd9..d37483eba947 100644 --- a/spec/std/dir_spec.cr +++ b/spec/std/dir_spec.cr @@ -643,7 +643,7 @@ describe "Dir" do Dir.mkdir_p path # Resolve any symbolic links in path caused by tmpdir being a link. # For example on macOS, /tmp is a symlink to /private/tmp. - path = File.real_path(path) + path = File.realpath(path) target_path = File.join(path, "target") link_path = File.join(path, "link") diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 07b919bd4a6e..0f88b2028c2f 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -236,136 +236,6 @@ describe "File" do end end - describe "executable?" do - it "gives true" do - crystal = Process.executable_path || pending! "Unable to locate compiler executable" - File.executable?(crystal).should be_true - end - - it "gives false" do - File.executable?(datapath("test_file.txt")).should be_false - end - - it "gives false when the file doesn't exist" do - File.executable?(datapath("non_existing_file.txt")).should be_false - end - - it "gives false when a component of the path is a file" do - File.executable?(datapath("dir", "test_file.txt", "")).should be_false - end - - it "follows symlinks" do - with_tempfile("good_symlink_x.txt", "bad_symlink_x.txt") do |good_path, bad_path| - crystal = Process.executable_path || pending! "Unable to locate compiler executable" - File.symlink(File.expand_path(crystal), good_path) - File.symlink(File.expand_path(datapath("non_existing_file.txt")), bad_path) - - File.executable?(good_path).should be_true - File.executable?(bad_path).should be_false - end - end - end - - describe "readable?" do - it "gives true" do - File.readable?(datapath("test_file.txt")).should be_true - end - - it "gives false when the file doesn't exist" do - File.readable?(datapath("non_existing_file.txt")).should be_false - end - - it "gives false when a component of the path is a file" do - File.readable?(datapath("dir", "test_file.txt", "")).should be_false - end - - # win32 doesn't have a way to make files unreadable via chmod - {% unless flag?(:win32) %} - it "gives false when the file has no read permissions" do - with_tempfile("unreadable.txt") do |path| - File.write(path, "") - File.chmod(path, 0o222) - pending_if_superuser! - File.readable?(path).should be_false - end - end - - it "gives false when the file has no permissions" do - with_tempfile("unaccessible.txt") do |path| - File.write(path, "") - File.chmod(path, 0o000) - pending_if_superuser! - File.readable?(path).should be_false - end - end - - it "follows symlinks" do - with_tempfile("good_symlink_r.txt", "bad_symlink_r.txt", "unreadable.txt") do |good_path, bad_path, unreadable| - File.write(unreadable, "") - File.chmod(unreadable, 0o222) - pending_if_superuser! - - File.symlink(File.expand_path(datapath("test_file.txt")), good_path) - File.symlink(File.expand_path(unreadable), bad_path) - - File.readable?(good_path).should be_true - File.readable?(bad_path).should be_false - end - end - {% end %} - - it "gives false when the symbolic link destination doesn't exist" do - with_tempfile("missing_symlink_r.txt") do |missing_path| - File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) - File.readable?(missing_path).should be_false - end - end - end - - describe "writable?" do - it "gives true" do - File.writable?(datapath("test_file.txt")).should be_true - end - - it "gives false when the file doesn't exist" do - File.writable?(datapath("non_existing_file.txt")).should be_false - end - - it "gives false when a component of the path is a file" do - File.writable?(datapath("dir", "test_file.txt", "")).should be_false - end - - it "gives false when the file has no write permissions" do - with_tempfile("readonly.txt") do |path| - File.write(path, "") - File.chmod(path, 0o444) - pending_if_superuser! - File.writable?(path).should be_false - end - end - - it "follows symlinks" do - with_tempfile("good_symlink_w.txt", "bad_symlink_w.txt", "readonly.txt") do |good_path, bad_path, readonly| - File.write(readonly, "") - File.chmod(readonly, 0o444) - pending_if_superuser! - - File.symlink(File.expand_path(datapath("test_file.txt")), good_path) - File.symlink(File.expand_path(readonly), bad_path) - - File.writable?(good_path).should be_true - File.writable?(bad_path).should be_false - end - end - - it "gives false when the symbolic link destination doesn't exist" do - with_tempfile("missing_symlink_w.txt") do |missing_path| - File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) - File.writable?(missing_path).should be_false - end - end - end - describe "file?" do it "gives true" do File.file?(datapath("test_file.txt")).should be_true @@ -701,6 +571,139 @@ describe "File" do it "tests unequal for file and directory" do File.info(datapath("dir")).should_not eq(File.info(datapath("test_file.txt"))) end + + describe ".executable?" do + it "gives true" do + crystal = Process.executable_path || pending! "Unable to locate compiler executable" + File::Info.executable?(crystal).should be_true + File.executable?(crystal).should be_true # deprecated + end + + it "gives false" do + File::Info.executable?(datapath("test_file.txt")).should be_false + end + + it "gives false when the file doesn't exist" do + File::Info.executable?(datapath("non_existing_file.txt")).should be_false + end + + it "gives false when a component of the path is a file" do + File::Info.executable?(datapath("dir", "test_file.txt", "")).should be_false + end + + it "follows symlinks" do + with_tempfile("good_symlink_x.txt", "bad_symlink_x.txt") do |good_path, bad_path| + crystal = Process.executable_path || pending! "Unable to locate compiler executable" + File.symlink(File.expand_path(crystal), good_path) + File.symlink(File.expand_path(datapath("non_existing_file.txt")), bad_path) + + File::Info.executable?(good_path).should be_true + File::Info.executable?(bad_path).should be_false + end + end + end + + describe ".readable?" do + it "gives true" do + File::Info.readable?(datapath("test_file.txt")).should be_true + File.readable?(datapath("test_file.txt")).should be_true # deprecated + end + + it "gives false when the file doesn't exist" do + File::Info.readable?(datapath("non_existing_file.txt")).should be_false + end + + it "gives false when a component of the path is a file" do + File::Info.readable?(datapath("dir", "test_file.txt", "")).should be_false + end + + # win32 doesn't have a way to make files unreadable via chmod + {% unless flag?(:win32) %} + it "gives false when the file has no read permissions" do + with_tempfile("unreadable.txt") do |path| + File.write(path, "") + File.chmod(path, 0o222) + pending_if_superuser! + File::Info.readable?(path).should be_false + end + end + + it "gives false when the file has no permissions" do + with_tempfile("unaccessible.txt") do |path| + File.write(path, "") + File.chmod(path, 0o000) + pending_if_superuser! + File::Info.readable?(path).should be_false + end + end + + it "follows symlinks" do + with_tempfile("good_symlink_r.txt", "bad_symlink_r.txt", "unreadable.txt") do |good_path, bad_path, unreadable| + File.write(unreadable, "") + File.chmod(unreadable, 0o222) + pending_if_superuser! + + File.symlink(File.expand_path(datapath("test_file.txt")), good_path) + File.symlink(File.expand_path(unreadable), bad_path) + + File::Info.readable?(good_path).should be_true + File::Info.readable?(bad_path).should be_false + end + end + {% end %} + + it "gives false when the symbolic link destination doesn't exist" do + with_tempfile("missing_symlink_r.txt") do |missing_path| + File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) + File::Info.readable?(missing_path).should be_false + end + end + end + + describe ".writable?" do + it "gives true" do + File::Info.writable?(datapath("test_file.txt")).should be_true + File.writable?(datapath("test_file.txt")).should be_true # deprecated + end + + it "gives false when the file doesn't exist" do + File::Info.writable?(datapath("non_existing_file.txt")).should be_false + end + + it "gives false when a component of the path is a file" do + File::Info.writable?(datapath("dir", "test_file.txt", "")).should be_false + end + + it "gives false when the file has no write permissions" do + with_tempfile("readonly.txt") do |path| + File.write(path, "") + File.chmod(path, 0o444) + pending_if_superuser! + File::Info.writable?(path).should be_false + end + end + + it "follows symlinks" do + with_tempfile("good_symlink_w.txt", "bad_symlink_w.txt", "readonly.txt") do |good_path, bad_path, readonly| + File.write(readonly, "") + File.chmod(readonly, 0o444) + pending_if_superuser! + + File.symlink(File.expand_path(datapath("test_file.txt")), good_path) + File.symlink(File.expand_path(readonly), bad_path) + + File::Info.writable?(good_path).should be_true + File::Info.writable?(bad_path).should be_false + end + end + + it "gives false when the symbolic link destination doesn't exist" do + with_tempfile("missing_symlink_w.txt") do |missing_path| + File.symlink(File.expand_path(datapath("non_existing_file.txt")), missing_path) + File::Info.writable?(missing_path).should be_false + end + end + end end describe "size" do @@ -1372,15 +1375,15 @@ describe "File" do end it_raises_on_null_byte "readable?" do - File.readable?("foo\0bar") + File::Info.readable?("foo\0bar") end it_raises_on_null_byte "writable?" do - File.writable?("foo\0bar") + File::Info.writable?("foo\0bar") end it_raises_on_null_byte "executable?" do - File.executable?("foo\0bar") + File::Info.executable?("foo\0bar") end it_raises_on_null_byte "file?" do diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 6bd04ab3e2f2..4cd51bf83075 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -357,7 +357,7 @@ module HTTP test_server("localhost", 0, 0.5.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSARecv timed out" {% else %} "Read timed out" {% end %}) do - client.read_timeout = 0.001 + client.read_timeout = 1.millisecond client.get("/?sleep=1") end end @@ -371,7 +371,7 @@ module HTTP test_server("localhost", 0, 0.seconds, write_response: false) do |server| client = Client.new("localhost", server.local_address.port) expect_raises(IO::TimeoutError, {% if flag?(:win32) %} "WSASend timed out" {% else %} "Write timed out" {% end %}) do - client.write_timeout = 0.001 + client.write_timeout = 1.millisecond client.post("/", body: "a" * 5_000_000) end end @@ -380,7 +380,7 @@ module HTTP it "tests connect_timeout" do test_server("localhost", 0, 0.seconds) do |server| client = Client.new("localhost", server.local_address.port) - client.connect_timeout = 0.5 + client.connect_timeout = 0.5.seconds client.get("/") end end diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index 3980084ea414..3c634d755abf 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -433,7 +433,7 @@ describe HTTP::Server do begin ch.receive client = HTTP::Client.new(address.address, address.port, client_context) - client.read_timeout = client.connect_timeout = 3 + client.read_timeout = client.connect_timeout = 3.seconds client.get("/").body.should eq "ok" ensure ch.send nil diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index c584ec81a1e8..3be5c07e1479 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -105,11 +105,11 @@ describe IO do write.puts "hello" slice = Bytes.new 1024 - read.read_timeout = 1 + read.read_timeout = 1.second read.read(slice).should eq(6) expect_raises(IO::TimeoutError) do - read.read_timeout = 0.0000001 + read.read_timeout = 0.1.microseconds read.read(slice) end end diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index 98555937dea3..f4ff7c90972b 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -79,7 +79,7 @@ describe Socket, tags: "network" do server = Socket.new(Socket::Family::INET, Socket::Type::STREAM, Socket::Protocol::TCP) port = unused_local_port server.bind("0.0.0.0", port) - server.read_timeout = 0.1 + server.read_timeout = 0.1.seconds server.listen expect_raises(IO::TimeoutError) { server.accept } diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index c51f37193c0e..b3bc4931ec78 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -82,8 +82,8 @@ describe UNIXSocket do it "tests read and write timeouts" do UNIXSocket.pair do |left, right| # BUG: shrink the socket buffers first - left.write_timeout = 0.0001 - right.read_timeout = 0.0001 + left.write_timeout = 0.1.milliseconds + right.read_timeout = 0.1.milliseconds buf = ("a" * IO::DEFAULT_BUFFER_SIZE).to_slice expect_raises(IO::TimeoutError, "Write timed out") do From b5f24681038628de57bf42114cb6eb5a6acfa5d2 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 10 Sep 2024 14:26:10 +0800 Subject: [PATCH 100/193] Fix exponent overflow in `BigFloat#to_s` for very large values (#14982) This is similar to #14971 where the base-10 exponent returned by `LibGMP.mpf_get_str` is not large enough to handle all values internally representable by GMP / MPIR. --- spec/std/big/big_float_spec.cr | 7 +++++ src/big/big_float.cr | 53 +++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 73c6bcf06de8..1b5e4e3893fc 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -345,6 +345,13 @@ describe "BigFloat" do it { assert_prints (0.1).to_big_f.to_s, "0.100000000000000005551" } it { assert_prints Float64::MAX.to_big_f.to_s, "1.79769313486231570815e+308" } it { assert_prints Float64::MIN_POSITIVE.to_big_f.to_s, "2.22507385850720138309e-308" } + + it { (2.to_big_f ** 7133786264).to_s.should end_with("e+2147483648") } # least power of two with a base-10 exponent greater than Int32::MAX + it { (2.to_big_f ** -7133786264).to_s.should end_with("e-2147483649") } # least power of two with a base-10 exponent less than Int32::MIN + it { (10.to_big_f ** 3000000000 * 1.5).to_s.should end_with("e+3000000000") } + it { (10.to_big_f ** -3000000000 * 1.5).to_s.should end_with("e-3000000000") } + it { (10.to_big_f ** 10000000000 * 1.5).to_s.should end_with("e+10000000000") } + it { (10.to_big_f ** -10000000000 * 1.5).to_s.should end_with("e-10000000000") } end describe "#inspect" do diff --git a/src/big/big_float.cr b/src/big/big_float.cr index ff78b7de8290..f6ab46def36d 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -362,10 +362,12 @@ struct BigFloat < Float end def to_s(io : IO) : Nil - cstr = LibGMP.mpf_get_str(nil, out decimal_exponent, 10, 0, self) + cstr = LibGMP.mpf_get_str(nil, out orig_decimal_exponent, 10, 0, self) length = LibC.strlen(cstr) buffer = Slice.new(cstr, length) + decimal_exponent = fix_exponent_overflow(orig_decimal_exponent) + # add negative sign if buffer[0]? == 45 # '-' io << '-' @@ -415,6 +417,55 @@ struct BigFloat < Float end end + # The same `LibGMP::MpExp` is used in `LibGMP::MPF` to represent a + # `BigFloat`'s exponent in base `256 ** sizeof(LibGMP::MpLimb)`, and to return + # a base-10 exponent in `LibGMP.mpf_get_str`. The latter is around 9.6x the + # former when `MpLimb` is 32-bit, or around 19.3x when `MpLimb` is 64-bit. + # This means the base-10 exponent will overflow for the majority of `MpExp`'s + # domain, even though `BigFloat`s will work correctly in this exponent range + # otherwise. This method exists to recover the original exponent for `#to_s`. + # + # Note that if `MpExp` is 64-bit, which is the case for non-Windows 64-bit + # targets, then `mpf_get_str` will simply crash for values above + # `2 ** 0x1_0000_0000_0000_0080`; here `exponent10` is around 5.553e+18, and + # never overflows. Thus there is no need to check for overflow in that case. + private def fix_exponent_overflow(exponent10) + {% if LibGMP::MpExp == Int64 %} + exponent10 + {% else %} + # When `self` is non-zero, + # + # @mpf.@_mp_exp == Math.log(abs, 256.0 ** sizeof(LibGMP::MpLimb)).floor + 1 + # @mpf.@_mp_exp - 1 <= Math.log(abs, 256.0 ** sizeof(LibGMP::MpLimb)) < @mpf.@_mp_exp + # @mpf.@_mp_exp - 1 <= Math.log10(abs) / Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) < @mpf.@_mp_exp + # Math.log10(abs) >= (@mpf.@_mp_exp - 1) * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) + # Math.log10(abs) < @mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) + # + # And also, + # + # exponent10 == Math.log10(abs).floor + 1 + # exponent10 - 1 <= Math.log10(abs) < exponent10 + # + # When `exponent10` overflows, it differs from its real value by an + # integer multiple of `256.0 ** sizeof(LibGMP::MpExp)`. We have to recover + # the integer `overflow_n` such that: + # + # LibGMP::MpExp::MIN <= exponent10 <= LibGMP::MpExp::MAX + # Math.log10(abs) ~= exponent10 + overflow_n * 256.0 ** sizeof(LibGMP::MpExp) + # ~= @mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) + # + # Because the possible intervals for the real `exponent10` are so far apart, + # it suffices to approximate `overflow_n` as follows: + # + # overflow_n ~= (@mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) - exponent10) / 256.0 ** sizeof(LibGMP::MpExp) + # + # This value will be very close to an integer, which we then obtain with + # `#round`. + overflow_n = ((@mpf.@_mp_exp * Math.log10(256.0 ** sizeof(LibGMP::MpLimb)) - exponent10) / 256.0 ** sizeof(LibGMP::MpExp)) + exponent10.to_i64 + overflow_n.round.to_i64 * (256_i64 ** sizeof(LibGMP::MpExp)) + {% end %} + end + def clone self end From ce76bf573f3dadba4c936bba54cc3a4b10d410bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 10 Sep 2024 08:26:24 +0200 Subject: [PATCH 101/193] Add type restriction to `String#byte_index` `offset` parameter (#14981) The return type is `Int32?`, so `offset` must be `Int32` as well. --- src/string.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/string.cr b/src/string.cr index 35c33b903939..f0dbd1a1eae3 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3715,7 +3715,7 @@ class String # "Dizzy Miss Lizzy".byte_index('z'.ord, -4) # => 13 # "Dizzy Miss Lizzy".byte_index('z'.ord, -17) # => nil # ``` - def byte_index(byte : Int, offset = 0) : Int32? + def byte_index(byte : Int, offset : Int32 = 0) : Int32? offset += bytesize if offset < 0 return if offset < 0 From fdbaeb4c3dec8e1174eb22310602119cab0f5d4f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 11 Sep 2024 05:54:01 +0800 Subject: [PATCH 102/193] Implement floating-point manipulation functions for `BigFloat` (#11007) `BigFloat` already has an implementation of `frexp` because `hash` needs it (I think); this PR adds the remaining floating-point manipulation operations. Methods that take an additional integer accept an `Int` directly, so this takes care of the allowed conversions in #10907. `BigFloat` from GMP doesn't have the notion of signed zeros, so `copysign` is only an approximation. (As usual, MPFR floats feature signed zeros.) --- spec/std/big/big_float_spec.cr | 54 ++++++++++++++++++++++++++++++++++ src/big/big_float.cr | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 1b5e4e3893fc..23c782aa3de8 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -554,6 +554,48 @@ describe "BigFloat" do end describe "BigFloat Math" do + it ".ilogb" do + Math.ilogb(0.2.to_big_f).should eq(-3) + Math.ilogb(123.45.to_big_f).should eq(6) + Math.ilogb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000) + Math.ilogb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000) + Math.ilogb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000) + expect_raises(ArgumentError) { Math.ilogb(0.to_big_f) } + end + + it ".logb" do + Math.logb(0.2.to_big_f).should eq(-3.to_big_f) + Math.logb(123.45.to_big_f).should eq(6.to_big_f) + Math.logb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000.to_big_f) + Math.logb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000.to_big_f) + Math.logb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000.to_big_f) + expect_raises(ArgumentError) { Math.logb(0.to_big_f) } + end + + it ".ldexp" do + Math.ldexp(0.2.to_big_f, 2).should eq(0.8.to_big_f) + Math.ldexp(0.2.to_big_f, -2).should eq(0.05.to_big_f) + Math.ldexp(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) + Math.ldexp(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.ldexp(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + end + + it ".scalbn" do + Math.scalbn(0.2.to_big_f, 2).should eq(0.8.to_big_f) + Math.scalbn(0.2.to_big_f, -2).should eq(0.05.to_big_f) + Math.scalbn(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) + Math.scalbn(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbn(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + end + + it ".scalbln" do + Math.scalbln(0.2.to_big_f, 2).should eq(0.8.to_big_f) + Math.scalbln(0.2.to_big_f, -2).should eq(0.05.to_big_f) + Math.scalbln(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) + Math.scalbln(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbln(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + end + it ".frexp" do Math.frexp(0.to_big_f).should eq({0.0, 0}) Math.frexp(1.to_big_f).should eq({0.5, 1}) @@ -574,6 +616,18 @@ describe "BigFloat Math" do Math.frexp(-(2.to_big_f ** -0x100000000)).should eq({-0.5, -0xFFFFFFFF}) end + it ".copysign" do + Math.copysign(3.to_big_f, 2.to_big_f).should eq(3.to_big_f) + Math.copysign(3.to_big_f, 0.to_big_f).should eq(3.to_big_f) + Math.copysign(3.to_big_f, -2.to_big_f).should eq(-3.to_big_f) + Math.copysign(0.to_big_f, 2.to_big_f).should eq(0.to_big_f) + Math.copysign(0.to_big_f, 0.to_big_f).should eq(0.to_big_f) + Math.copysign(0.to_big_f, -2.to_big_f).should eq(0.to_big_f) + Math.copysign(-3.to_big_f, 2.to_big_f).should eq(3.to_big_f) + Math.copysign(-3.to_big_f, 0.to_big_f).should eq(3.to_big_f) + Math.copysign(-3.to_big_f, -2.to_big_f).should eq(-3.to_big_f) + end + it ".sqrt" do Math.sqrt(BigFloat.new("1" + "0"*48)).should eq(BigFloat.new("1" + "0"*24)) end diff --git a/src/big/big_float.cr b/src/big/big_float.cr index f6ab46def36d..5a57500fbdd7 100644 --- a/src/big/big_float.cr +++ b/src/big/big_float.cr @@ -586,6 +586,47 @@ class String end module Math + # Returns the unbiased base 2 exponent of the given floating-point *value*. + # + # Raises `ArgumentError` if *value* is zero. + def ilogb(value : BigFloat) : Int64 + raise ArgumentError.new "Cannot get exponent of zero" if value.zero? + leading_zeros = value.@mpf._mp_d[value.@mpf._mp_size.abs - 1].leading_zeros_count + 8_i64 * sizeof(LibGMP::MpLimb) * value.@mpf._mp_exp - leading_zeros - 1 + end + + # Returns the unbiased radix-independent exponent of the given floating-point *value*. + # + # For `BigFloat` this is equivalent to `ilogb`. + # + # Raises `ArgumentError` is *value* is zero. + def logb(value : BigFloat) : BigFloat + ilogb(value).to_big_f + end + + # Multiplies the given floating-point *value* by 2 raised to the power *exp*. + def ldexp(value : BigFloat, exp : Int) : BigFloat + BigFloat.new do |mpf| + if exp >= 0 + LibGMP.mpf_mul_2exp(mpf, value, exp.to_u64) + else + LibGMP.mpf_div_2exp(mpf, value, exp.abs.to_u64) + end + end + end + + # Returns the floating-point *value* with its exponent raised by *exp*. + # + # For `BigFloat` this is equivalent to `ldexp`. + def scalbn(value : BigFloat, exp : Int) : BigFloat + ldexp(value, exp) + end + + # :ditto: + def scalbln(value : BigFloat, exp : Int) : BigFloat + ldexp(value, exp) + end + # Decomposes the given floating-point *value* into a normalized fraction and an integral power of two. def frexp(value : BigFloat) : {BigFloat, Int64} return {BigFloat.zero, 0_i64} if value.zero? @@ -608,6 +649,17 @@ module Math {frac, exp} end + # Returns the floating-point value with the magnitude of *value1* and the sign of *value2*. + # + # `BigFloat` does not support signed zeros; if `value2 == 0`, the returned value is non-negative. + def copysign(value1 : BigFloat, value2 : BigFloat) : BigFloat + if value1.negative? != value2.negative? # opposite signs + -value1 + else + value1 + end + end + # Calculates the square root of *value*. # # ``` From 8b1a3916fb702c8f55895139d2984d256288f13a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 11 Sep 2024 05:54:21 +0800 Subject: [PATCH 103/193] Remove TODO in `Crystal::Loader` on Windows (#14988) This has been addressed by #14146 from outside the loader, and there is essentially only one TODO left for #11575. --- src/compiler/crystal/loader/msvc.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/crystal/loader/msvc.cr b/src/compiler/crystal/loader/msvc.cr index 772e4c5c232f..966f6ec5d246 100644 --- a/src/compiler/crystal/loader/msvc.cr +++ b/src/compiler/crystal/loader/msvc.cr @@ -185,7 +185,6 @@ class Crystal::Loader end private def open_library(path : String) - # TODO: respect `@[Link(dll:)]`'s search order LibC.LoadLibraryExW(System.to_wstr(path), nil, 0) end From dea39ad408d06f8c3d9017e1769c8726e67c012e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 13 Sep 2024 18:38:41 +0800 Subject: [PATCH 104/193] Async DNS resolution on Windows (#14979) Uses `GetAddrInfoExW` with IOCP --- spec/std/socket/addrinfo_spec.cr | 32 +++++++++-- src/crystal/system/addrinfo.cr | 6 +- src/crystal/system/win32/addrinfo.cr | 43 ++++++++++++--- src/crystal/system/win32/addrinfo_win7.cr | 61 +++++++++++++++++++++ src/crystal/system/win32/iocp.cr | 36 ++++++++++++ src/http/client.cr | 8 +-- src/lib_c/x86_64-windows-msvc/c/winsock2.cr | 7 +++ src/lib_c/x86_64-windows-msvc/c/ws2def.cr | 14 +++++ src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr | 20 +++++++ src/socket/addrinfo.cr | 30 ++++++---- src/socket/tcp_socket.cr | 2 +- src/winerror.cr | 5 +- 12 files changed, 233 insertions(+), 31 deletions(-) create mode 100644 src/crystal/system/win32/addrinfo_win7.cr diff --git a/spec/std/socket/addrinfo_spec.cr b/spec/std/socket/addrinfo_spec.cr index 615058472525..109eb383562b 100644 --- a/spec/std/socket/addrinfo_spec.cr +++ b/spec/std/socket/addrinfo_spec.cr @@ -22,6 +22,20 @@ describe Socket::Addrinfo, tags: "network" do end end end + + it "raises helpful message on getaddrinfo failure" do + expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname failed: ") do + Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::DGRAM) + end + end + + {% if flag?(:win32) %} + it "raises timeout error" do + expect_raises(IO::TimeoutError) do + Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::STREAM, timeout: 0.milliseconds) + end + end + {% end %} end describe ".tcp" do @@ -37,11 +51,13 @@ describe Socket::Addrinfo, tags: "network" do end end - it "raises helpful message on getaddrinfo failure" do - expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname failed: ") do - Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::DGRAM) + {% if flag?(:win32) %} + it "raises timeout error" do + expect_raises(IO::TimeoutError) do + Socket::Addrinfo.tcp("badhostname", 80, timeout: 0.milliseconds) + end end - end + {% end %} end describe ".udp" do @@ -56,6 +72,14 @@ describe Socket::Addrinfo, tags: "network" do typeof(addrinfo).should eq(Socket::Addrinfo) end end + + {% if flag?(:win32) %} + it "raises timeout error" do + expect_raises(IO::TimeoutError) do + Socket::Addrinfo.udp("badhostname", 80, timeout: 0.milliseconds) + end + end + {% end %} end describe "#ip_address" do diff --git a/src/crystal/system/addrinfo.cr b/src/crystal/system/addrinfo.cr index 23513e6f763e..ff9166f3aca1 100644 --- a/src/crystal/system/addrinfo.cr +++ b/src/crystal/system/addrinfo.cr @@ -30,7 +30,11 @@ end {% elsif flag?(:unix) %} require "./unix/addrinfo" {% elsif flag?(:win32) %} - require "./win32/addrinfo" + {% if flag?(:win7) %} + require "./win32/addrinfo_win7" + {% else %} + require "./win32/addrinfo" + {% end %} {% else %} {% raise "No Crystal::System::Addrinfo implementation available" %} {% end %} diff --git a/src/crystal/system/win32/addrinfo.cr b/src/crystal/system/win32/addrinfo.cr index b033d61f16e7..91ebb1620a43 100644 --- a/src/crystal/system/win32/addrinfo.cr +++ b/src/crystal/system/win32/addrinfo.cr @@ -1,5 +1,5 @@ module Crystal::System::Addrinfo - alias Handle = LibC::Addrinfo* + alias Handle = LibC::ADDRINFOEXW* @addr : LibC::SockaddrIn6 @@ -30,7 +30,7 @@ module Crystal::System::Addrinfo end def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle - hints = LibC::Addrinfo.new + hints = LibC::ADDRINFOEXW.new hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 hints.ai_socktype = type hints.ai_protocol = protocol @@ -43,12 +43,39 @@ module Crystal::System::Addrinfo end end - ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) - unless ret.zero? - error = WinError.new(ret.to_u32!) - raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + Crystal::IOCP::GetAddrInfoOverlappedOperation.run(Crystal::EventLoop.current.iocp) do |operation| + completion_routine = LibC::LPLOOKUPSERVICE_COMPLETION_ROUTINE.new do |dwError, dwBytes, lpOverlapped| + orig_operation = Crystal::IOCP::GetAddrInfoOverlappedOperation.unbox(lpOverlapped) + LibC.PostQueuedCompletionStatus(orig_operation.iocp, 0, 0, lpOverlapped) + end + + # NOTE: we handle the timeout ourselves so we don't pass a `LibC::Timeval` + # to Win32 here + result = LibC.GetAddrInfoExW( + Crystal::System.to_wstr(domain), Crystal::System.to_wstr(service.to_s), LibC::NS_DNS, nil, pointerof(hints), + out addrinfos, nil, operation, completion_routine, out cancel_handle) + + if result == 0 + return addrinfos + else + case error = WinError.new(result.to_u32!) + when .wsa_io_pending? + # used in `Crystal::IOCP::OverlappedOperation#try_cancel_getaddrinfo` + operation.cancel_handle = cancel_handle + else + raise ::Socket::Addrinfo::Error.from_os_error("GetAddrInfoExW", error, domain: domain, type: type, protocol: protocol, service: service) + end + end + + operation.wait_for_result(timeout) do |error| + case error + when .wsa_e_cancelled? + raise IO::TimeoutError.new("GetAddrInfoExW timed out") + else + raise ::Socket::Addrinfo::Error.from_os_error("GetAddrInfoExW", error, domain: domain, type: type, protocol: protocol, service: service) + end + end end - ptr end def self.next_addrinfo(addrinfo : Handle) : Handle @@ -56,6 +83,6 @@ module Crystal::System::Addrinfo end def self.free_addrinfo(addrinfo : Handle) - LibC.freeaddrinfo(addrinfo) + LibC.FreeAddrInfoExW(addrinfo) end end diff --git a/src/crystal/system/win32/addrinfo_win7.cr b/src/crystal/system/win32/addrinfo_win7.cr new file mode 100644 index 000000000000..b033d61f16e7 --- /dev/null +++ b/src/crystal/system/win32/addrinfo_win7.cr @@ -0,0 +1,61 @@ +module Crystal::System::Addrinfo + alias Handle = LibC::Addrinfo* + + @addr : LibC::SockaddrIn6 + + protected def initialize(addrinfo : Handle) + @family = ::Socket::Family.from_value(addrinfo.value.ai_family) + @type = ::Socket::Type.from_value(addrinfo.value.ai_socktype) + @protocol = ::Socket::Protocol.from_value(addrinfo.value.ai_protocol) + @size = addrinfo.value.ai_addrlen.to_i + + @addr = uninitialized LibC::SockaddrIn6 + + case @family + when ::Socket::Family::INET6 + addrinfo.value.ai_addr.as(LibC::SockaddrIn6*).copy_to(pointerof(@addr).as(LibC::SockaddrIn6*), 1) + when ::Socket::Family::INET + addrinfo.value.ai_addr.as(LibC::SockaddrIn*).copy_to(pointerof(@addr).as(LibC::SockaddrIn*), 1) + else + # TODO: (asterite) UNSPEC and UNIX unsupported? + end + end + + def system_ip_address : ::Socket::IPAddress + ::Socket::IPAddress.from(to_unsafe, size) + end + + def to_unsafe + pointerof(@addr).as(LibC::Sockaddr*) + end + + def self.getaddrinfo(domain, service, family, type, protocol, timeout) : Handle + hints = LibC::Addrinfo.new + hints.ai_family = (family || ::Socket::Family::UNSPEC).to_i32 + hints.ai_socktype = type + hints.ai_protocol = protocol + hints.ai_flags = 0 + + if service.is_a?(Int) + hints.ai_flags |= LibC::AI_NUMERICSERV + if service < 0 + raise ::Socket::Addrinfo::Error.from_os_error(nil, WinError::WSATYPE_NOT_FOUND, domain: domain, type: type, protocol: protocol, service: service) + end + end + + ret = LibC.getaddrinfo(domain, service.to_s, pointerof(hints), out ptr) + unless ret.zero? + error = WinError.new(ret.to_u32!) + raise ::Socket::Addrinfo::Error.from_os_error(nil, error, domain: domain, type: type, protocol: protocol, service: service) + end + ptr + end + + def self.next_addrinfo(addrinfo : Handle) : Handle + addrinfo.value.ai_next + end + + def self.free_addrinfo(addrinfo : Handle) + LibC.freeaddrinfo(addrinfo) + end +end diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index 384784a193db..19c92c8f8725 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -210,6 +210,42 @@ module Crystal::IOCP end end + class GetAddrInfoOverlappedOperation < OverlappedOperation + getter iocp + setter cancel_handle : LibC::HANDLE = LibC::INVALID_HANDLE_VALUE + + def initialize(@iocp : LibC::HANDLE) + end + + def wait_for_result(timeout, & : WinError ->) + wait_for_completion(timeout) + + result = LibC.GetAddrInfoExOverlappedResult(self) + unless result.zero? + error = WinError.new(result.to_u32!) + yield error + + raise Socket::Addrinfo::Error.from_os_error("GetAddrInfoExOverlappedResult", error) + end + + @overlapped.union.pointer.as(LibC::ADDRINFOEXW**).value + end + + private def try_cancel : Bool + ret = LibC.GetAddrInfoExCancel(pointerof(@cancel_handle)) + unless ret.zero? + case error = WinError.new(ret.to_u32!) + when .wsa_invalid_handle? + # Operation has already completed, do nothing + return false + else + raise Socket::Addrinfo::Error.from_os_error("GetAddrInfoExCancel", error) + end + end + true + end + end + def self.overlapped_operation(file_descriptor, method, timeout, *, offset = nil, writing = false, &) handle = file_descriptor.windows_handle seekable = LibC.SetFilePointerEx(handle, 0, out original_offset, IO::Seek::Current) != 0 diff --git a/src/http/client.cr b/src/http/client.cr index b641065ac930..7324bdf7d639 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -343,10 +343,10 @@ class HTTP::Client # ``` setter connect_timeout : Time::Span? - # **This method has no effect right now** - # # Sets the number of seconds to wait when resolving a name, before raising an `IO::TimeoutError`. # + # NOTE: *dns_timeout* is currently only supported on Windows. + # # ``` # require "http/client" # @@ -363,10 +363,10 @@ class HTTP::Client self.dns_timeout = dns_timeout.seconds end - # **This method has no effect right now** - # # Sets the number of seconds to wait when resolving a name with a `Time::Span`, before raising an `IO::TimeoutError`. # + # NOTE: *dns_timeout* is currently only supported on Windows. + # # ``` # require "http/client" # diff --git a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr index 223c2366b072..68ce6f9ef421 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr @@ -20,6 +20,8 @@ lib LibC lpVendorInfo : Char* end + NS_DNS = 12_u32 + INVALID_SOCKET = ~SOCKET.new(0) SOCKET_ERROR = -1 @@ -111,6 +113,11 @@ lib LibC alias WSAOVERLAPPED_COMPLETION_ROUTINE = Proc(DWORD, DWORD, WSAOVERLAPPED*, DWORD, Void) + struct Timeval + tv_sec : Long + tv_usec : Long + end + struct Linger l_onoff : UShort l_linger : UShort diff --git a/src/lib_c/x86_64-windows-msvc/c/ws2def.cr b/src/lib_c/x86_64-windows-msvc/c/ws2def.cr index 9fc19857f4a3..41e0a1a408eb 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ws2def.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ws2def.cr @@ -208,4 +208,18 @@ lib LibC ai_addr : Sockaddr* ai_next : Addrinfo* end + + struct ADDRINFOEXW + ai_flags : Int + ai_family : Int + ai_socktype : Int + ai_protocol : Int + ai_addrlen : SizeT + ai_canonname : LPWSTR + ai_addr : Sockaddr* + ai_blob : Void* + ai_bloblen : SizeT + ai_provider : GUID* + ai_next : ADDRINFOEXW* + end end diff --git a/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr b/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr index 338063ccf6f6..3b3f61ba7fdb 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ws2tcpip.cr @@ -17,4 +17,24 @@ lib LibC fun getaddrinfo(pNodeName : Char*, pServiceName : Char*, pHints : Addrinfo*, ppResult : Addrinfo**) : Int fun inet_ntop(family : Int, pAddr : Void*, pStringBuf : Char*, stringBufSize : SizeT) : Char* fun inet_pton(family : Int, pszAddrString : Char*, pAddrBuf : Void*) : Int + + fun FreeAddrInfoExW(pAddrInfoEx : ADDRINFOEXW*) + + alias LPLOOKUPSERVICE_COMPLETION_ROUTINE = DWORD, DWORD, WSAOVERLAPPED* -> + + fun GetAddrInfoExW( + pName : LPWSTR, + pServiceName : LPWSTR, + dwNameSpace : DWORD, + lpNspId : GUID*, + hints : ADDRINFOEXW*, + ppResult : ADDRINFOEXW**, + timeout : Timeval*, + lpOverlapped : OVERLAPPED*, + lpCompletionRoutine : LPLOOKUPSERVICE_COMPLETION_ROUTINE, + lpHandle : HANDLE*, + ) : Int + + fun GetAddrInfoExOverlappedResult(lpOverlapped : OVERLAPPED*) : Int + fun GetAddrInfoExCancel(lpHandle : HANDLE*) : Int end diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index cdf55c912601..ef76d0e285b6 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -23,6 +23,9 @@ class Socket # specified. # - *protocol* is the intended socket protocol (e.g. `Protocol::TCP`) and # should be specified. + # - *timeout* is optional and specifies the maximum time to wait before + # `IO::TimeoutError` is raised. Currently this is only supported on + # Windows. # # Example: # ``` @@ -107,8 +110,11 @@ class Socket "Hostname lookup for #{domain} failed" end - def self.os_error_message(os_error : Errno, *, type, service, protocol, **opts) - case os_error.value + def self.os_error_message(os_error : Errno | WinError, *, type, service, protocol, **opts) + # when `EAI_NONAME` etc. is an integer then only `os_error.value` can + # match; when `EAI_NONAME` is a `WinError` then `os_error` itself can + # match + case os_error.is_a?(Errno) ? os_error.value : os_error when LibC::EAI_NONAME "No address found" when LibC::EAI_SOCKTYPE @@ -116,12 +122,14 @@ class Socket when LibC::EAI_SERVICE "The requested service #{service} is not available for the requested socket type #{type}" else - {% unless flag?(:win32) %} - # There's no need for a special win32 branch because the os_error on Windows - # is of type WinError, which wouldn't match this overload anyways. - - String.new(LibC.gai_strerror(os_error.value)) + # Win32 also has this method, but `WinError` is already sufficient + {% if LibC.has_method?(:gai_strerror) %} + if os_error.is_a?(Errno) + return String.new(LibC.gai_strerror(os_error)) + end {% end %} + + super end end end @@ -148,13 +156,13 @@ class Socket # addrinfos = Socket::Addrinfo.tcp("example.org", 80) # ``` def self.tcp(domain : String, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) - resolve(domain, service, family, Type::STREAM, Protocol::TCP) + resolve(domain, service, family, Type::STREAM, Protocol::TCP, timeout) end # Resolves a domain for the TCP protocol with STREAM type, and yields each # possible `Addrinfo`. See `#resolve` for details. def self.tcp(domain : String, service, family = Family::UNSPEC, timeout = nil, &) - resolve(domain, service, family, Type::STREAM, Protocol::TCP) { |addrinfo| yield addrinfo } + resolve(domain, service, family, Type::STREAM, Protocol::TCP, timeout) { |addrinfo| yield addrinfo } end # Resolves *domain* for the UDP protocol and returns an `Array` of possible @@ -167,13 +175,13 @@ class Socket # addrinfos = Socket::Addrinfo.udp("example.org", 53) # ``` def self.udp(domain : String, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) - resolve(domain, service, family, Type::DGRAM, Protocol::UDP) + resolve(domain, service, family, Type::DGRAM, Protocol::UDP, timeout) end # Resolves a domain for the UDP protocol with DGRAM type, and yields each # possible `Addrinfo`. See `#resolve` for details. def self.udp(domain : String, service, family = Family::UNSPEC, timeout = nil, &) - resolve(domain, service, family, Type::DGRAM, Protocol::UDP) { |addrinfo| yield addrinfo } + resolve(domain, service, family, Type::DGRAM, Protocol::UDP, timeout) { |addrinfo| yield addrinfo } end # Returns an `IPAddress` matching this addrinfo. diff --git a/src/socket/tcp_socket.cr b/src/socket/tcp_socket.cr index 387417211a1a..4edcb3d08e5f 100644 --- a/src/socket/tcp_socket.cr +++ b/src/socket/tcp_socket.cr @@ -25,7 +25,7 @@ class TCPSocket < IPSocket # connection time to the remote server with `connect_timeout`. Both values # must be in seconds (integers or floats). # - # Note that `dns_timeout` is currently ignored. + # NOTE: *dns_timeout* is currently only supported on Windows. def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false) Addrinfo.tcp(host, port, timeout: dns_timeout) do |addrinfo| super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) diff --git a/src/winerror.cr b/src/winerror.cr index ab978769d553..fbb2fb553873 100644 --- a/src/winerror.cr +++ b/src/winerror.cr @@ -2305,6 +2305,7 @@ enum WinError : UInt32 ERROR_STATE_CONTAINER_NAME_SIZE_LIMIT_EXCEEDED = 15818_u32 ERROR_API_UNAVAILABLE = 15841_u32 - WSA_IO_PENDING = ERROR_IO_PENDING - WSA_IO_INCOMPLETE = ERROR_IO_INCOMPLETE + WSA_IO_PENDING = ERROR_IO_PENDING + WSA_IO_INCOMPLETE = ERROR_IO_INCOMPLETE + WSA_INVALID_HANDLE = ERROR_INVALID_HANDLE end From 833d90fc7fc0e8dd1f3ccc4b3084c9c8f0849922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 13 Sep 2024 12:39:03 +0200 Subject: [PATCH 105/193] Add error handling for linker flag sub commands (#14932) Co-authored-by: Quinton Miller --- src/compiler/crystal/compiler.cr | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 38880ee9ed64..f25713c6385e 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -411,7 +411,24 @@ module Crystal if program.has_flag? "msvc" lib_flags = program.lib_flags # Execute and expand `subcommands`. - lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}` } if expand + if expand + lib_flags = lib_flags.gsub(/`(.*?)`/) do + command = $1 + begin + error_io = IO::Memory.new + output = Process.run(command, shell: true, output: :pipe, error: error_io) do |process| + process.output.gets_to_end + end + unless $?.success? + error_io.rewind + error "Error executing subcommand for linker flags: #{command.inspect}: #{error_io}" + end + output + rescue exc + error "Error executing subcommand for linker flags: #{command.inspect}: #{exc}" + end + end + end object_arg = Process.quote_windows(object_names) output_arg = Process.quote_windows("/Fe#{output_filename}") From 5125f6619f7c5ec0f9825463a87830723ccd5269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustavo=20Gir=C3=A1ldez?= Date: Mon, 16 Sep 2024 08:12:48 -0300 Subject: [PATCH 106/193] Avoid unwinding the stack on hot path in method call lookups (#15002) The method lookup code uses exceptions to retry lookups using auto-casting. This is effectively using an exception for execution control, which is not what they are intended for. On `raise`, Crystal will try to unwind the call stack and save it to be able to report the original place where the exception was thrown, and this is a very expensive operation. To avoid that, we initialize the callstack of the special `RetryLookupWithLiterals` exception class always with the same fixed value. --- src/compiler/crystal/semantic/call.cr | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/compiler/crystal/semantic/call.cr b/src/compiler/crystal/semantic/call.cr index f581ea79d577..e265829a919e 100644 --- a/src/compiler/crystal/semantic/call.cr +++ b/src/compiler/crystal/semantic/call.cr @@ -13,6 +13,11 @@ class Crystal::Call property? uses_with_scope = false class RetryLookupWithLiterals < ::Exception + @@dummy_call_stack = Exception::CallStack.new + + def initialize + self.callstack = @@dummy_call_stack + end end def program From 849f71e2aa97453a45229df346c9127ecddaf5dc Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 16 Sep 2024 19:39:12 +0800 Subject: [PATCH 107/193] Drop the non-release Windows compiler artifact (#15000) Since #14964 we're building the compiler in release mode on every workflow run. We can use that build instead of the non-release build for workflow jobs that need a compiler. --- .github/workflows/win.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 568828b17bee..a256a6806a3f 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -214,16 +214,16 @@ jobs: if: steps.cache-llvm-dlls.outputs.cache-hit != 'true' run: .\etc\win-ci\build-llvm.ps1 -BuildTree deps\llvm -Version ${{ env.CI_LLVM_VERSION }} -TargetsToBuild X86,AArch64 -Dynamic - x86_64-windows: + x86_64-windows-release: needs: [x86_64-windows-libs, x86_64-windows-dlls, x86_64-windows-llvm-libs, x86_64-windows-llvm-dlls] uses: ./.github/workflows/win_build_portable.yml with: - release: false + release: true llvm_version: "18.1.1" x86_64-windows-test: runs-on: windows-2022 - needs: [x86_64-windows] + needs: [x86_64-windows-release] steps: - name: Disable CRLF line ending substitution run: | @@ -238,7 +238,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal + name: crystal-release path: build - name: Restore LLVM @@ -266,13 +266,6 @@ jobs: - name: Build samples run: make -f Makefile.win samples - x86_64-windows-release: - needs: [x86_64-windows-libs, x86_64-windows-dlls, x86_64-windows-llvm-libs, x86_64-windows-llvm-dlls] - uses: ./.github/workflows/win_build_portable.yml - with: - release: true - llvm_version: "18.1.1" - x86_64-windows-test-interpreter: runs-on: windows-2022 needs: [x86_64-windows-release] From 9134f9f3ba27b099ebbf048b20560c473cdeb87f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 16 Sep 2024 19:56:21 +0800 Subject: [PATCH 108/193] Fix `Process.exec` stream redirection on Windows (#14986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following should print the compiler help message to the file `foo.txt`: ```crystal File.open("foo.txt", "w") do |f| Process.exec("crystal", output: f) end ``` It used to work on Windows in Crystal 1.12, but is now broken since 1.13. This is because `LibC._wexecvp` only inherits file descriptors in the C runtime, not arbitrary Win32 file handles; since we stopped calling `LibC._open_osfhandle`, the C runtime knows nothing about any reopened standard streams in Win32. Thus the above merely prints the help message to the standard output. This PR creates the missing C file descriptors right before `LibC._wexecvp`. It also fixes a different regression of #14947 where reconfiguring `STDIN.blocking` always fails. Co-authored-by: Johannes Müller --- spec/std/process_spec.cr | 21 ++++++++++ src/crystal/system/win32/process.cr | 59 +++++++++++++++++++-------- src/lib_c/x86_64-windows-msvc/c/io.cr | 5 ++- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index d41ee0bed242..01a154ccb010 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -484,6 +484,27 @@ describe Process do {% end %} describe ".exec" do + it "redirects STDIN and STDOUT to files", tags: %w[slow] do + with_tempfile("crystal-exec-stdin", "crystal-exec-stdout") do |stdin_path, stdout_path| + File.write(stdin_path, "foobar") + + status, _, _ = compile_and_run_source <<-CRYSTAL + command = #{stdin_to_stdout_command[0].inspect} + args = #{stdin_to_stdout_command[1].to_a} of String + stdin_path = #{stdin_path.inspect} + stdout_path = #{stdout_path.inspect} + File.open(stdin_path) do |input| + File.open(stdout_path, "w") do |output| + Process.exec(command, args, input: input, output: output) + end + end + CRYSTAL + + status.success?.should be_true + File.read(stdout_path).chomp.should eq("foobar") + end + end + it "gets error from exec" do expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do Process.exec("foobarbaz") diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 2c6d81720636..7031654d2299 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -326,9 +326,9 @@ struct Crystal::System::Process end private def self.try_replace(command_args, env, clear_env, input, output, error, chdir) - reopen_io(input, ORIGINAL_STDIN) - reopen_io(output, ORIGINAL_STDOUT) - reopen_io(error, ORIGINAL_STDERR) + old_input_fd = reopen_io(input, ORIGINAL_STDIN) + old_output_fd = reopen_io(output, ORIGINAL_STDOUT) + old_error_fd = reopen_io(error, ORIGINAL_STDERR) ENV.clear if clear_env env.try &.each do |key, val| @@ -351,11 +351,18 @@ struct Crystal::System::Process argv << Pointer(LibC::WCHAR).null LibC._wexecvp(command, argv) + + # exec failed; restore the original C runtime file descriptors + errno = Errno.value + LibC._dup2(old_input_fd, 0) + LibC._dup2(old_output_fd, 1) + LibC._dup2(old_error_fd, 2) + errno end def self.replace(command_args, env, clear_env, input, output, error, chdir) : NoReturn - try_replace(command_args, env, clear_env, input, output, error, chdir) - raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0]) + errno = try_replace(command_args, env, clear_env, input, output, error, chdir) + raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0], errno) end private def self.raise_exception_from_errno(command, errno = Errno.value) @@ -367,21 +374,41 @@ struct Crystal::System::Process end end + # Replaces the C standard streams' file descriptors, not Win32's, since + # `try_replace` uses the C `LibC._wexecvp` and only cares about the former. + # Returns a duplicate of the original file descriptor private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) - src_io = to_real_fd(src_io) + unless src_io.system_blocking? + raise IO::Error.new("Non-blocking streams are not supported in `Process.exec`", target: src_io) + end - dst_io.reopen(src_io) - dst_io.blocking = true - dst_io.close_on_exec = false - end + src_fd = + case src_io + when STDIN then 0 + when STDOUT then 1 + when STDERR then 2 + else + LibC._open_osfhandle(src_io.windows_handle, 0) + end - private def self.to_real_fd(fd : IO::FileDescriptor) - case fd - when STDIN then ORIGINAL_STDIN - when STDOUT then ORIGINAL_STDOUT - when STDERR then ORIGINAL_STDERR - else fd + dst_fd = + case dst_io + when ORIGINAL_STDIN then 0 + when ORIGINAL_STDOUT then 1 + when ORIGINAL_STDERR then 2 + else + raise "BUG: Invalid destination IO" + end + + return src_fd if dst_fd == src_fd + + orig_src_fd = LibC._dup(src_fd) + + if LibC._dup2(src_fd, dst_fd) == -1 + raise IO::Error.from_errno("Failed to replace C file descriptor", target: dst_io) end + + orig_src_fd end def self.chroot(path) diff --git a/src/lib_c/x86_64-windows-msvc/c/io.cr b/src/lib_c/x86_64-windows-msvc/c/io.cr index 75da8c18e5b9..ccbaa15f2d1b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/io.cr +++ b/src/lib_c/x86_64-windows-msvc/c/io.cr @@ -2,12 +2,13 @@ require "c/stdint" lib LibC fun _wexecvp(cmdname : WCHAR*, argv : WCHAR**) : IntPtrT + fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int + fun _dup(fd : Int) : Int + fun _dup2(fd1 : Int, fd2 : Int) : Int # unused - fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int fun _get_osfhandle(fd : Int) : IntPtrT fun _close(fd : Int) : Int - fun _dup2(fd1 : Int, fd2 : Int) : Int fun _isatty(fd : Int) : Int fun _write(fd : Int, buffer : UInt8*, count : UInt) : Int fun _read(fd : Int, buffer : UInt8*, count : UInt) : Int From 5c18900fb7fa65bfb6cdb4da1f3d763700022b8c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 17 Sep 2024 18:47:36 +0800 Subject: [PATCH 109/193] Use Cygwin to build libiconv on Windows CI (#14999) This uses the official releases and build instructions. To compile code with this patch using a Windows Crystal compiler without this patch, either the new library files (`lib\iconv.lib`, `lib\iconv-dynamic.lib`, `iconv-2.dll`) shall be copied to that existing Crystal installation, or `CRYSTAL_LIBRARY_PATH` shall include the new `lib` directory so that the `@[Link]` annotation will pick up the new `iconv-2.dll` on program builds. Otherwise, compiled programs will continue to look for the old `libiconv.dll`, and silently break if it is not in `%PATH%` (which is hopefully rare since most of the time Crystal itself is also in `%PATH%`). Cygwin's location is currently hardcoded to `C:\cygwin64`, the default installation location for 64-bit Cygwin. Cygwin itself doesn't have native ARM64 support, but cross-compilation should be possible by simply using the x64-to-ARM64 cross tools MSVC developer prompt on an ARM64 machine. --- .github/workflows/win.yml | 20 ++++++++-- .github/workflows/win_build_portable.yml | 7 +++- etc/win-ci/build-iconv.ps1 | 47 +++++------------------- etc/win-ci/cygwin-build-iconv.sh | 32 ++++++++++++++++ src/crystal/lib_iconv.cr | 2 +- 5 files changed, 66 insertions(+), 42 deletions(-) create mode 100644 etc/win-ci/cygwin-build-iconv.sh diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index a256a6806a3f..d4b9316ef1a2 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -21,6 +21,13 @@ jobs: - name: Enable Developer Command Prompt uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + - name: Set up Cygwin + uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + with: + packages: make + install-dir: C:\cygwin64 + add-to-path: false + - name: Download Crystal source uses: actions/checkout@v4 @@ -50,7 +57,7 @@ jobs: run: .\etc\win-ci\build-pcre2.ps1 -BuildTree deps\pcre2 -Version 10.43 - name: Build libiconv if: steps.cache-libs.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv + run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 - name: Build libffi if: steps.cache-libs.outputs.cache-hit != 'true' run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 @@ -93,6 +100,13 @@ jobs: - name: Enable Developer Command Prompt uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + - name: Set up Cygwin + uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # v4 + with: + packages: make + install-dir: C:\cygwin64 + add-to-path: false + - name: Download Crystal source uses: actions/checkout@v4 @@ -112,7 +126,7 @@ jobs: libs/xml2-dynamic.lib dlls/pcre.dll dlls/pcre2-8.dll - dlls/libiconv.dll + dlls/iconv-2.dll dlls/gc.dll dlls/libffi.dll dlls/zlib1.dll @@ -131,7 +145,7 @@ jobs: run: .\etc\win-ci\build-pcre2.ps1 -BuildTree deps\pcre2 -Version 10.43 -Dynamic - name: Build libiconv if: steps.cache-dlls.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Dynamic + run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 -Dynamic - name: Build libffi if: steps.cache-dlls.outputs.cache-hit != 'true' run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 -Dynamic diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 12ee17da9e68..98c428ee5bad 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -23,6 +23,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 + id: install-crystal with: crystal: "1.13.2" @@ -68,7 +69,7 @@ jobs: libs/xml2-dynamic.lib dlls/pcre.dll dlls/pcre2-8.dll - dlls/libiconv.dll + dlls/iconv-2.dll dlls/gc.dll dlls/libffi.dll dlls/zlib1.dll @@ -107,6 +108,10 @@ jobs: run: | echo "CRYSTAL_LIBRARY_PATH=$(pwd)\libs" >> ${env:GITHUB_ENV} echo "LLVM_CONFIG=$(pwd)\llvm\bin\llvm-config.exe" >> ${env:GITHUB_ENV} + # NOTE: the name of the libiconv DLL has changed, so we manually copy + # the new one to the existing Crystal installation; remove after + # updating the base compiler to 1.14 + cp dlls/iconv-2.dll ${{ steps.install-crystal.outputs.path }} - name: Build LLVM extensions run: make -f Makefile.win deps diff --git a/etc/win-ci/build-iconv.ps1 b/etc/win-ci/build-iconv.ps1 index 56d0417bd729..541066c6327f 100644 --- a/etc/win-ci/build-iconv.ps1 +++ b/etc/win-ci/build-iconv.ps1 @@ -1,47 +1,20 @@ param( [Parameter(Mandatory)] [string] $BuildTree, + [Parameter(Mandatory)] [string] $Version, [switch] $Dynamic ) . "$(Split-Path -Parent $MyInvocation.MyCommand.Path)\setup.ps1" [void](New-Item -Name (Split-Path -Parent $BuildTree) -ItemType Directory -Force) -Setup-Git -Path $BuildTree -Url https://github.com/pffang/libiconv-for-Windows.git -Ref 1353455a6c4e15c9db6865fd9c2bf7203b59c0ec # master@{2022-10-11} +Invoke-WebRequest "https://ftp.gnu.org/pub/gnu/libiconv/libiconv-${Version}.tar.gz" -OutFile libiconv.tar.gz +tar -xzf libiconv.tar.gz +mv libiconv-* $BuildTree +rm libiconv.tar.gz Run-InDirectory $BuildTree { - Replace-Text libiconv\include\iconv.h '__declspec (dllimport) ' '' - - echo ' - - $(MsbuildThisFileDirectory)\Override.props - - ' > 'Directory.Build.props' - - echo " - - false - - - - None - false - - - false - - - - - MultiThreadedDLL - - - " > 'Override.props' - - if ($Dynamic) { - MSBuild.exe /p:Platform=x64 /p:Configuration=Release libiconv.vcxproj - } else { - MSBuild.exe /p:Platform=x64 /p:Configuration=ReleaseStatic libiconv.vcxproj - } + $env:CHERE_INVOKING = 1 + & 'C:\cygwin64\bin\bash.exe' --login "$PSScriptRoot\cygwin-build-iconv.sh" "$Version" "$(if ($Dynamic) { 1 })" if (-not $?) { Write-Host "Error: Failed to build libiconv" -ForegroundColor Red Exit 1 @@ -49,8 +22,8 @@ Run-InDirectory $BuildTree { } if ($Dynamic) { - mv -Force $BuildTree\output\x64\Release\libiconv.lib libs\iconv-dynamic.lib - mv -Force $BuildTree\output\x64\Release\libiconv.dll dlls\ + mv -Force $BuildTree\iconv\lib\iconv.dll.lib libs\iconv-dynamic.lib + mv -Force $BuildTree\iconv\bin\iconv-2.dll dlls\ } else { - mv -Force $BuildTree\output\x64\ReleaseStatic\libiconvStatic.lib libs\iconv.lib + mv -Force $BuildTree\iconv\lib\iconv.lib libs\ } diff --git a/etc/win-ci/cygwin-build-iconv.sh b/etc/win-ci/cygwin-build-iconv.sh new file mode 100644 index 000000000000..a8507542e646 --- /dev/null +++ b/etc/win-ci/cygwin-build-iconv.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -eo pipefail + +Version=$1 +Dynamic=$2 + +export PATH="$(pwd)/build-aux:$PATH" +export CC="$(pwd)/build-aux/compile cl -nologo" +export CXX="$(pwd)/build-aux/compile cl -nologo" +export AR="$(pwd)/build-aux/ar-lib lib" +export LD="link" +export NM="dumpbin -symbols" +export STRIP=":" +export RANLIB=":" +if [ -n "$Dynamic" ]; then + export CFLAGS="-MD" + export CXXFLAGS="-MD" + enable_shared=yes + enable_static=no +else + export CFLAGS="-MT" + export CXXFLAGS="-MT" + enable_shared=no + enable_static=yes +fi +export CPPFLAGS="-D_WIN32_WINNT=_WIN32_WINNT_WIN7 -I$(pwd)/iconv/include" +export LDFLAGS="-L$(pwd)/iconv/lib" + +./configure --host=x86_64-w64-mingw32 --prefix="$(pwd)/iconv" --enable-shared="${enable_shared}" --enable-static="${enable_static}" +make +make install diff --git a/src/crystal/lib_iconv.cr b/src/crystal/lib_iconv.cr index 5f1506758454..07100ff9c1dc 100644 --- a/src/crystal/lib_iconv.cr +++ b/src/crystal/lib_iconv.cr @@ -6,7 +6,7 @@ require "c/stddef" @[Link("iconv")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} - @[Link(dll: "libiconv.dll")] + @[Link(dll: "iconv-2.dll")] {% end %} lib LibIconv type IconvT = Void* From f17b565cd5cd9bd96dbba85d1e94c7d2cacbf21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 17 Sep 2024 21:17:40 +0200 Subject: [PATCH 110/193] Enable runners from `runs-on.com` for Aarch64 CI (#15007) https://runs-on.com is a service that provisions machines on AWS on demand to run workflow jobs. It triggers when a job is tagged as `runs-on` and you can configure what kind of machine you like this to run on. The machine specification could likely use some fine tuning (we can use other instance types, and theoretically 8GB should be sufficient). But that'll be a follow-up. For now we know that this works. We expect this to be more price-efficient setup than renting a fixed server or a CI runner service. This patch also includes an update to the latest base image. The old arm base images were using Crystal 1.0 and some outdated libraries which caused problems on the new runners (#15005). The build images are based on the docker images from [84codes/crystal](https://hub.docker.com/r/84codes/crystal). --- .github/workflows/aarch64.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index da252904fa37..85a8af2c8b37 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -8,13 +8,13 @@ concurrency: jobs: aarch64-musl-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -38,12 +38,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: - args: make std_spec FLAGS=-Duse_pcre + args: make std_spec aarch64-musl-test-compiler: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -55,17 +55,17 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make primitives_spec compiler_spec FLAGS=-Dwithout_ffi aarch64-gnu-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -89,12 +89,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make std_spec aarch64-gnu-test-compiler: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -106,6 +106,6 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make primitives_spec compiler_spec From 80b2484c3fb95cae09bce60cde3930a0df826cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 17 Sep 2024 21:17:40 +0200 Subject: [PATCH 111/193] Enable runners from `runs-on.com` for Aarch64 CI (#15007) https://runs-on.com is a service that provisions machines on AWS on demand to run workflow jobs. It triggers when a job is tagged as `runs-on` and you can configure what kind of machine you like this to run on. The machine specification could likely use some fine tuning (we can use other instance types, and theoretically 8GB should be sufficient). But that'll be a follow-up. For now we know that this works. We expect this to be more price-efficient setup than renting a fixed server or a CI runner service. This patch also includes an update to the latest base image. The old arm base images were using Crystal 1.0 and some outdated libraries which caused problems on the new runners (#15005). The build images are based on the docker images from [84codes/crystal](https://hub.docker.com/r/84codes/crystal). --- .github/workflows/aarch64.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index da252904fa37..85a8af2c8b37 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -8,13 +8,13 @@ concurrency: jobs: aarch64-musl-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -38,12 +38,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: - args: make std_spec FLAGS=-Duse_pcre + args: make std_spec aarch64-musl-test-compiler: needs: aarch64-musl-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -55,17 +55,17 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-alpine-build + uses: docker://crystallang/crystal:1.13.2-alpine-84codes-build with: args: make primitives_spec compiler_spec FLAGS=-Dwithout_ffi aarch64-gnu-build: - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v4 - name: Build Crystal - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make crystal - name: Upload Crystal executable @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -89,12 +89,12 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run stdlib specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make std_spec aarch64-gnu-test-compiler: needs: aarch64-gnu-build - runs-on: [linux, ARM64] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -106,6 +106,6 @@ jobs: - name: Mark downloaded compiler as executable run: chmod +x .build/crystal - name: Run compiler specs - uses: docker://jhass/crystal:1.0.0-build + uses: docker://crystallang/crystal:1.13.2-ubuntu-84codes-build with: args: make primitives_spec compiler_spec From 47cd33b259ac46533f8898bdd70fbd15fa9ab306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 6 Sep 2024 08:19:38 +0200 Subject: [PATCH 112/193] Fix use global paths in macro bodies (#14965) Macros inject code into other scopes. Paths are resolved in the expanded scope and there can be namespace conflicts. This fixes non-global paths in macro bodies that expand into uncontrolled scopes where namespaces could clash. This is a fixup for #14282 (released in 1.12.0). --- src/crystal/pointer_linked_list.cr | 4 ++-- src/ecr/macros.cr | 2 +- src/intrinsics.cr | 34 +++++++++++++++--------------- src/json/serialization.cr | 6 +++--- src/number.cr | 8 +++---- src/object.cr | 12 +++++------ src/slice.cr | 6 +++--- src/spec/dsl.cr | 4 ++-- src/spec/helpers/iterate.cr | 8 +++---- src/static_array.cr | 2 +- src/syscall/aarch64-linux.cr | 2 +- src/syscall/arm-linux.cr | 2 +- src/syscall/i386-linux.cr | 2 +- src/syscall/x86_64-linux.cr | 2 +- src/yaml/serialization.cr | 14 ++++++------ 15 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/crystal/pointer_linked_list.cr b/src/crystal/pointer_linked_list.cr index 03109979d662..cde9b0b79ddc 100644 --- a/src/crystal/pointer_linked_list.cr +++ b/src/crystal/pointer_linked_list.cr @@ -7,8 +7,8 @@ struct Crystal::PointerLinkedList(T) module Node macro included - property previous : Pointer(self) = Pointer(self).null - property next : Pointer(self) = Pointer(self).null + property previous : ::Pointer(self) = ::Pointer(self).null + property next : ::Pointer(self) = ::Pointer(self).null end end diff --git a/src/ecr/macros.cr b/src/ecr/macros.cr index 92c02cc4284a..5e051232271b 100644 --- a/src/ecr/macros.cr +++ b/src/ecr/macros.cr @@ -34,7 +34,7 @@ module ECR # ``` macro def_to_s(filename) def to_s(__io__ : IO) : Nil - ECR.embed {{filename}}, "__io__" + ::ECR.embed {{filename}}, "__io__" end end diff --git a/src/intrinsics.cr b/src/intrinsics.cr index c5ae837d8931..7cdc462ce543 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -179,7 +179,7 @@ end module Intrinsics macro debugtrap - LibIntrinsics.debugtrap + ::LibIntrinsics.debugtrap end def self.pause @@ -191,15 +191,15 @@ module Intrinsics end macro memcpy(dest, src, len, is_volatile) - LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memcpy({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memmove(dest, src, len, is_volatile) - LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memmove({{dest}}, {{src}}, {{len}}, {{is_volatile}}) end macro memset(dest, val, len, is_volatile) - LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) + ::LibIntrinsics.memset({{dest}}, {{val}}, {{len}}, {{is_volatile}}) end def self.read_cycle_counter @@ -263,43 +263,43 @@ module Intrinsics end macro countleading8(src, zero_is_undef) - LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading8({{src}}, {{zero_is_undef}}) end macro countleading16(src, zero_is_undef) - LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading16({{src}}, {{zero_is_undef}}) end macro countleading32(src, zero_is_undef) - LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading32({{src}}, {{zero_is_undef}}) end macro countleading64(src, zero_is_undef) - LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading64({{src}}, {{zero_is_undef}}) end macro countleading128(src, zero_is_undef) - LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.countleading128({{src}}, {{zero_is_undef}}) end macro counttrailing8(src, zero_is_undef) - LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing8({{src}}, {{zero_is_undef}}) end macro counttrailing16(src, zero_is_undef) - LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing16({{src}}, {{zero_is_undef}}) end macro counttrailing32(src, zero_is_undef) - LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing32({{src}}, {{zero_is_undef}}) end macro counttrailing64(src, zero_is_undef) - LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing64({{src}}, {{zero_is_undef}}) end macro counttrailing128(src, zero_is_undef) - LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) + ::LibIntrinsics.counttrailing128({{src}}, {{zero_is_undef}}) end def self.fshl8(a, b, count) : UInt8 @@ -343,14 +343,14 @@ module Intrinsics end macro va_start(ap) - LibIntrinsics.va_start({{ap}}) + ::LibIntrinsics.va_start({{ap}}) end macro va_end(ap) - LibIntrinsics.va_end({{ap}}) + ::LibIntrinsics.va_end({{ap}}) end end macro debugger - Intrinsics.debugtrap + ::Intrinsics.debugtrap end diff --git a/src/json/serialization.cr b/src/json/serialization.cr index b1eb86d15082..15d948f02f40 100644 --- a/src/json/serialization.cr +++ b/src/json/serialization.cr @@ -164,7 +164,7 @@ module JSON private def self.new_from_json_pull_parser(pull : ::JSON::PullParser) instance = allocate instance.initialize(__pull_for_json_serializable: pull) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -422,8 +422,8 @@ module JSON # Try to find the discriminator while also getting the raw # string value of the parsed JSON, so then we can pass it # to the final type. - json = String.build do |io| - JSON.build(io) do |builder| + json = ::String.build do |io| + ::JSON.build(io) do |builder| builder.start_object pull.read_object do |key| if key == {{field.id.stringify}} diff --git a/src/number.cr b/src/number.cr index f7c82aa4cded..9d955c065df3 100644 --- a/src/number.cr +++ b/src/number.cr @@ -59,7 +59,7 @@ struct Number # :nodoc: macro expand_div(rhs_types, result_type) {% for rhs in rhs_types %} - @[AlwaysInline] + @[::AlwaysInline] def /(other : {{rhs}}) : {{result_type}} {{result_type}}.new(self) / {{result_type}}.new(other) end @@ -84,7 +84,7 @@ struct Number # [1, 2, 3, 4] of Int64 # : Array(Int64) # ``` macro [](*nums) - Array({{@type}}).build({{nums.size}}) do |%buffer| + ::Array({{@type}}).build({{nums.size}}) do |%buffer| {% for num, i in nums %} %buffer[{{i}}] = {{@type}}.new({{num}}) {% end %} @@ -113,7 +113,7 @@ struct Number # Slice[1_i64, 2_i64, 3_i64, 4_i64] # : Slice(Int64) # ``` macro slice(*nums, read_only = false) - %slice = Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) + %slice = ::Slice({{@type}}).new({{nums.size}}, read_only: {{read_only}}) {% for num, i in nums %} %slice.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} @@ -139,7 +139,7 @@ struct Number # StaticArray[1_i64, 2_i64, 3_i64, 4_i64] # : StaticArray(Int64) # ``` macro static_array(*nums) - %array = uninitialized StaticArray({{@type}}, {{nums.size}}) + %array = uninitialized ::StaticArray({{@type}}, {{nums.size}}) {% for num, i in nums %} %array.to_unsafe[{{i}}] = {{@type}}.new!({{num}}) {% end %} diff --git a/src/object.cr b/src/object.cr index ba818ac2979e..800736687788 100644 --- a/src/object.cr +++ b/src/object.cr @@ -562,7 +562,7 @@ class Object def {{method_prefix}}\{{name.var.id}} : \{{name.type}} if (value = {{var_prefix}}\{{name.var.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.var.id}} cannot be nil") else value end @@ -574,7 +574,7 @@ class Object def {{method_prefix}}\{{name.id}} if (value = {{var_prefix}}\{{name.id}}).nil? - ::raise NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") + ::raise ::NilAssertionError.new("\{{@type}}\{{"{{doc_prefix}}".id}}\{{name.id}} cannot be nil") else value end @@ -1293,7 +1293,7 @@ class Object # wrapper.capitalize # => "Hello" # ``` macro delegate(*methods, to object) - {% if compare_versions(Crystal::VERSION, "1.12.0-dev") >= 0 %} + {% if compare_versions(::Crystal::VERSION, "1.12.0-dev") >= 0 %} {% eq_operators = %w(<= >= == != []= ===) %} {% for method in methods %} {% if method.id.ends_with?('=') && !eq_operators.includes?(method.id.stringify) %} @@ -1427,18 +1427,18 @@ class Object macro def_clone # Returns a copy of `self` with all instance variables cloned. def clone - \{% if @type < Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} + \{% if @type < ::Reference && !@type.instance_vars.map(&.type).all? { |t| t == ::Bool || t == ::Char || t == ::Symbol || t == ::String || t < ::Number::Primitive } %} exec_recursive_clone do |hash| clone = \{{@type}}.allocate hash[object_id] = clone.object_id clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone end \{% else %} clone = \{{@type}}.allocate clone.initialize_copy(self) - GC.add_finalizer(clone) if clone.responds_to?(:finalize) + ::GC.add_finalizer(clone) if clone.responds_to?(:finalize) clone \{% end %} end diff --git a/src/slice.cr b/src/slice.cr index 196a29a768dd..087679d37cb7 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -34,14 +34,14 @@ struct Slice(T) macro [](*args, read_only = false) # TODO: there should be a better way to check this, probably # asking if @type was instantiated or if T is defined - {% if @type.name != "Slice(T)" && T < Number %} + {% if @type.name != "Slice(T)" && T < ::Number %} {{T}}.slice({{args.splat(", ")}}read_only: {{read_only}}) {% else %} - %ptr = Pointer(typeof({{args.splat}})).malloc({{args.size}}) + %ptr = ::Pointer(typeof({{args.splat}})).malloc({{args.size}}) {% for arg, i in args %} %ptr[{{i}}] = {{arg}} {% end %} - Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) + ::Slice.new(%ptr, {{args.size}}, read_only: {{read_only}}) {% end %} end diff --git a/src/spec/dsl.cr b/src/spec/dsl.cr index 578076b86d69..d712aa59da4f 100644 --- a/src/spec/dsl.cr +++ b/src/spec/dsl.cr @@ -298,8 +298,8 @@ module Spec # If the "log" module is required it is configured to emit no entries by default. def log_setup defined?(::Log) do - if Log.responds_to?(:setup) - Log.setup_from_env(default_level: :none) + if ::Log.responds_to?(:setup) + ::Log.setup_from_env(default_level: :none) end end end diff --git a/src/spec/helpers/iterate.cr b/src/spec/helpers/iterate.cr index be302ebb49c2..7a70f83408ca 100644 --- a/src/spec/helpers/iterate.cr +++ b/src/spec/helpers/iterate.cr @@ -47,7 +47,7 @@ module Spec::Methods # See `.it_iterates` for details. macro assert_iterates_yielding(expected, method, *, infinite = false, tuple = false) %remaining = ({{expected}}).size - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) {{ method.id }} do |{% if tuple %}*{% end %}x| if %remaining == 0 if {{ infinite }} @@ -73,11 +73,11 @@ module Spec::Methods # # See `.it_iterates` for details. macro assert_iterates_iterator(expected, method, *, infinite = false) - %ary = [] of typeof(Enumerable.element_type({{ expected }})) + %ary = [] of typeof(::Enumerable.element_type({{ expected }})) %iter = {{ method.id }} ({{ expected }}).size.times do %v = %iter.next - if %v.is_a?(Iterator::Stop) + if %v.is_a?(::Iterator::Stop) # Compare the actual value directly. Since there are less # then expected values, the expectation will fail and raise. %ary.should eq({{ expected }}) @@ -86,7 +86,7 @@ module Spec::Methods %ary << %v end unless {{ infinite }} - %iter.next.should be_a(Iterator::Stop) + %iter.next.should be_a(::Iterator::Stop) end %ary.should eq({{ expected }}) diff --git a/src/static_array.cr b/src/static_array.cr index 2c09e21df166..3d00705bc21a 100644 --- a/src/static_array.cr +++ b/src/static_array.cr @@ -50,7 +50,7 @@ struct StaticArray(T, N) # * `Number.static_array` is a convenient alternative for designating a # specific numerical item type. macro [](*args) - %array = uninitialized StaticArray(typeof({{args.splat}}), {{args.size}}) + %array = uninitialized ::StaticArray(typeof({{args.splat}}), {{args.size}}) {% for arg, i in args %} %array.to_unsafe[{{i}}] = {{arg}} {% end %} diff --git a/src/syscall/aarch64-linux.cr b/src/syscall/aarch64-linux.cr index 5a61e8e7eed8..77b891fe2a7c 100644 --- a/src/syscall/aarch64-linux.cr +++ b/src/syscall/aarch64-linux.cr @@ -334,7 +334,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/arm-linux.cr b/src/syscall/arm-linux.cr index 97119fc4b3f3..da349dd45301 100644 --- a/src/syscall/arm-linux.cr +++ b/src/syscall/arm-linux.cr @@ -409,7 +409,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/i386-linux.cr b/src/syscall/i386-linux.cr index 843b2d1fd856..a0f94a51160a 100644 --- a/src/syscall/i386-linux.cr +++ b/src/syscall/i386-linux.cr @@ -445,7 +445,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/syscall/x86_64-linux.cr b/src/syscall/x86_64-linux.cr index 1f01c9226658..5a63b6ee2e1a 100644 --- a/src/syscall/x86_64-linux.cr +++ b/src/syscall/x86_64-linux.cr @@ -368,7 +368,7 @@ module Syscall end macro def_syscall(name, return_type, *args) - @[AlwaysInline] + @[::AlwaysInline] def self.{{name.id}}({{args.splat}}) : {{return_type}} ret = uninitialized {{return_type}} diff --git a/src/yaml/serialization.cr b/src/yaml/serialization.cr index d5fae8dfe9c0..4a1521469dea 100644 --- a/src/yaml/serialization.cr +++ b/src/yaml/serialization.cr @@ -156,11 +156,11 @@ module YAML # Define a `new` directly in the included type, # so it overloads well with other possible initializes - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end - private def self.new_from_yaml_node(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + private def self.new_from_yaml_node(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, self) do |obj| return obj end @@ -170,7 +170,7 @@ module YAML ctx.record_anchor(node, instance) instance.initialize(__context_for_yaml_serializable: ctx, __node_for_yaml_serializable: node) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) + ::GC.add_finalizer(instance) if instance.responds_to?(:finalize) instance end @@ -178,7 +178,7 @@ module YAML # so it can compete with other possible initializes macro inherited - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) new_from_yaml_node(ctx, node) end end @@ -409,17 +409,17 @@ module YAML {% mapping.raise "Mapping argument must be a HashLiteral or a NamedTupleLiteral, not #{mapping.class_name.id}" %} {% end %} - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + def self.new(ctx : ::YAML::ParseContext, node : ::YAML::Nodes::Node) ctx.read_alias(node, \{{@type}}) do |obj| return obj end - unless node.is_a?(YAML::Nodes::Mapping) + unless node.is_a?(::YAML::Nodes::Mapping) node.raise "Expected YAML mapping, not #{node.class}" end node.each do |key, value| - next unless key.is_a?(YAML::Nodes::Scalar) && value.is_a?(YAML::Nodes::Scalar) + next unless key.is_a?(::YAML::Nodes::Scalar) && value.is_a?(::YAML::Nodes::Scalar) next unless key.value == {{field.id.stringify}} discriminator_value = value.value From 0f257ef57bdf34f156dd7bb5bcb0624258eb07cb Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 16 Sep 2024 19:56:21 +0800 Subject: [PATCH 113/193] Fix `Process.exec` stream redirection on Windows (#14986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following should print the compiler help message to the file `foo.txt`: ```crystal File.open("foo.txt", "w") do |f| Process.exec("crystal", output: f) end ``` It used to work on Windows in Crystal 1.12, but is now broken since 1.13. This is because `LibC._wexecvp` only inherits file descriptors in the C runtime, not arbitrary Win32 file handles; since we stopped calling `LibC._open_osfhandle`, the C runtime knows nothing about any reopened standard streams in Win32. Thus the above merely prints the help message to the standard output. This PR creates the missing C file descriptors right before `LibC._wexecvp`. It also fixes a different regression of #14947 where reconfiguring `STDIN.blocking` always fails. Co-authored-by: Johannes Müller --- spec/std/process_spec.cr | 21 ++++++++++ src/crystal/system/win32/process.cr | 59 +++++++++++++++++++-------- src/lib_c/x86_64-windows-msvc/c/io.cr | 5 ++- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index f067d2f5c775..f1437656fffa 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -465,6 +465,27 @@ pending_interpreted describe: Process do {% end %} describe ".exec" do + it "redirects STDIN and STDOUT to files", tags: %w[slow] do + with_tempfile("crystal-exec-stdin", "crystal-exec-stdout") do |stdin_path, stdout_path| + File.write(stdin_path, "foobar") + + status, _, _ = compile_and_run_source <<-CRYSTAL + command = #{stdin_to_stdout_command[0].inspect} + args = #{stdin_to_stdout_command[1].to_a} of String + stdin_path = #{stdin_path.inspect} + stdout_path = #{stdout_path.inspect} + File.open(stdin_path) do |input| + File.open(stdout_path, "w") do |output| + Process.exec(command, args, input: input, output: output) + end + end + CRYSTAL + + status.success?.should be_true + File.read(stdout_path).chomp.should eq("foobar") + end + end + it "gets error from exec" do expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do Process.exec("foobarbaz") diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 05b2ea36584e..1817be9ee27f 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -326,9 +326,9 @@ struct Crystal::System::Process end private def self.try_replace(command_args, env, clear_env, input, output, error, chdir) - reopen_io(input, ORIGINAL_STDIN) - reopen_io(output, ORIGINAL_STDOUT) - reopen_io(error, ORIGINAL_STDERR) + old_input_fd = reopen_io(input, ORIGINAL_STDIN) + old_output_fd = reopen_io(output, ORIGINAL_STDOUT) + old_error_fd = reopen_io(error, ORIGINAL_STDERR) ENV.clear if clear_env env.try &.each do |key, val| @@ -351,11 +351,18 @@ struct Crystal::System::Process argv << Pointer(LibC::WCHAR).null LibC._wexecvp(command, argv) + + # exec failed; restore the original C runtime file descriptors + errno = Errno.value + LibC._dup2(old_input_fd, 0) + LibC._dup2(old_output_fd, 1) + LibC._dup2(old_error_fd, 2) + errno end def self.replace(command_args, env, clear_env, input, output, error, chdir) : NoReturn - try_replace(command_args, env, clear_env, input, output, error, chdir) - raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0]) + errno = try_replace(command_args, env, clear_env, input, output, error, chdir) + raise_exception_from_errno(command_args.is_a?(String) ? command_args : command_args[0], errno) end private def self.raise_exception_from_errno(command, errno = Errno.value) @@ -367,21 +374,41 @@ struct Crystal::System::Process end end + # Replaces the C standard streams' file descriptors, not Win32's, since + # `try_replace` uses the C `LibC._wexecvp` and only cares about the former. + # Returns a duplicate of the original file descriptor private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) - src_io = to_real_fd(src_io) + unless src_io.system_blocking? + raise IO::Error.new("Non-blocking streams are not supported in `Process.exec`", target: src_io) + end - dst_io.reopen(src_io) - dst_io.blocking = true - dst_io.close_on_exec = false - end + src_fd = + case src_io + when STDIN then 0 + when STDOUT then 1 + when STDERR then 2 + else + LibC._open_osfhandle(src_io.windows_handle, 0) + end - private def self.to_real_fd(fd : IO::FileDescriptor) - case fd - when STDIN then ORIGINAL_STDIN - when STDOUT then ORIGINAL_STDOUT - when STDERR then ORIGINAL_STDERR - else fd + dst_fd = + case dst_io + when ORIGINAL_STDIN then 0 + when ORIGINAL_STDOUT then 1 + when ORIGINAL_STDERR then 2 + else + raise "BUG: Invalid destination IO" + end + + return src_fd if dst_fd == src_fd + + orig_src_fd = LibC._dup(src_fd) + + if LibC._dup2(src_fd, dst_fd) == -1 + raise IO::Error.from_errno("Failed to replace C file descriptor", target: dst_io) end + + orig_src_fd end def self.chroot(path) diff --git a/src/lib_c/x86_64-windows-msvc/c/io.cr b/src/lib_c/x86_64-windows-msvc/c/io.cr index 75da8c18e5b9..ccbaa15f2d1b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/io.cr +++ b/src/lib_c/x86_64-windows-msvc/c/io.cr @@ -2,12 +2,13 @@ require "c/stdint" lib LibC fun _wexecvp(cmdname : WCHAR*, argv : WCHAR**) : IntPtrT + fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int + fun _dup(fd : Int) : Int + fun _dup2(fd1 : Int, fd2 : Int) : Int # unused - fun _open_osfhandle(osfhandle : HANDLE, flags : LibC::Int) : LibC::Int fun _get_osfhandle(fd : Int) : IntPtrT fun _close(fd : Int) : Int - fun _dup2(fd1 : Int, fd2 : Int) : Int fun _isatty(fd : Int) : Int fun _write(fd : Int, buffer : UInt8*, count : UInt) : Int fun _read(fd : Int, buffer : UInt8*, count : UInt) : Int From ced75401dac79f9abfe77e2aa6181a62c1283107 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 25 Aug 2024 19:15:11 +0800 Subject: [PATCH 114/193] Fix `String#index` and `#rindex` for `Char::REPLACEMENT` (#14937) If the string consists only of ASCII characters and invalid UTF-8 byte sequences, all the latter should correspond to `Char::REPLACEMENT`, and so `#index` and `#rindex` should detect them, but this was broken since #14461. --- spec/std/string_spec.cr | 6 ++++++ src/string.cr | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 00310bfcbc47..9c4f3cfc5d12 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -945,6 +945,7 @@ describe "String" do it { "日本語".index('本').should eq(1) } it { "bar".index('あ').should be_nil } it { "あいう_えお".index('_').should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}').should eq(3) } describe "with offset" do it { "foobarbaz".index('a', 5).should eq(7) } @@ -952,6 +953,8 @@ describe "String" do it { "foo".index('g', 1).should be_nil } it { "foo".index('g', -20).should be_nil } it { "日本語日本語".index('本', 2).should eq(4) } + it { "xyz\xFFxyz".index('\u{FFFD}', 2).should eq(3) } + it { "xyz\xFFxyz".index('\u{FFFD}', 4).should be_nil } # Check offset type it { "foobarbaz".index('a', 5_i64).should eq(7) } @@ -1094,6 +1097,7 @@ describe "String" do it { "foobar".rindex('g').should be_nil } it { "日本語日本語".rindex('本').should eq(4) } it { "あいう_えお".rindex('_').should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}').should eq(3) } describe "with offset" do it { "bbbb".rindex('b', 2).should eq(2) } @@ -1106,6 +1110,8 @@ describe "String" do it { "faobar".rindex('a', 3).should eq(1) } it { "faobarbaz".rindex('a', -3).should eq(4) } it { "日本語日本語".rindex('本', 3).should eq(1) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 4).should eq(3) } + it { "xyz\xFFxyz".rindex('\u{FFFD}', 2).should be_nil } # Check offset type it { "bbbb".rindex('b', 2_i64).should eq(2) } diff --git a/src/string.cr b/src/string.cr index d3bc7d6998b2..ab107c454e8c 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3335,11 +3335,21 @@ class String def index(search : Char, offset = 0) : Int32? # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.fast_index(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.upto(bytesize - 1) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.fast_index(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 @@ -3455,11 +3465,21 @@ class String def rindex(search : Char, offset = size - 1) # If it's ASCII we can delegate to slice if single_byte_optimizable? - # With `single_byte_optimizable?` there are only ASCII characters and invalid UTF-8 byte - # sequences and we can immediately reject any non-ASCII codepoint. - return unless search.ascii? + # With `single_byte_optimizable?` there are only ASCII characters and + # invalid UTF-8 byte sequences, and we can reject anything that is neither + # ASCII nor the replacement character. + case search + when .ascii? + return to_slice.rindex(search.ord.to_u8!, offset) + when Char::REPLACEMENT + offset.downto(0) do |i| + if to_unsafe[i] >= 0x80 + return i.to_i + end + end + end - return to_slice.rindex(search.ord.to_u8, offset) + return nil end offset += size if offset < 0 From d14d04562125ee1f2c3985d756c5f0e4cd68a9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 18 Sep 2024 15:43:39 +0200 Subject: [PATCH 115/193] Changelog for 1.13.3 (#14991) --- CHANGELOG.md | 24 ++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 2 +- src/VERSION | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f97d0bedeb1b..341586a8fb95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [1.13.3] (2024-09-18) + +[1.13.3]: https://github.com/crystal-lang/crystal/releases/1.13.3 + +### Bugfixes + +#### stdlib + +- **[regression]** Fix use global paths in macro bodies ([#14965], thanks @straight-shoota) +- *(system)* **[regression]** Fix `Process.exec` stream redirection on Windows ([#14986], thanks @HertzDevil) +- *(text)* **[regression]** Fix `String#index` and `#rindex` for `Char::REPLACEMENT` ([#14937], thanks @HertzDevil) + +[#14965]: https://github.com/crystal-lang/crystal/pull/14965 +[#14986]: https://github.com/crystal-lang/crystal/pull/14986 +[#14937]: https://github.com/crystal-lang/crystal/pull/14937 + +### Infrastructure + +- Changelog for 1.13.3 ([#14991], thanks @straight-shoota) +- *(ci)* Enable runners from `runs-on.com` for Aarch64 CI ([#15007], thanks @straight-shoota) + +[#14991]: https://github.com/crystal-lang/crystal/pull/14991 +[#15007]: https://github.com/crystal-lang/crystal/pull/15007 + ## [1.13.2] (2024-08-20) [1.13.2]: https://github.com/crystal-lang/crystal/releases/1.13.2 diff --git a/shard.yml b/shard.yml index 0dd8c2abf3a1..6463a5681c65 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.13.2 +version: 1.13.3 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH index 0ea6bd82d669..13a79f624e9d 100644 --- a/src/SOURCE_DATE_EPOCH +++ b/src/SOURCE_DATE_EPOCH @@ -1 +1 @@ -1724112000 +1726617600 diff --git a/src/VERSION b/src/VERSION index 61ce01b30118..01b7568230eb 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.13.2 +1.13.3 From 626e8f7c55cc573c321476b5fc4dd2e0986167bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 18 Sep 2024 23:01:49 +0200 Subject: [PATCH 116/193] Remove `XML::Error.errors` (#14936) Co-authored-by: Beta Ziliani --- spec/std/xml/reader_spec.cr | 10 ---------- src/xml/error.cr | 15 +-------------- src/xml/reader.cr | 4 +--- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/spec/std/xml/reader_spec.cr b/spec/std/xml/reader_spec.cr index d89593620970..4ec3d8cddc5c 100644 --- a/spec/std/xml/reader_spec.cr +++ b/spec/std/xml/reader_spec.cr @@ -577,15 +577,5 @@ module XML reader.errors.map(&.to_s).should eq ["Opening and ending tag mismatch: people line 1 and foo"] end - - it "adds errors to `XML::Error.errors` (deprecated)" do - XML::Error.errors # clear class error list - - reader = XML::Reader.new(%()) - reader.read - reader.expand? - - XML::Error.errors.try(&.map(&.to_s)).should eq ["Opening and ending tag mismatch: people line 1 and foo"] - end end end diff --git a/src/xml/error.cr b/src/xml/error.cr index 868dfeb4bd00..389aa53910c2 100644 --- a/src/xml/error.cr +++ b/src/xml/error.cr @@ -11,22 +11,9 @@ class XML::Error < Exception super(message) end - @@errors = [] of self - - # :nodoc: - protected def self.add_errors(errors) - @@errors.concat(errors) - end - @[Deprecated("This class accessor is deprecated. XML errors are accessible directly in the respective context via `XML::Reader#errors` and `XML::Node#errors`.")] def self.errors : Array(XML::Error)? - if @@errors.empty? - nil - else - errors = @@errors.dup - @@errors.clear - errors - end + {% raise "`XML::Error.errors` was removed because it leaks memory when it's not used. XML errors are accessible directly in the respective context via `XML::Reader#errors` and `XML::Node#errors`.\nSee https://github.com/crystal-lang/crystal/issues/14934 for details. " %} end def self.collect(errors, &) diff --git a/src/xml/reader.cr b/src/xml/reader.cr index decdd8468185..d4dbe91f7eeb 100644 --- a/src/xml/reader.cr +++ b/src/xml/reader.cr @@ -198,9 +198,7 @@ class XML::Reader end private def collect_errors(&) - Error.collect(@errors) { yield }.tap do - Error.add_errors(@errors) - end + Error.collect(@errors) { yield } end private def check_no_null_byte(attribute) From 62541690fe775778b9b1be2728202e71d2e76dda Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 19 Sep 2024 21:20:45 +0800 Subject: [PATCH 117/193] Use our own `libffi` repository on Windows CI (#14998) I forked the libffi upstream and wrote [my own `CMakeLists.txt`](https://github.com/crystal-lang/libffi/blob/441390ce33ae2d9bf2916184fe6b7207b306dd3e/CMakeLists.txt). It only handles x64 MSVC, but we could easily extend it to support ARM64 in the near future. Note that the Windows CI already uses libffi since there are interpreter tests and stdlib tests running with the interpreter. --- .github/workflows/win.yml | 4 ++-- etc/win-ci/build-ffi.ps1 | 43 +++++++++------------------------------ 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index d4b9316ef1a2..89c13959e8cb 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -60,7 +60,7 @@ jobs: run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 - name: Build libffi if: steps.cache-libs.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 + run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.4.6 - name: Build zlib if: steps.cache-libs.outputs.cache-hit != 'true' run: .\etc\win-ci\build-z.ps1 -BuildTree deps\z -Version 1.3.1 @@ -148,7 +148,7 @@ jobs: run: .\etc\win-ci\build-iconv.ps1 -BuildTree deps\iconv -Version 1.17 -Dynamic - name: Build libffi if: steps.cache-dlls.outputs.cache-hit != 'true' - run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.3 -Dynamic + run: .\etc\win-ci\build-ffi.ps1 -BuildTree deps\ffi -Version 3.4.6 -Dynamic - name: Build zlib if: steps.cache-dlls.outputs.cache-hit != 'true' run: .\etc\win-ci\build-z.ps1 -BuildTree deps\z -Version 1.3.1 -Dynamic diff --git a/etc/win-ci/build-ffi.ps1 b/etc/win-ci/build-ffi.ps1 index 4340630bea64..eb5ec1e5403c 100644 --- a/etc/win-ci/build-ffi.ps1 +++ b/etc/win-ci/build-ffi.ps1 @@ -7,40 +7,17 @@ param( . "$(Split-Path -Parent $MyInvocation.MyCommand.Path)\setup.ps1" [void](New-Item -Name (Split-Path -Parent $BuildTree) -ItemType Directory -Force) -Setup-Git -Path $BuildTree -Url https://github.com/winlibs/libffi.git -Ref libffi-$Version +Setup-Git -Path $BuildTree -Url https://github.com/crystal-lang/libffi.git -Ref v$Version Run-InDirectory $BuildTree { + $args = "-DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF" if ($Dynamic) { - Replace-Text win32\vs16_x64\libffi\libffi.vcxproj 'StaticLibrary' 'DynamicLibrary' + $args = "-DBUILD_SHARED_LIBS=ON $args" + } else { + $args = "-DBUILD_SHARED_LIBS=OFF -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded $args" } - - echo ' - - $(MsbuildThisFileDirectory)\Override.props - - ' > 'Directory.Build.props' - - echo " - - false - - - - $(if ($Dynamic) { - 'FFI_BUILDING_DLL;%(PreprocessorDefinitions)' - } else { - 'MultiThreaded' - }) - None - false - - - false - - - " > 'Override.props' - - MSBuild.exe /p:PlatformToolset=v143 /p:Platform=x64 /p:Configuration=Release win32\vs16_x64\libffi-msvc.sln -target:libffi:Rebuild + & $cmake . $args.split(' ') + & $cmake --build . --config Release if (-not $?) { Write-Host "Error: Failed to build libffi" -ForegroundColor Red Exit 1 @@ -48,8 +25,8 @@ Run-InDirectory $BuildTree { } if ($Dynamic) { - mv -Force $BuildTree\win32\vs16_x64\x64\Release\libffi.lib libs\ffi-dynamic.lib - mv -Force $BuildTree\win32\vs16_x64\x64\Release\libffi.dll dlls\ + mv -Force $BuildTree\Release\libffi.lib libs\ffi-dynamic.lib + mv -Force $BuildTree\Release\libffi.dll dlls\ } else { - mv -Force $BuildTree\win32\vs16_x64\x64\Release\libffi.lib libs\ffi.lib + mv -Force $BuildTree\Release\libffi.lib libs\ffi.lib } From 47d174825e674c57c0d3b2036fa6975277528614 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 19 Sep 2024 21:20:55 +0800 Subject: [PATCH 118/193] Support Unicode 16.0.0 (#14997) --- spec/std/string/grapheme_break_spec.cr | 454 ++++++++++--------------- src/string/grapheme/properties.cr | 103 +++--- src/unicode/data.cr | 267 ++++++++++++--- src/unicode/unicode.cr | 2 +- 4 files changed, 459 insertions(+), 367 deletions(-) diff --git a/spec/std/string/grapheme_break_spec.cr b/spec/std/string/grapheme_break_spec.cr index f1a86656ef12..2ea30c104016 100644 --- a/spec/std/string/grapheme_break_spec.cr +++ b/spec/std/string/grapheme_break_spec.cr @@ -16,8 +16,8 @@ describe "String#each_grapheme" do it_iterates_graphemes " \u0308\n", [" \u0308", '\n'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes " \u0001", [' ', '\u0001'] # ÷ [0.2] SPACE (Other) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes " \u0308\u0001", [" \u0308", '\u0001'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes " \u034F", [" \u034F"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes " \u0308\u034F", [" \u0308\u034F"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes " \u200C", [" \u200C"] # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes " \u0308\u200C", [" \u0308\u200C"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes " \u{1F1E6}", [' ', '\u{1F1E6}'] # ÷ [0.2] SPACE (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes " \u0308\u{1F1E6}", [" \u0308", '\u{1F1E6}'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes " \u0600", [' ', '\u0600'] # ÷ [0.2] SPACE (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -34,8 +34,6 @@ describe "String#each_grapheme" do it_iterates_graphemes " \u0308\uAC00", [" \u0308", '\uAC00'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes " \uAC01", [' ', '\uAC01'] # ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes " \u0308\uAC01", [" \u0308", '\uAC01'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes " \u0900", [" \u0900"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes " \u0308\u0900", [" \u0308\u0900"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes " \u0903", [" \u0903"] # ÷ [0.2] SPACE (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes " \u0308\u0903", [" \u0308\u0903"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes " \u0904", [' ', '\u0904'] # ÷ [0.2] SPACE (Other) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -48,8 +46,8 @@ describe "String#each_grapheme" do it_iterates_graphemes " \u0308\u231A", [" \u0308", '\u231A'] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes " \u0300", [" \u0300"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u0308\u0300", [" \u0308\u0300"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes " \u093C", [" \u093C"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes " \u0308\u093C", [" \u0308\u093C"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes " \u0900", [" \u0900"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes " \u0308\u0900", [" \u0308\u0900"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u094D", [" \u094D"] # ÷ [0.2] SPACE (Other) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u0308\u094D", [" \u0308\u094D"] # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes " \u200D", [" \u200D"] # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -64,8 +62,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\r\u0308\n", ['\r', '\u0308', '\n'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\r\u0001", ['\r', '\u0001'] # ÷ [0.2] (CR) ÷ [4.0] (Control) ÷ [0.3] it_iterates_graphemes "\r\u0308\u0001", ['\r', '\u0308', '\u0001'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\r\u034F", ['\r', '\u034F'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\r\u0308\u034F", ['\r', "\u0308\u034F"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\r\u200C", ['\r', '\u200C'] # ÷ [0.2] (CR) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\r\u0308\u200C", ['\r', "\u0308\u200C"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\r\u{1F1E6}", ['\r', '\u{1F1E6}'] # ÷ [0.2] (CR) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\r\u0308\u{1F1E6}", ['\r', '\u0308', '\u{1F1E6}'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\r\u0600", ['\r', '\u0600'] # ÷ [0.2] (CR) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -82,8 +80,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\r\u0308\uAC00", ['\r', '\u0308', '\uAC00'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\r\uAC01", ['\r', '\uAC01'] # ÷ [0.2] (CR) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\r\u0308\uAC01", ['\r', '\u0308', '\uAC01'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\r\u0900", ['\r', '\u0900'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\r\u0308\u0900", ['\r', "\u0308\u0900"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\r\u0903", ['\r', '\u0903'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\r\u0308\u0903", ['\r', "\u0308\u0903"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\r\u0904", ['\r', '\u0904'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -96,8 +92,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\r\u0308\u231A", ['\r', '\u0308', '\u231A'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\r\u0300", ['\r', '\u0300'] # ÷ [0.2] (CR) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u0308\u0300", ['\r', "\u0308\u0300"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\r\u093C", ['\r', '\u093C'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\r\u0308\u093C", ['\r', "\u0308\u093C"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\r\u0900", ['\r', '\u0900'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\r\u0308\u0900", ['\r', "\u0308\u0900"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u094D", ['\r', '\u094D'] # ÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u0308\u094D", ['\r', "\u0308\u094D"] # ÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\r\u200D", ['\r', '\u200D'] # ÷ [0.2] (CR) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -112,8 +108,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\n\u0308\n", ['\n', '\u0308', '\n'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\n\u0001", ['\n', '\u0001'] # ÷ [0.2] (LF) ÷ [4.0] (Control) ÷ [0.3] it_iterates_graphemes "\n\u0308\u0001", ['\n', '\u0308', '\u0001'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\n\u034F", ['\n', '\u034F'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\n\u0308\u034F", ['\n', "\u0308\u034F"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\n\u200C", ['\n', '\u200C'] # ÷ [0.2] (LF) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\n\u0308\u200C", ['\n', "\u0308\u200C"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\n\u{1F1E6}", ['\n', '\u{1F1E6}'] # ÷ [0.2] (LF) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\n\u0308\u{1F1E6}", ['\n', '\u0308', '\u{1F1E6}'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\n\u0600", ['\n', '\u0600'] # ÷ [0.2] (LF) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -130,8 +126,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\n\u0308\uAC00", ['\n', '\u0308', '\uAC00'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\n\uAC01", ['\n', '\uAC01'] # ÷ [0.2] (LF) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\n\u0308\uAC01", ['\n', '\u0308', '\uAC01'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\n\u0900", ['\n', '\u0900'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\n\u0308\u0900", ['\n', "\u0308\u0900"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\n\u0903", ['\n', '\u0903'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\n\u0308\u0903", ['\n', "\u0308\u0903"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\n\u0904", ['\n', '\u0904'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -144,8 +138,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\n\u0308\u231A", ['\n', '\u0308', '\u231A'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\n\u0300", ['\n', '\u0300'] # ÷ [0.2] (LF) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u0308\u0300", ['\n', "\u0308\u0300"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\n\u093C", ['\n', '\u093C'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\n\u0308\u093C", ['\n', "\u0308\u093C"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\n\u0900", ['\n', '\u0900'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\n\u0308\u0900", ['\n', "\u0308\u0900"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u094D", ['\n', '\u094D'] # ÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u0308\u094D", ['\n', "\u0308\u094D"] # ÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\n\u200D", ['\n', '\u200D'] # ÷ [0.2] (LF) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -160,8 +154,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0001\u0308\n", ['\u0001', '\u0308', '\n'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0001\u0001", ['\u0001', '\u0001'] # ÷ [0.2] (Control) ÷ [4.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0001", ['\u0001', '\u0308', '\u0001'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0001\u034F", ['\u0001', '\u034F'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0001\u0308\u034F", ['\u0001', "\u0308\u034F"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0001\u200C", ['\u0001', '\u200C'] # ÷ [0.2] (Control) ÷ [4.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0001\u0308\u200C", ['\u0001', "\u0308\u200C"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0001\u{1F1E6}", ['\u0001', '\u{1F1E6}'] # ÷ [0.2] (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u{1F1E6}", ['\u0001', '\u0308', '\u{1F1E6}'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0001\u0600", ['\u0001', '\u0600'] # ÷ [0.2] (Control) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -178,8 +172,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0001\u0308\uAC00", ['\u0001', '\u0308', '\uAC00'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0001\uAC01", ['\u0001', '\uAC01'] # ÷ [0.2] (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\uAC01", ['\u0001', '\u0308', '\uAC01'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0001\u0900", ['\u0001', '\u0900'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0001\u0308\u0900", ['\u0001', "\u0308\u0900"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0001\u0903", ['\u0001', '\u0903'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0903", ['\u0001', "\u0308\u0903"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0001\u0904", ['\u0001', '\u0904'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -192,62 +184,60 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0001\u0308\u231A", ['\u0001', '\u0308', '\u231A'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0001\u0300", ['\u0001', '\u0300'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0300", ['\u0001', "\u0308\u0300"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0001\u093C", ['\u0001', '\u093C'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0001\u0308\u093C", ['\u0001', "\u0308\u093C"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0001\u0900", ['\u0001', '\u0900'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0001\u0308\u0900", ['\u0001', "\u0308\u0900"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u094D", ['\u0001', '\u094D'] # ÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u094D", ['\u0001', "\u0308\u094D"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u200D", ['\u0001', '\u200D'] # ÷ [0.2] (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u200D", ['\u0001', "\u0308\u200D"] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0001\u0378", ['\u0001', '\u0378'] # ÷ [0.2] (Control) ÷ [4.0] (Other) ÷ [0.3] it_iterates_graphemes "\u0001\u0308\u0378", ['\u0001', '\u0308', '\u0378'] # ÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u034F ", ['\u034F', ' '] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308 ", ["\u034F\u0308", ' '] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u034F\r", ['\u034F', '\r'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\r", ["\u034F\u0308", '\r'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u034F\n", ['\u034F', '\n'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\n", ["\u034F\u0308", '\n'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u034F\u0001", ['\u034F', '\u0001'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0001", ["\u034F\u0308", '\u0001'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u034F\u034F", ["\u034F\u034F"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u034F", ["\u034F\u0308\u034F"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u034F\u{1F1E6}", ['\u034F', '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u{1F1E6}", ["\u034F\u0308", '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u034F\u0600", ['\u034F', '\u0600'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0600", ["\u034F\u0308", '\u0600'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u034F\u0A03", ["\u034F\u0A03"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0A03", ["\u034F\u0308\u0A03"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u034F\u1100", ['\u034F', '\u1100'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u1100", ["\u034F\u0308", '\u1100'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u034F\u1160", ['\u034F', '\u1160'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u1160", ["\u034F\u0308", '\u1160'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u034F\u11A8", ['\u034F', '\u11A8'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u11A8", ["\u034F\u0308", '\u11A8'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u034F\uAC00", ['\u034F', '\uAC00'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\uAC00", ["\u034F\u0308", '\uAC00'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u034F\uAC01", ['\u034F', '\uAC01'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\uAC01", ["\u034F\u0308", '\uAC01'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u034F\u0900", ["\u034F\u0900"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0900", ["\u034F\u0308\u0900"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0903", ["\u034F\u0903"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0903", ["\u034F\u0308\u0903"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0904", ['\u034F', '\u0904'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0904", ["\u034F\u0308", '\u0904'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0D4E", ['\u034F', '\u0D4E'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0D4E", ["\u034F\u0308", '\u0D4E'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u034F\u0915", ['\u034F', '\u0915'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0915", ["\u034F\u0308", '\u0915'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u034F\u231A", ['\u034F', '\u231A'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u231A", ["\u034F\u0308", '\u231A'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u034F\u0300", ["\u034F\u0300"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0300", ["\u034F\u0308\u0300"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u093C", ["\u034F\u093C"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u093C", ["\u034F\u0308\u093C"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u094D", ["\u034F\u094D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u094D", ["\u034F\u0308\u094D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u200D", ["\u034F\u200D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u200D", ["\u034F\u0308\u200D"] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u034F\u0378", ['\u034F', '\u0378'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u034F\u0308\u0378", ["\u034F\u0308", '\u0378'] # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u200C ", ['\u200C', ' '] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308 ", ["\u200C\u0308", ' '] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u200C\r", ['\u200C', '\r'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\r", ["\u200C\u0308", '\r'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u200C\n", ['\u200C', '\n'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\n", ["\u200C\u0308", '\n'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u200C\u0001", ['\u200C', '\u0001'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0001", ["\u200C\u0308", '\u0001'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u200C\u200C", ["\u200C\u200C"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u200C", ["\u200C\u0308\u200C"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200C\u{1F1E6}", ['\u200C', '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u{1F1E6}", ["\u200C\u0308", '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u200C\u0600", ['\u200C', '\u0600'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0600", ["\u200C\u0308", '\u0600'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u200C\u0A03", ["\u200C\u0A03"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0A03", ["\u200C\u0308\u0A03"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u200C\u1100", ['\u200C', '\u1100'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u1100", ["\u200C\u0308", '\u1100'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u200C\u1160", ['\u200C', '\u1160'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u1160", ["\u200C\u0308", '\u1160'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u200C\u11A8", ['\u200C', '\u11A8'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u11A8", ["\u200C\u0308", '\u11A8'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u200C\uAC00", ['\u200C', '\uAC00'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\uAC00", ["\u200C\u0308", '\uAC00'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u200C\uAC01", ['\u200C', '\uAC01'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\uAC01", ["\u200C\u0308", '\uAC01'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u200C\u0903", ["\u200C\u0903"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0903", ["\u200C\u0308\u0903"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0904", ['\u200C', '\u0904'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0904", ["\u200C\u0308", '\u0904'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0D4E", ['\u200C', '\u0D4E'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0D4E", ["\u200C\u0308", '\u0D4E'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u200C\u0915", ['\u200C', '\u0915'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0915", ["\u200C\u0308", '\u0915'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u200C\u231A", ['\u200C', '\u231A'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u231A", ["\u200C\u0308", '\u231A'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u200C\u0300", ["\u200C\u0300"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0300", ["\u200C\u0308\u0300"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0900", ["\u200C\u0900"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0900", ["\u200C\u0308\u0900"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u094D", ["\u200C\u094D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u094D", ["\u200C\u0308\u094D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u200D", ["\u200C\u200D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u200D", ["\u200C\u0308\u200D"] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200C\u0378", ['\u200C', '\u0378'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u200C\u0308\u0378", ["\u200C\u0308", '\u0378'] # ÷ [0.2] ZERO WIDTH NON-JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u{1F1E6} ", ['\u{1F1E6}', ' '] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308 ", ["\u{1F1E6}\u0308", ' '] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\r", ['\u{1F1E6}', '\r'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] (CR) ÷ [0.3] @@ -256,8 +246,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u{1F1E6}\u0308\n", ["\u{1F1E6}\u0308", '\n'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0001", ['\u{1F1E6}', '\u0001'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u0001", ["\u{1F1E6}\u0308", '\u0001'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u034F", ["\u{1F1E6}\u034F"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0308\u034F", ["\u{1F1E6}\u0308\u034F"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u200C", ["\u{1F1E6}\u200C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u0308\u200C", ["\u{1F1E6}\u0308\u200C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u{1F1E6}", ["\u{1F1E6}\u{1F1E6}"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [12.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u{1F1E6}", ["\u{1F1E6}\u0308", '\u{1F1E6}'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0600", ['\u{1F1E6}', '\u0600'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -274,8 +264,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u{1F1E6}\u0308\uAC00", ["\u{1F1E6}\u0308", '\uAC00'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\uAC01", ['\u{1F1E6}', '\uAC01'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\uAC01", ["\u{1F1E6}\u0308", '\uAC01'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0900", ["\u{1F1E6}\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0308\u0900", ["\u{1F1E6}\u0308\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0903", ["\u{1F1E6}\u0903"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u0903", ["\u{1F1E6}\u0308\u0903"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0904", ['\u{1F1E6}', '\u0904'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -288,8 +276,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u{1F1E6}\u0308\u231A", ["\u{1F1E6}\u0308", '\u231A'] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0300", ["\u{1F1E6}\u0300"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u0300", ["\u{1F1E6}\u0308\u0300"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u093C", ["\u{1F1E6}\u093C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u{1F1E6}\u0308\u093C", ["\u{1F1E6}\u0308\u093C"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u0900", ["\u{1F1E6}\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u{1F1E6}\u0308\u0900", ["\u{1F1E6}\u0308\u0900"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u094D", ["\u{1F1E6}\u094D"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u0308\u094D", ["\u{1F1E6}\u0308\u094D"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F1E6}\u200D", ["\u{1F1E6}\u200D"] # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -304,8 +292,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0600\u0308\n", ["\u0600\u0308", '\n'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0600\u0001", ['\u0600', '\u0001'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u0001", ["\u0600\u0308", '\u0001'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0600\u034F", ["\u0600\u034F"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0600\u0308\u034F", ["\u0600\u0308\u034F"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0600\u200C", ["\u0600\u200C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0600\u0308\u200C", ["\u0600\u0308\u200C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0600\u{1F1E6}", ["\u0600\u{1F1E6}"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u{1F1E6}", ["\u0600\u0308", '\u{1F1E6}'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0600\u0600", ["\u0600\u0600"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -322,8 +310,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0600\u0308\uAC00", ["\u0600\u0308", '\uAC00'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0600\uAC01", ["\u0600\uAC01"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\uAC01", ["\u0600\u0308", '\uAC01'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0600\u0900", ["\u0600\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0600\u0308\u0900", ["\u0600\u0308\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0600\u0903", ["\u0600\u0903"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u0903", ["\u0600\u0308\u0903"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0600\u0904", ["\u0600\u0904"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -336,8 +322,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0600\u0308\u231A", ["\u0600\u0308", '\u231A'] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0600\u0300", ["\u0600\u0300"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u0300", ["\u0600\u0308\u0300"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0600\u093C", ["\u0600\u093C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0600\u0308\u093C", ["\u0600\u0308\u093C"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0600\u0900", ["\u0600\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0600\u0308\u0900", ["\u0600\u0308\u0900"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u094D", ["\u0600\u094D"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u0308\u094D", ["\u0600\u0308\u094D"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0600\u200D", ["\u0600\u200D"] # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -352,8 +338,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0A03\u0308\n", ["\u0A03\u0308", '\n'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0A03\u0001", ['\u0A03', '\u0001'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u0001", ["\u0A03\u0308", '\u0001'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0A03\u034F", ["\u0A03\u034F"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0308\u034F", ["\u0A03\u0308\u034F"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0A03\u200C", ["\u0A03\u200C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0A03\u0308\u200C", ["\u0A03\u0308\u200C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0A03\u{1F1E6}", ['\u0A03', '\u{1F1E6}'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u{1F1E6}", ["\u0A03\u0308", '\u{1F1E6}'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0A03\u0600", ['\u0A03', '\u0600'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -370,8 +356,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0A03\u0308\uAC00", ["\u0A03\u0308", '\uAC00'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0A03\uAC01", ['\u0A03', '\uAC01'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\uAC01", ["\u0A03\u0308", '\uAC01'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0900", ["\u0A03\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0308\u0900", ["\u0A03\u0308\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0A03\u0903", ["\u0A03\u0903"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u0903", ["\u0A03\u0308\u0903"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0A03\u0904", ['\u0A03', '\u0904'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -384,8 +368,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0A03\u0308\u231A", ["\u0A03\u0308", '\u231A'] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0A03\u0300", ["\u0A03\u0300"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u0300", ["\u0A03\u0308\u0300"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0A03\u093C", ["\u0A03\u093C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0A03\u0308\u093C", ["\u0A03\u0308\u093C"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0A03\u0900", ["\u0A03\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0A03\u0308\u0900", ["\u0A03\u0308\u0900"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u094D", ["\u0A03\u094D"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u0308\u094D", ["\u0A03\u0308\u094D"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0A03\u200D", ["\u0A03\u200D"] # ÷ [0.2] GURMUKHI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -400,8 +384,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1100\u0308\n", ["\u1100\u0308", '\n'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u1100\u0001", ['\u1100', '\u0001'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u0001", ["\u1100\u0308", '\u0001'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u1100\u034F", ["\u1100\u034F"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u1100\u0308\u034F", ["\u1100\u0308\u034F"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1100\u200C", ["\u1100\u200C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1100\u0308\u200C", ["\u1100\u0308\u200C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u1100\u{1F1E6}", ['\u1100', '\u{1F1E6}'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u{1F1E6}", ["\u1100\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1100\u0600", ['\u1100', '\u0600'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -418,8 +402,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1100\u0308\uAC00", ["\u1100\u0308", '\uAC00'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u1100\uAC01", ["\u1100\uAC01"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\uAC01", ["\u1100\u0308", '\uAC01'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u1100\u0900", ["\u1100\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u1100\u0308\u0900", ["\u1100\u0308\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1100\u0903", ["\u1100\u0903"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u0903", ["\u1100\u0308\u0903"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1100\u0904", ['\u1100', '\u0904'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -432,8 +414,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1100\u0308\u231A", ["\u1100\u0308", '\u231A'] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u1100\u0300", ["\u1100\u0300"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u0300", ["\u1100\u0308\u0300"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1100\u093C", ["\u1100\u093C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1100\u0308\u093C", ["\u1100\u0308\u093C"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1100\u0900", ["\u1100\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1100\u0308\u0900", ["\u1100\u0308\u0900"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u094D", ["\u1100\u094D"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u0308\u094D", ["\u1100\u0308\u094D"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1100\u200D", ["\u1100\u200D"] # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -448,8 +430,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1160\u0308\n", ["\u1160\u0308", '\n'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u1160\u0001", ['\u1160', '\u0001'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u0001", ["\u1160\u0308", '\u0001'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u1160\u034F", ["\u1160\u034F"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u1160\u0308\u034F", ["\u1160\u0308\u034F"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1160\u200C", ["\u1160\u200C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u1160\u0308\u200C", ["\u1160\u0308\u200C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u1160\u{1F1E6}", ['\u1160', '\u{1F1E6}'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u{1F1E6}", ["\u1160\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u1160\u0600", ['\u1160', '\u0600'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -466,8 +448,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1160\u0308\uAC00", ["\u1160\u0308", '\uAC00'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u1160\uAC01", ['\u1160', '\uAC01'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\uAC01", ["\u1160\u0308", '\uAC01'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u1160\u0900", ["\u1160\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u1160\u0308\u0900", ["\u1160\u0308\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1160\u0903", ["\u1160\u0903"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u0903", ["\u1160\u0308\u0903"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u1160\u0904", ['\u1160', '\u0904'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -480,8 +460,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u1160\u0308\u231A", ["\u1160\u0308", '\u231A'] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u1160\u0300", ["\u1160\u0300"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u0300", ["\u1160\u0308\u0300"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1160\u093C", ["\u1160\u093C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u1160\u0308\u093C", ["\u1160\u0308\u093C"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1160\u0900", ["\u1160\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u1160\u0308\u0900", ["\u1160\u0308\u0900"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u094D", ["\u1160\u094D"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u0308\u094D", ["\u1160\u0308\u094D"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u1160\u200D", ["\u1160\u200D"] # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -496,8 +476,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u11A8\u0308\n", ["\u11A8\u0308", '\n'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u11A8\u0001", ['\u11A8', '\u0001'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u0001", ["\u11A8\u0308", '\u0001'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u11A8\u034F", ["\u11A8\u034F"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0308\u034F", ["\u11A8\u0308\u034F"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u11A8\u200C", ["\u11A8\u200C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u11A8\u0308\u200C", ["\u11A8\u0308\u200C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u11A8\u{1F1E6}", ['\u11A8', '\u{1F1E6}'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u{1F1E6}", ["\u11A8\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u11A8\u0600", ['\u11A8', '\u0600'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -514,8 +494,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u11A8\u0308\uAC00", ["\u11A8\u0308", '\uAC00'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u11A8\uAC01", ['\u11A8', '\uAC01'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\uAC01", ["\u11A8\u0308", '\uAC01'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0900", ["\u11A8\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0308\u0900", ["\u11A8\u0308\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u11A8\u0903", ["\u11A8\u0903"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u0903", ["\u11A8\u0308\u0903"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u11A8\u0904", ['\u11A8', '\u0904'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -528,8 +506,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u11A8\u0308\u231A", ["\u11A8\u0308", '\u231A'] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u11A8\u0300", ["\u11A8\u0300"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u0300", ["\u11A8\u0308\u0300"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u11A8\u093C", ["\u11A8\u093C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u11A8\u0308\u093C", ["\u11A8\u0308\u093C"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u11A8\u0900", ["\u11A8\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u11A8\u0308\u0900", ["\u11A8\u0308\u0900"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u094D", ["\u11A8\u094D"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u0308\u094D", ["\u11A8\u0308\u094D"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u11A8\u200D", ["\u11A8\u200D"] # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -544,8 +522,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC00\u0308\n", ["\uAC00\u0308", '\n'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\uAC00\u0001", ['\uAC00', '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u0001", ["\uAC00\u0308", '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\uAC00\u034F", ["\uAC00\u034F"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0308\u034F", ["\uAC00\u0308\u034F"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC00\u200C", ["\uAC00\u200C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC00\u0308\u200C", ["\uAC00\u0308\u200C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\uAC00\u{1F1E6}", ['\uAC00', '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u{1F1E6}", ["\uAC00\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC00\u0600", ['\uAC00', '\u0600'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -562,8 +540,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC00\u0308\uAC00", ["\uAC00\u0308", '\uAC00'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\uAC00\uAC01", ['\uAC00', '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\uAC01", ["\uAC00\u0308", '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0900", ["\uAC00\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0308\u0900", ["\uAC00\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC00\u0903", ["\uAC00\u0903"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u0903", ["\uAC00\u0308\u0903"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC00\u0904", ['\uAC00', '\u0904'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -576,8 +552,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC00\u0308\u231A", ["\uAC00\u0308", '\u231A'] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\uAC00\u0300", ["\uAC00\u0300"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u0300", ["\uAC00\u0308\u0300"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC00\u093C", ["\uAC00\u093C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC00\u0308\u093C", ["\uAC00\u0308\u093C"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC00\u0900", ["\uAC00\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC00\u0308\u0900", ["\uAC00\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u094D", ["\uAC00\u094D"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u0308\u094D", ["\uAC00\u0308\u094D"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC00\u200D", ["\uAC00\u200D"] # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -592,8 +568,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC01\u0308\n", ["\uAC01\u0308", '\n'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\uAC01\u0001", ['\uAC01', '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0001", ["\uAC01\u0308", '\u0001'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\uAC01\u034F", ["\uAC01\u034F"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0308\u034F", ["\uAC01\u0308\u034F"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC01\u200C", ["\uAC01\u200C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\uAC01\u0308\u200C", ["\uAC01\u0308\u200C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\uAC01\u{1F1E6}", ['\uAC01', '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u{1F1E6}", ["\uAC01\u0308", '\u{1F1E6}'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\uAC01\u0600", ['\uAC01', '\u0600'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -610,8 +586,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC01\u0308\uAC00", ["\uAC01\u0308", '\uAC00'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\uAC01\uAC01", ['\uAC01', '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\uAC01", ["\uAC01\u0308", '\uAC01'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0900", ["\uAC01\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0308\u0900", ["\uAC01\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC01\u0903", ["\uAC01\u0903"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0903", ["\uAC01\u0308\u0903"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\uAC01\u0904", ['\uAC01', '\u0904'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -624,62 +598,14 @@ describe "String#each_grapheme" do it_iterates_graphemes "\uAC01\u0308\u231A", ["\uAC01\u0308", '\u231A'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\uAC01\u0300", ["\uAC01\u0300"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0300", ["\uAC01\u0308\u0300"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC01\u093C", ["\uAC01\u093C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\uAC01\u0308\u093C", ["\uAC01\u0308\u093C"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC01\u0900", ["\uAC01\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\uAC01\u0308\u0900", ["\uAC01\u0308\u0900"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u094D", ["\uAC01\u094D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u094D", ["\uAC01\u0308\u094D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u200D", ["\uAC01\u200D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u200D", ["\uAC01\u0308\u200D"] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\uAC01\u0378", ['\uAC01', '\u0378'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\uAC01\u0308\u0378", ["\uAC01\u0308", '\u0378'] # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u0900 ", ['\u0900', ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308 ", ["\u0900\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u0900\r", ['\u0900', '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\r", ["\u0900\u0308", '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u0900\n", ['\u0900', '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\n", ["\u0900\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u0900\u0001", ['\u0900', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0001", ["\u0900\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0900\u034F", ["\u0900\u034F"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u034F", ["\u0900\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0900\u{1F1E6}", ['\u0900', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u{1F1E6}", ["\u0900\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u0900\u0600", ['\u0900', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0600", ["\u0900\u0308", '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u0900\u0A03", ["\u0900\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0A03", ["\u0900\u0308\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u0900\u1100", ['\u0900', '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u1100", ["\u0900\u0308", '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u0900\u1160", ['\u0900', '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u1160", ["\u0900\u0308", '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u0900\u11A8", ['\u0900', '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u11A8", ["\u0900\u0308", '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u0900\uAC00", ['\u0900', '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\uAC00", ["\u0900\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u0900\uAC01", ['\u0900', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\uAC01", ["\u0900\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0900\u0900", ["\u0900\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0900", ["\u0900\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0903", ["\u0900\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0903", ["\u0900\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0904", ['\u0900', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0904", ["\u0900\u0308", '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0D4E", ['\u0900', '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0D4E", ["\u0900\u0308", '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0900\u0915", ['\u0900', '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0915", ["\u0900\u0308", '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u0900\u231A", ['\u0900', '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u231A", ["\u0900\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u0900\u0300", ["\u0900\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0300", ["\u0900\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u093C", ["\u0900\u093C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u093C", ["\u0900\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u094D", ["\u0900\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u094D", ["\u0900\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u200D", ["\u0900\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u200D", ["\u0900\u0308\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0900\u0378", ['\u0900', '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u0900\u0308\u0378", ["\u0900\u0308", '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u0903 ", ['\u0903', ' '] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u0903\u0308 ", ["\u0903\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u0903\r", ['\u0903', '\r'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [5.0] (CR) ÷ [0.3] @@ -688,8 +614,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0903\u0308\n", ["\u0903\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0903\u0001", ['\u0903', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u0001", ["\u0903\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0903\u034F", ["\u0903\u034F"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0903\u0308\u034F", ["\u0903\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0903\u200C", ["\u0903\u200C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0903\u0308\u200C", ["\u0903\u0308\u200C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0903\u{1F1E6}", ['\u0903', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u{1F1E6}", ["\u0903\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0903\u0600", ['\u0903', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -706,8 +632,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0903\u0308\uAC00", ["\u0903\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0903\uAC01", ['\u0903', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\uAC01", ["\u0903\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0903\u0900", ["\u0903\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0903\u0308\u0900", ["\u0903\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0903\u0903", ["\u0903\u0903"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u0903", ["\u0903\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0903\u0904", ['\u0903', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -720,8 +644,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0903\u0308\u231A", ["\u0903\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0903\u0300", ["\u0903\u0300"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u0300", ["\u0903\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0903\u093C", ["\u0903\u093C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0903\u0308\u093C", ["\u0903\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0903\u0900", ["\u0903\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0903\u0308\u0900", ["\u0903\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u094D", ["\u0903\u094D"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u0308\u094D", ["\u0903\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0903\u200D", ["\u0903\u200D"] # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -736,8 +660,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0904\u0308\n", ["\u0904\u0308", '\n'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0904\u0001", ['\u0904', '\u0001'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u0001", ["\u0904\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0904\u034F", ["\u0904\u034F"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0904\u0308\u034F", ["\u0904\u0308\u034F"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0904\u200C", ["\u0904\u200C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0904\u0308\u200C", ["\u0904\u0308\u200C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0904\u{1F1E6}", ['\u0904', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u{1F1E6}", ["\u0904\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0904\u0600", ['\u0904', '\u0600'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -754,8 +678,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0904\u0308\uAC00", ["\u0904\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0904\uAC01", ['\u0904', '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\uAC01", ["\u0904\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0904\u0900", ["\u0904\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0904\u0308\u0900", ["\u0904\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0904\u0903", ["\u0904\u0903"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u0903", ["\u0904\u0308\u0903"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0904\u0904", ['\u0904', '\u0904'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -768,8 +690,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0904\u0308\u231A", ["\u0904\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0904\u0300", ["\u0904\u0300"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u0300", ["\u0904\u0308\u0300"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0904\u093C", ["\u0904\u093C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0904\u0308\u093C", ["\u0904\u0308\u093C"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0904\u0900", ["\u0904\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0904\u0308\u0900", ["\u0904\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u094D", ["\u0904\u094D"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u0308\u094D", ["\u0904\u0308\u094D"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0904\u200D", ["\u0904\u200D"] # ÷ [0.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -784,8 +706,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0D4E\u0308\n", ["\u0D4E\u0308", '\n'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0001", ['\u0D4E', '\u0001'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u0001", ["\u0D4E\u0308", '\u0001'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u034F", ["\u0D4E\u034F"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0308\u034F", ["\u0D4E\u0308\u034F"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u200C", ["\u0D4E\u200C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u0308\u200C", ["\u0D4E\u0308\u200C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0D4E\u{1F1E6}", ["\u0D4E\u{1F1E6}"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u{1F1E6}", ["\u0D4E\u0308", '\u{1F1E6}'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0600", ["\u0D4E\u0600"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -802,8 +724,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0D4E\u0308\uAC00", ["\u0D4E\u0308", '\uAC00'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0D4E\uAC01", ["\u0D4E\uAC01"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\uAC01", ["\u0D4E\u0308", '\uAC01'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0900", ["\u0D4E\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0308\u0900", ["\u0D4E\u0308\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0903", ["\u0D4E\u0903"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u0903", ["\u0D4E\u0308\u0903"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0904", ["\u0D4E\u0904"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.2] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -816,8 +736,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0D4E\u0308\u231A", ["\u0D4E\u0308", '\u231A'] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0300", ["\u0D4E\u0300"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u0300", ["\u0D4E\u0308\u0300"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u093C", ["\u0D4E\u093C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0D4E\u0308\u093C", ["\u0D4E\u0308\u093C"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u0900", ["\u0D4E\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0D4E\u0308\u0900", ["\u0D4E\u0308\u0900"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u094D", ["\u0D4E\u094D"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u0308\u094D", ["\u0D4E\u0308\u094D"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0D4E\u200D", ["\u0D4E\u200D"] # ÷ [0.2] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -832,8 +752,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0915\u0308\n", ["\u0915\u0308", '\n'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0915\u0001", ['\u0915', '\u0001'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u0001", ["\u0915\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0915\u034F", ["\u0915\u034F"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0915\u0308\u034F", ["\u0915\u0308\u034F"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0915\u200C", ["\u0915\u200C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0915\u0308\u200C", ["\u0915\u0308\u200C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0915\u{1F1E6}", ['\u0915', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u{1F1E6}", ["\u0915\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0915\u0600", ['\u0915', '\u0600'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -850,8 +770,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0915\u0308\uAC00", ["\u0915\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0915\uAC01", ['\u0915', '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\uAC01", ["\u0915\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0915\u0900", ["\u0915\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0915\u0308\u0900", ["\u0915\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0915\u0903", ["\u0915\u0903"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u0903", ["\u0915\u0308\u0903"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0915\u0904", ['\u0915', '\u0904'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -864,8 +782,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0915\u0308\u231A", ["\u0915\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0915\u0300", ["\u0915\u0300"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u0300", ["\u0915\u0308\u0300"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0915\u093C", ["\u0915\u093C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0915\u0308\u093C", ["\u0915\u0308\u093C"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0915\u0900", ["\u0915\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0915\u0308\u0900", ["\u0915\u0308\u0900"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u094D", ["\u0915\u094D"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u0308\u094D", ["\u0915\u0308\u094D"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0915\u200D", ["\u0915\u200D"] # ÷ [0.2] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -880,8 +798,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u231A\u0308\n", ["\u231A\u0308", '\n'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u231A\u0001", ['\u231A', '\u0001'] # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u0001", ["\u231A\u0308", '\u0001'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u231A\u034F", ["\u231A\u034F"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u231A\u0308\u034F", ["\u231A\u0308\u034F"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u231A\u200C", ["\u231A\u200C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u231A\u0308\u200C", ["\u231A\u0308\u200C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u231A\u{1F1E6}", ['\u231A', '\u{1F1E6}'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u{1F1E6}", ["\u231A\u0308", '\u{1F1E6}'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u231A\u0600", ['\u231A', '\u0600'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -898,8 +816,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u231A\u0308\uAC00", ["\u231A\u0308", '\uAC00'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u231A\uAC01", ['\u231A', '\uAC01'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\uAC01", ["\u231A\u0308", '\uAC01'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u231A\u0900", ["\u231A\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u231A\u0308\u0900", ["\u231A\u0308\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u231A\u0903", ["\u231A\u0903"] # ÷ [0.2] WATCH (ExtPict) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u0903", ["\u231A\u0308\u0903"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u231A\u0904", ['\u231A', '\u0904'] # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -912,8 +828,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u231A\u0308\u231A", ["\u231A\u0308", '\u231A'] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u231A\u0300", ["\u231A\u0300"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u0300", ["\u231A\u0308\u0300"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u231A\u093C", ["\u231A\u093C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u231A\u0308\u093C", ["\u231A\u0308\u093C"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u231A\u0900", ["\u231A\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u231A\u0308\u0900", ["\u231A\u0308\u0900"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u094D", ["\u231A\u094D"] # ÷ [0.2] WATCH (ExtPict) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u0308\u094D", ["\u231A\u0308\u094D"] # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u231A\u200D", ["\u231A\u200D"] # ÷ [0.2] WATCH (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -928,8 +844,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0300\u0308\n", ["\u0300\u0308", '\n'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0300\u0001", ['\u0300', '\u0001'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0001", ["\u0300\u0308", '\u0001'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0300\u034F", ["\u0300\u034F"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0300\u0308\u034F", ["\u0300\u0308\u034F"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0300\u200C", ["\u0300\u200C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0300\u0308\u200C", ["\u0300\u0308\u200C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0300\u{1F1E6}", ['\u0300', '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u{1F1E6}", ["\u0300\u0308", '\u{1F1E6}'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0300\u0600", ['\u0300', '\u0600'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -946,8 +862,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0300\u0308\uAC00", ["\u0300\u0308", '\uAC00'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0300\uAC01", ['\u0300', '\uAC01'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\uAC01", ["\u0300\u0308", '\uAC01'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0300\u0900", ["\u0300\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0300\u0308\u0900", ["\u0300\u0308\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0300\u0903", ["\u0300\u0903"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0903", ["\u0300\u0308\u0903"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0300\u0904", ['\u0300', '\u0904'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -960,62 +874,60 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0300\u0308\u231A", ["\u0300\u0308", '\u231A'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0300\u0300", ["\u0300\u0300"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0300", ["\u0300\u0308\u0300"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0300\u093C", ["\u0300\u093C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0300\u0308\u093C", ["\u0300\u0308\u093C"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0300\u0900", ["\u0300\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0300\u0308\u0900", ["\u0300\u0308\u0900"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u094D", ["\u0300\u094D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u094D", ["\u0300\u0308\u094D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u200D", ["\u0300\u200D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u200D", ["\u0300\u0308\u200D"] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0300\u0378", ['\u0300', '\u0378'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u0300\u0308\u0378", ["\u0300\u0308", '\u0378'] # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u093C ", ['\u093C', ' '] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308 ", ["\u093C\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] - it_iterates_graphemes "\u093C\r", ['\u093C', '\r'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\r", ["\u093C\u0308", '\r'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] - it_iterates_graphemes "\u093C\n", ['\u093C', '\n'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\n", ["\u093C\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] - it_iterates_graphemes "\u093C\u0001", ['\u093C', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0001", ["\u093C\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u093C\u034F", ["\u093C\u034F"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u034F", ["\u093C\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u093C\u{1F1E6}", ['\u093C', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u{1F1E6}", ["\u093C\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] - it_iterates_graphemes "\u093C\u0600", ['\u093C', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0600", ["\u093C\u0308", '\u0600'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] - it_iterates_graphemes "\u093C\u0A03", ["\u093C\u0A03"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0A03", ["\u093C\u0308\u0A03"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] - it_iterates_graphemes "\u093C\u1100", ['\u093C', '\u1100'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u1100", ["\u093C\u0308", '\u1100'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] - it_iterates_graphemes "\u093C\u1160", ['\u093C', '\u1160'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u1160", ["\u093C\u0308", '\u1160'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] - it_iterates_graphemes "\u093C\u11A8", ['\u093C', '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u11A8", ["\u093C\u0308", '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] - it_iterates_graphemes "\u093C\uAC00", ['\u093C', '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\uAC00", ["\u093C\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] - it_iterates_graphemes "\u093C\uAC01", ['\u093C', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\uAC01", ["\u093C\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u093C\u0900", ["\u093C\u0900"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0900", ["\u093C\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0903", ["\u093C\u0903"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0903", ["\u093C\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0904", ['\u093C', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0904", ["\u093C\u0308", '\u0904'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0D4E", ['\u093C', '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0D4E", ["\u093C\u0308", '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u093C\u0915", ['\u093C', '\u0915'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0915", ["\u093C\u0308", '\u0915'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] - it_iterates_graphemes "\u093C\u231A", ['\u093C', '\u231A'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u231A", ["\u093C\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u093C\u0300", ["\u093C\u0300"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0300", ["\u093C\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u093C", ["\u093C\u093C"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u093C", ["\u093C\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u094D", ["\u093C\u094D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u094D", ["\u093C\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u200D", ["\u093C\u200D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u200D", ["\u093C\u0308\u200D"] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u093C\u0378", ['\u093C', '\u0378'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] - it_iterates_graphemes "\u093C\u0308\u0378", ["\u093C\u0308", '\u0378'] # ÷ [0.2] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u0900 ", ['\u0900', ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308 ", ["\u0900\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] + it_iterates_graphemes "\u0900\r", ['\u0900', '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\r", ["\u0900\u0308", '\r'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] + it_iterates_graphemes "\u0900\n", ['\u0900', '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\n", ["\u0900\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] + it_iterates_graphemes "\u0900\u0001", ['\u0900', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0001", ["\u0900\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] + it_iterates_graphemes "\u0900\u200C", ["\u0900\u200C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u200C", ["\u0900\u0308\u200C"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0900\u{1F1E6}", ['\u0900', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u{1F1E6}", ["\u0900\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] + it_iterates_graphemes "\u0900\u0600", ['\u0900', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0600", ["\u0900\u0308", '\u0600'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] + it_iterates_graphemes "\u0900\u0A03", ["\u0900\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0A03", ["\u0900\u0308\u0A03"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] GURMUKHI SIGN VISARGA (SpacingMark) ÷ [0.3] + it_iterates_graphemes "\u0900\u1100", ['\u0900', '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u1100", ["\u0900\u0308", '\u1100'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] + it_iterates_graphemes "\u0900\u1160", ['\u0900', '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u1160", ["\u0900\u0308", '\u1160'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] + it_iterates_graphemes "\u0900\u11A8", ['\u0900', '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u11A8", ["\u0900\u0308", '\u11A8'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] + it_iterates_graphemes "\u0900\uAC00", ['\u0900', '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\uAC00", ["\u0900\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] + it_iterates_graphemes "\u0900\uAC01", ['\u0900', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\uAC01", ["\u0900\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] + it_iterates_graphemes "\u0900\u0903", ["\u0900\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0903", ["\u0900\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0904", ['\u0900', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0904", ["\u0900\u0308", '\u0904'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0D4E", ['\u0900', '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0D4E", ["\u0900\u0308", '\u0D4E'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] MALAYALAM LETTER DOT REPH (Prepend_ConjunctLinkingScripts) ÷ [0.3] + it_iterates_graphemes "\u0900\u0915", ['\u0900', '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0915", ["\u0900\u0308", '\u0915'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER KA (ConjunctLinkingScripts_LinkingConsonant) ÷ [0.3] + it_iterates_graphemes "\u0900\u231A", ['\u0900', '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u231A", ["\u0900\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u0900\u0300", ["\u0900\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0300", ["\u0900\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0900", ["\u0900\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0900", ["\u0900\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u094D", ["\u0900\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u094D", ["\u0900\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u200D", ["\u0900\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u200D", ["\u0900\u0308\u200D"] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0900\u0378", ['\u0900', '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] + it_iterates_graphemes "\u0900\u0308\u0378", ["\u0900\u0308", '\u0378'] # ÷ [0.2] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] (Other) ÷ [0.3] it_iterates_graphemes "\u094D ", ['\u094D', ' '] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u094D\u0308 ", ["\u094D\u0308", ' '] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] it_iterates_graphemes "\u094D\r", ['\u094D', '\r'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [5.0] (CR) ÷ [0.3] @@ -1024,8 +936,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u094D\u0308\n", ["\u094D\u0308", '\n'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u094D\u0001", ['\u094D', '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u0001", ["\u094D\u0308", '\u0001'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u094D\u034F", ["\u094D\u034F"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u094D\u0308\u034F", ["\u094D\u0308\u034F"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u094D\u200C", ["\u094D\u200C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u094D\u0308\u200C", ["\u094D\u0308\u200C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u094D\u{1F1E6}", ['\u094D', '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u{1F1E6}", ["\u094D\u0308", '\u{1F1E6}'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u094D\u0600", ['\u094D', '\u0600'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -1042,8 +954,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u094D\u0308\uAC00", ["\u094D\u0308", '\uAC00'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u094D\uAC01", ['\u094D', '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\uAC01", ["\u094D\u0308", '\uAC01'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u094D\u0900", ["\u094D\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u094D\u0308\u0900", ["\u094D\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u094D\u0903", ["\u094D\u0903"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u0903", ["\u094D\u0308\u0903"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u094D\u0904", ['\u094D', '\u0904'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -1056,8 +966,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u094D\u0308\u231A", ["\u094D\u0308", '\u231A'] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u094D\u0300", ["\u094D\u0300"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u0300", ["\u094D\u0308\u0300"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u094D\u093C", ["\u094D\u093C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u094D\u0308\u093C", ["\u094D\u0308\u093C"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u094D\u0900", ["\u094D\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u094D\u0308\u0900", ["\u094D\u0308\u0900"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u094D", ["\u094D\u094D"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u0308\u094D", ["\u094D\u0308\u094D"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u094D\u200D", ["\u094D\u200D"] # ÷ [0.2] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -1072,8 +982,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u200D\u0308\n", ["\u200D\u0308", '\n'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u200D\u0001", ['\u200D', '\u0001'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u0001", ["\u200D\u0308", '\u0001'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u200D\u034F", ["\u200D\u034F"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u200D\u0308\u034F", ["\u200D\u0308\u034F"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200D\u200C", ["\u200D\u200C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u200D\u0308\u200C", ["\u200D\u0308\u200C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u200D\u{1F1E6}", ['\u200D', '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u{1F1E6}", ["\u200D\u0308", '\u{1F1E6}'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u200D\u0600", ['\u200D', '\u0600'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -1090,8 +1000,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u200D\u0308\uAC00", ["\u200D\u0308", '\uAC00'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u200D\uAC01", ['\u200D', '\uAC01'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\uAC01", ["\u200D\u0308", '\uAC01'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u200D\u0900", ["\u200D\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u200D\u0308\u0900", ["\u200D\u0308\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u200D\u0903", ["\u200D\u0903"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u0903", ["\u200D\u0308\u0903"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u200D\u0904", ['\u200D', '\u0904'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -1104,8 +1012,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u200D\u0308\u231A", ["\u200D\u0308", '\u231A'] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u200D\u0300", ["\u200D\u0300"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u0300", ["\u200D\u0308\u0300"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u200D\u093C", ["\u200D\u093C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u200D\u0308\u093C", ["\u200D\u0308\u093C"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200D\u0900", ["\u200D\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u200D\u0308\u0900", ["\u200D\u0308\u0900"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u094D", ["\u200D\u094D"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u0308\u094D", ["\u200D\u0308\u094D"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u200D\u200D", ["\u200D\u200D"] # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -1120,8 +1028,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0378\u0308\n", ["\u0378\u0308", '\n'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (LF) ÷ [0.3] it_iterates_graphemes "\u0378\u0001", ['\u0378', '\u0001'] # ÷ [0.2] (Other) ÷ [5.0] (Control) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u0001", ["\u0378\u0308", '\u0001'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] (Control) ÷ [0.3] - it_iterates_graphemes "\u0378\u034F", ["\u0378\u034F"] # ÷ [0.2] (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] - it_iterates_graphemes "\u0378\u0308\u034F", ["\u0378\u0308\u034F"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0378\u200C", ["\u0378\u200C"] # ÷ [0.2] (Other) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] + it_iterates_graphemes "\u0378\u0308\u200C", ["\u0378\u0308\u200C"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH NON-JOINER (Extend) ÷ [0.3] it_iterates_graphemes "\u0378\u{1F1E6}", ['\u0378', '\u{1F1E6}'] # ÷ [0.2] (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u{1F1E6}", ["\u0378\u0308", '\u{1F1E6}'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] it_iterates_graphemes "\u0378\u0600", ['\u0378', '\u0600'] # ÷ [0.2] (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] @@ -1138,8 +1046,6 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0378\u0308\uAC00", ["\u0378\u0308", '\uAC00'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] it_iterates_graphemes "\u0378\uAC01", ['\u0378', '\uAC01'] # ÷ [0.2] (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\uAC01", ["\u0378\u0308", '\uAC01'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] - it_iterates_graphemes "\u0378\u0900", ["\u0378\u0900"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] - it_iterates_graphemes "\u0378\u0308\u0900", ["\u0378\u0308\u0900"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0378\u0903", ["\u0378\u0903"] # ÷ [0.2] (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u0903", ["\u0378\u0308\u0903"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [0.3] it_iterates_graphemes "\u0378\u0904", ['\u0378', '\u0904'] # ÷ [0.2] (Other) ÷ [999.0] DEVANAGARI LETTER SHORT A (ConjunctLinkingScripts) ÷ [0.3] @@ -1152,8 +1058,8 @@ describe "String#each_grapheme" do it_iterates_graphemes "\u0378\u0308\u231A", ["\u0378\u0308", '\u231A'] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] it_iterates_graphemes "\u0378\u0300", ["\u0378\u0300"] # ÷ [0.2] (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u0300", ["\u0378\u0308\u0300"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0378\u093C", ["\u0378\u093C"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] - it_iterates_graphemes "\u0378\u0308\u093C", ["\u0378\u0308\u093C"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN NUKTA (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0378\u0900", ["\u0378\u0900"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] + it_iterates_graphemes "\u0378\u0308\u0900", ["\u0378\u0308\u0900"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN INVERTED CANDRABINDU (Extend_ConjunctLinkingScripts_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u094D", ["\u0378\u094D"] # ÷ [0.2] (Other) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u0308\u094D", ["\u0378\u0308\u094D"] # ÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] DEVANAGARI SIGN VIRAMA (Extend_ConjunctLinkingScripts_ConjunctLinker_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u0378\u200D", ["\u0378\u200D"] # ÷ [0.2] (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] @@ -1176,10 +1082,10 @@ describe "String#each_grapheme" do it_iterates_graphemes "a\u0308b", ["a\u0308", 'b'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] it_iterates_graphemes "a\u0903b", ["a\u0903", 'b'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark_ConjunctLinkingScripts) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] it_iterates_graphemes "a\u0600b", ['a', "\u0600b"] # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) × [9.2] LATIN SMALL LETTER B (Other) ÷ [0.3] - it_iterates_graphemes "\u{1F476}\u{1F3FF}\u{1F476}", ["\u{1F476}\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3] - it_iterates_graphemes "a\u{1F3FF}\u{1F476}", ["a\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3] - it_iterates_graphemes "a\u{1F3FF}\u{1F476}\u200D\u{1F6D1}", ["a\u{1F3FF}", "\u{1F476}\u200D\u{1F6D1}"] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] - it_iterates_graphemes "\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}", ["\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}"] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [0.3] + it_iterates_graphemes "\u{1F476}\u{1F3FF}\u{1F476}", ["\u{1F476}\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) ÷ [0.3] + it_iterates_graphemes "a\u{1F3FF}\u{1F476}", ["a\u{1F3FF}", '\u{1F476}'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) ÷ [0.3] + it_iterates_graphemes "a\u{1F3FF}\u{1F476}\u200D\u{1F6D1}", ["a\u{1F3FF}", "\u{1F476}\u200D\u{1F6D1}"] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [999.0] BABY (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] + it_iterates_graphemes "\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}", ["\u{1F476}\u{1F3FF}\u0308\u200D\u{1F476}\u{1F3FF}"] # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend_ExtCccZwj) ÷ [0.3] it_iterates_graphemes "\u{1F6D1}\u200D\u{1F6D1}", ["\u{1F6D1}\u200D\u{1F6D1}"] # ÷ [0.2] OCTAGONAL SIGN (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] it_iterates_graphemes "a\u200D\u{1F6D1}", ["a\u200D", '\u{1F6D1}'] # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] it_iterates_graphemes "\u2701\u200D\u2701", ["\u2701\u200D\u2701"] # ÷ [0.2] UPPER BLADE SCISSORS (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] UPPER BLADE SCISSORS (Other) ÷ [0.3] diff --git a/src/string/grapheme/properties.cr b/src/string/grapheme/properties.cr index 65b51fba0935..4d87254b7600 100644 --- a/src/string/grapheme/properties.cr +++ b/src/string/grapheme/properties.cr @@ -58,9 +58,9 @@ struct String::Grapheme # ranges in this slice are numerically sorted. # # These ranges were taken from - # http://www.unicode.org/Public/15.1.0/ucd/auxiliary/GraphemeBreakProperty.txt + # http://www.unicode.org/Public/16.0.0/ucd/auxiliary/GraphemeBreakProperty.txt # as well as - # http://www.unicode.org/Public/15.1.0/ucd/emoji/emoji-data.txt + # http://www.unicode.org/Public/16.0.0/ucd/emoji/emoji-data.txt # ("Extended_Pictographic" only). See # https://www.unicode.org/license.html for the Unicode license agreement. @@codepoints : Array(Tuple(Int32, Int32, Property))? @@ -68,7 +68,7 @@ struct String::Grapheme # :nodoc: protected def self.codepoints @@codepoints ||= begin - data = Array(Tuple(Int32, Int32, Property)).new(1447) + data = Array(Tuple(Int32, Int32, Property)).new(1452) put(data, {0x0000, 0x0009, Property::Control}) put(data, {0x000A, 0x000A, Property::LF}) put(data, {0x000B, 0x000C, Property::Control}) @@ -105,7 +105,7 @@ struct String::Grapheme put(data, {0x0829, 0x082D, Property::Extend}) put(data, {0x0859, 0x085B, Property::Extend}) put(data, {0x0890, 0x0891, Property::Prepend}) - put(data, {0x0898, 0x089F, Property::Extend}) + put(data, {0x0897, 0x089F, Property::Extend}) put(data, {0x08CA, 0x08E1, Property::Extend}) put(data, {0x08E2, 0x08E2, Property::Prepend}) put(data, {0x08E3, 0x0902, Property::Extend}) @@ -187,14 +187,12 @@ struct String::Grapheme put(data, {0x0C82, 0x0C83, Property::SpacingMark}) put(data, {0x0CBC, 0x0CBC, Property::Extend}) put(data, {0x0CBE, 0x0CBE, Property::SpacingMark}) - put(data, {0x0CBF, 0x0CBF, Property::Extend}) - put(data, {0x0CC0, 0x0CC1, Property::SpacingMark}) + put(data, {0x0CBF, 0x0CC0, Property::Extend}) + put(data, {0x0CC1, 0x0CC1, Property::SpacingMark}) put(data, {0x0CC2, 0x0CC2, Property::Extend}) put(data, {0x0CC3, 0x0CC4, Property::SpacingMark}) - put(data, {0x0CC6, 0x0CC6, Property::Extend}) - put(data, {0x0CC7, 0x0CC8, Property::SpacingMark}) - put(data, {0x0CCA, 0x0CCB, Property::SpacingMark}) - put(data, {0x0CCC, 0x0CCD, Property::Extend}) + put(data, {0x0CC6, 0x0CC8, Property::Extend}) + put(data, {0x0CCA, 0x0CCD, Property::Extend}) put(data, {0x0CD5, 0x0CD6, Property::Extend}) put(data, {0x0CE2, 0x0CE3, Property::Extend}) put(data, {0x0CF3, 0x0CF3, Property::SpacingMark}) @@ -259,10 +257,8 @@ struct String::Grapheme put(data, {0x1160, 0x11A7, Property::V}) put(data, {0x11A8, 0x11FF, Property::T}) put(data, {0x135D, 0x135F, Property::Extend}) - put(data, {0x1712, 0x1714, Property::Extend}) - put(data, {0x1715, 0x1715, Property::SpacingMark}) - put(data, {0x1732, 0x1733, Property::Extend}) - put(data, {0x1734, 0x1734, Property::SpacingMark}) + put(data, {0x1712, 0x1715, Property::Extend}) + put(data, {0x1732, 0x1734, Property::Extend}) put(data, {0x1752, 0x1753, Property::Extend}) put(data, {0x1772, 0x1773, Property::Extend}) put(data, {0x17B4, 0x17B5, Property::Extend}) @@ -302,29 +298,23 @@ struct String::Grapheme put(data, {0x1AB0, 0x1ACE, Property::Extend}) put(data, {0x1B00, 0x1B03, Property::Extend}) put(data, {0x1B04, 0x1B04, Property::SpacingMark}) - put(data, {0x1B34, 0x1B3A, Property::Extend}) - put(data, {0x1B3B, 0x1B3B, Property::SpacingMark}) - put(data, {0x1B3C, 0x1B3C, Property::Extend}) - put(data, {0x1B3D, 0x1B41, Property::SpacingMark}) - put(data, {0x1B42, 0x1B42, Property::Extend}) - put(data, {0x1B43, 0x1B44, Property::SpacingMark}) + put(data, {0x1B34, 0x1B3D, Property::Extend}) + put(data, {0x1B3E, 0x1B41, Property::SpacingMark}) + put(data, {0x1B42, 0x1B44, Property::Extend}) put(data, {0x1B6B, 0x1B73, Property::Extend}) put(data, {0x1B80, 0x1B81, Property::Extend}) put(data, {0x1B82, 0x1B82, Property::SpacingMark}) put(data, {0x1BA1, 0x1BA1, Property::SpacingMark}) put(data, {0x1BA2, 0x1BA5, Property::Extend}) put(data, {0x1BA6, 0x1BA7, Property::SpacingMark}) - put(data, {0x1BA8, 0x1BA9, Property::Extend}) - put(data, {0x1BAA, 0x1BAA, Property::SpacingMark}) - put(data, {0x1BAB, 0x1BAD, Property::Extend}) + put(data, {0x1BA8, 0x1BAD, Property::Extend}) put(data, {0x1BE6, 0x1BE6, Property::Extend}) put(data, {0x1BE7, 0x1BE7, Property::SpacingMark}) put(data, {0x1BE8, 0x1BE9, Property::Extend}) put(data, {0x1BEA, 0x1BEC, Property::SpacingMark}) put(data, {0x1BED, 0x1BED, Property::Extend}) put(data, {0x1BEE, 0x1BEE, Property::SpacingMark}) - put(data, {0x1BEF, 0x1BF1, Property::Extend}) - put(data, {0x1BF2, 0x1BF3, Property::SpacingMark}) + put(data, {0x1BEF, 0x1BF3, Property::Extend}) put(data, {0x1C24, 0x1C2B, Property::SpacingMark}) put(data, {0x1C2C, 0x1C33, Property::Extend}) put(data, {0x1C34, 0x1C35, Property::SpacingMark}) @@ -416,7 +406,8 @@ struct String::Grapheme put(data, {0xA8FF, 0xA8FF, Property::Extend}) put(data, {0xA926, 0xA92D, Property::Extend}) put(data, {0xA947, 0xA951, Property::Extend}) - put(data, {0xA952, 0xA953, Property::SpacingMark}) + put(data, {0xA952, 0xA952, Property::SpacingMark}) + put(data, {0xA953, 0xA953, Property::Extend}) put(data, {0xA960, 0xA97C, Property::L}) put(data, {0xA980, 0xA982, Property::Extend}) put(data, {0xA983, 0xA983, Property::SpacingMark}) @@ -425,7 +416,8 @@ struct String::Grapheme put(data, {0xA9B6, 0xA9B9, Property::Extend}) put(data, {0xA9BA, 0xA9BB, Property::SpacingMark}) put(data, {0xA9BC, 0xA9BD, Property::Extend}) - put(data, {0xA9BE, 0xA9C0, Property::SpacingMark}) + put(data, {0xA9BE, 0xA9BF, Property::SpacingMark}) + put(data, {0xA9C0, 0xA9C0, Property::Extend}) put(data, {0xA9E5, 0xA9E5, Property::Extend}) put(data, {0xAA29, 0xAA2E, Property::Extend}) put(data, {0xAA2F, 0xAA30, Property::SpacingMark}) @@ -1269,8 +1261,9 @@ struct String::Grapheme put(data, {0x10A3F, 0x10A3F, Property::Extend}) put(data, {0x10AE5, 0x10AE6, Property::Extend}) put(data, {0x10D24, 0x10D27, Property::Extend}) + put(data, {0x10D69, 0x10D6D, Property::Extend}) put(data, {0x10EAB, 0x10EAC, Property::Extend}) - put(data, {0x10EFD, 0x10EFF, Property::Extend}) + put(data, {0x10EFC, 0x10EFF, Property::Extend}) put(data, {0x10F46, 0x10F50, Property::Extend}) put(data, {0x10F82, 0x10F85, Property::Extend}) put(data, {0x11000, 0x11000, Property::SpacingMark}) @@ -1298,7 +1291,8 @@ struct String::Grapheme put(data, {0x11182, 0x11182, Property::SpacingMark}) put(data, {0x111B3, 0x111B5, Property::SpacingMark}) put(data, {0x111B6, 0x111BE, Property::Extend}) - put(data, {0x111BF, 0x111C0, Property::SpacingMark}) + put(data, {0x111BF, 0x111BF, Property::SpacingMark}) + put(data, {0x111C0, 0x111C0, Property::Extend}) put(data, {0x111C2, 0x111C3, Property::Prepend}) put(data, {0x111C9, 0x111CC, Property::Extend}) put(data, {0x111CE, 0x111CE, Property::SpacingMark}) @@ -1306,9 +1300,7 @@ struct String::Grapheme put(data, {0x1122C, 0x1122E, Property::SpacingMark}) put(data, {0x1122F, 0x11231, Property::Extend}) put(data, {0x11232, 0x11233, Property::SpacingMark}) - put(data, {0x11234, 0x11234, Property::Extend}) - put(data, {0x11235, 0x11235, Property::SpacingMark}) - put(data, {0x11236, 0x11237, Property::Extend}) + put(data, {0x11234, 0x11237, Property::Extend}) put(data, {0x1123E, 0x1123E, Property::Extend}) put(data, {0x11241, 0x11241, Property::Extend}) put(data, {0x112DF, 0x112DF, Property::Extend}) @@ -1322,11 +1314,24 @@ struct String::Grapheme put(data, {0x11340, 0x11340, Property::Extend}) put(data, {0x11341, 0x11344, Property::SpacingMark}) put(data, {0x11347, 0x11348, Property::SpacingMark}) - put(data, {0x1134B, 0x1134D, Property::SpacingMark}) + put(data, {0x1134B, 0x1134C, Property::SpacingMark}) + put(data, {0x1134D, 0x1134D, Property::Extend}) put(data, {0x11357, 0x11357, Property::Extend}) put(data, {0x11362, 0x11363, Property::SpacingMark}) put(data, {0x11366, 0x1136C, Property::Extend}) put(data, {0x11370, 0x11374, Property::Extend}) + put(data, {0x113B8, 0x113B8, Property::Extend}) + put(data, {0x113B9, 0x113BA, Property::SpacingMark}) + put(data, {0x113BB, 0x113C0, Property::Extend}) + put(data, {0x113C2, 0x113C2, Property::Extend}) + put(data, {0x113C5, 0x113C5, Property::Extend}) + put(data, {0x113C7, 0x113C9, Property::Extend}) + put(data, {0x113CA, 0x113CA, Property::SpacingMark}) + put(data, {0x113CC, 0x113CD, Property::SpacingMark}) + put(data, {0x113CE, 0x113D0, Property::Extend}) + put(data, {0x113D1, 0x113D1, Property::Prepend}) + put(data, {0x113D2, 0x113D2, Property::Extend}) + put(data, {0x113E1, 0x113E2, Property::Extend}) put(data, {0x11435, 0x11437, Property::SpacingMark}) put(data, {0x11438, 0x1143F, Property::Extend}) put(data, {0x11440, 0x11441, Property::SpacingMark}) @@ -1363,10 +1368,10 @@ struct String::Grapheme put(data, {0x116AC, 0x116AC, Property::SpacingMark}) put(data, {0x116AD, 0x116AD, Property::Extend}) put(data, {0x116AE, 0x116AF, Property::SpacingMark}) - put(data, {0x116B0, 0x116B5, Property::Extend}) - put(data, {0x116B6, 0x116B6, Property::SpacingMark}) - put(data, {0x116B7, 0x116B7, Property::Extend}) - put(data, {0x1171D, 0x1171F, Property::Extend}) + put(data, {0x116B0, 0x116B7, Property::Extend}) + put(data, {0x1171D, 0x1171D, Property::Extend}) + put(data, {0x1171E, 0x1171E, Property::SpacingMark}) + put(data, {0x1171F, 0x1171F, Property::Extend}) put(data, {0x11722, 0x11725, Property::Extend}) put(data, {0x11726, 0x11726, Property::SpacingMark}) put(data, {0x11727, 0x1172B, Property::Extend}) @@ -1377,9 +1382,7 @@ struct String::Grapheme put(data, {0x11930, 0x11930, Property::Extend}) put(data, {0x11931, 0x11935, Property::SpacingMark}) put(data, {0x11937, 0x11938, Property::SpacingMark}) - put(data, {0x1193B, 0x1193C, Property::Extend}) - put(data, {0x1193D, 0x1193D, Property::SpacingMark}) - put(data, {0x1193E, 0x1193E, Property::Extend}) + put(data, {0x1193B, 0x1193E, Property::Extend}) put(data, {0x1193F, 0x1193F, Property::Prepend}) put(data, {0x11940, 0x11940, Property::SpacingMark}) put(data, {0x11941, 0x11941, Property::Prepend}) @@ -1436,28 +1439,29 @@ struct String::Grapheme put(data, {0x11F34, 0x11F35, Property::SpacingMark}) put(data, {0x11F36, 0x11F3A, Property::Extend}) put(data, {0x11F3E, 0x11F3F, Property::SpacingMark}) - put(data, {0x11F40, 0x11F40, Property::Extend}) - put(data, {0x11F41, 0x11F41, Property::SpacingMark}) - put(data, {0x11F42, 0x11F42, Property::Extend}) + put(data, {0x11F40, 0x11F42, Property::Extend}) + put(data, {0x11F5A, 0x11F5A, Property::Extend}) put(data, {0x13430, 0x1343F, Property::Control}) put(data, {0x13440, 0x13440, Property::Extend}) put(data, {0x13447, 0x13455, Property::Extend}) + put(data, {0x1611E, 0x16129, Property::Extend}) + put(data, {0x1612A, 0x1612C, Property::SpacingMark}) + put(data, {0x1612D, 0x1612F, Property::Extend}) put(data, {0x16AF0, 0x16AF4, Property::Extend}) put(data, {0x16B30, 0x16B36, Property::Extend}) + put(data, {0x16D63, 0x16D63, Property::V}) + put(data, {0x16D67, 0x16D6A, Property::V}) put(data, {0x16F4F, 0x16F4F, Property::Extend}) put(data, {0x16F51, 0x16F87, Property::SpacingMark}) put(data, {0x16F8F, 0x16F92, Property::Extend}) put(data, {0x16FE4, 0x16FE4, Property::Extend}) - put(data, {0x16FF0, 0x16FF1, Property::SpacingMark}) + put(data, {0x16FF0, 0x16FF1, Property::Extend}) put(data, {0x1BC9D, 0x1BC9E, Property::Extend}) put(data, {0x1BCA0, 0x1BCA3, Property::Control}) put(data, {0x1CF00, 0x1CF2D, Property::Extend}) put(data, {0x1CF30, 0x1CF46, Property::Extend}) - put(data, {0x1D165, 0x1D165, Property::Extend}) - put(data, {0x1D166, 0x1D166, Property::SpacingMark}) - put(data, {0x1D167, 0x1D169, Property::Extend}) - put(data, {0x1D16D, 0x1D16D, Property::SpacingMark}) - put(data, {0x1D16E, 0x1D172, Property::Extend}) + put(data, {0x1D165, 0x1D169, Property::Extend}) + put(data, {0x1D16D, 0x1D172, Property::Extend}) put(data, {0x1D173, 0x1D17A, Property::Control}) put(data, {0x1D17B, 0x1D182, Property::Extend}) put(data, {0x1D185, 0x1D18B, Property::Extend}) @@ -1479,6 +1483,7 @@ struct String::Grapheme put(data, {0x1E2AE, 0x1E2AE, Property::Extend}) put(data, {0x1E2EC, 0x1E2EF, Property::Extend}) put(data, {0x1E4EC, 0x1E4EF, Property::Extend}) + put(data, {0x1E5EE, 0x1E5EF, Property::Extend}) put(data, {0x1E8D0, 0x1E8D6, Property::Extend}) put(data, {0x1E944, 0x1E94A, Property::Extend}) put(data, {0x1F000, 0x1F0FF, Property::ExtendedPictographic}) diff --git a/src/unicode/data.cr b/src/unicode/data.cr index a02db251d0c8..ccb7d702e892 100644 --- a/src/unicode/data.cr +++ b/src/unicode/data.cr @@ -8,7 +8,7 @@ module Unicode # Most case conversions map a range to another range. # Here we store: {from, to, delta} private class_getter upcase_ranges : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(141) + data = Array({Int32, Int32, Int32}).new(144) put(data, 97, 122, -32) put(data, 181, 181, 743) put(data, 224, 246, -32) @@ -19,6 +19,7 @@ module Unicode put(data, 384, 384, 195) put(data, 405, 405, 97) put(data, 410, 410, 163) + put(data, 411, 411, 42561) put(data, 414, 414, 130) put(data, 447, 447, 56) put(data, 454, 454, -2) @@ -39,6 +40,7 @@ module Unicode put(data, 608, 608, -205) put(data, 609, 609, 42315) put(data, 611, 611, -207) + put(data, 612, 612, 42343) put(data, 613, 613, 42280) put(data, 614, 614, 42308) put(data, 616, 616, -209) @@ -147,6 +149,7 @@ module Unicode put(data, 66995, 67001, -39) put(data, 67003, 67004, -39) put(data, 68800, 68850, -64) + put(data, 68976, 68997, -32) put(data, 71872, 71903, -32) put(data, 93792, 93823, -32) put(data, 125218, 125251, -34) @@ -156,7 +159,7 @@ module Unicode # Most case conversions map a range to another range. # Here we store: {from, to, delta} private class_getter downcase_ranges : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(125) + data = Array({Int32, Int32, Int32}).new(128) put(data, 65, 90, 32) put(data, 192, 214, 32) put(data, 216, 222, 32) @@ -271,6 +274,8 @@ module Unicode put(data, 42948, 42948, -48) put(data, 42949, 42949, -42307) put(data, 42950, 42950, -35384) + put(data, 42955, 42955, -42343) + put(data, 42972, 42972, -42561) put(data, 65313, 65338, 32) put(data, 66560, 66599, 40) put(data, 66736, 66771, 40) @@ -279,6 +284,7 @@ module Unicode put(data, 66956, 66962, 39) put(data, 66964, 66965, 39) put(data, 68736, 68786, 64) + put(data, 68944, 68965, 32) put(data, 71840, 71871, 32) put(data, 93760, 93791, 32) put(data, 125184, 125217, 34) @@ -289,7 +295,7 @@ module Unicode # of uppercase/lowercase transformations # Here we store {from, to} private class_getter alternate_ranges : Array({Int32, Int32}) do - data = Array({Int32, Int32}).new(60) + data = Array({Int32, Int32}).new(62) put(data, 256, 303) put(data, 306, 311) put(data, 313, 328) @@ -326,6 +332,7 @@ module Unicode put(data, 1162, 1215) put(data, 1217, 1230) put(data, 1232, 1327) + put(data, 7305, 7306) put(data, 7680, 7829) put(data, 7840, 7935) put(data, 8579, 8580) @@ -347,8 +354,9 @@ module Unicode put(data, 42902, 42921) put(data, 42932, 42947) put(data, 42951, 42954) + put(data, 42956, 42957) put(data, 42960, 42961) - put(data, 42966, 42969) + put(data, 42966, 42971) put(data, 42997, 42998) data end @@ -363,7 +371,7 @@ module Unicode # The values are: 1..10, 11, 13, 15 private class_getter category_Lu : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(149) + data = Array({Int32, Int32, Int32}).new(152) put(data, 65, 90, 1) put(data, 192, 214, 1) put(data, 216, 222, 1) @@ -420,7 +428,8 @@ module Unicode put(data, 4256, 4293, 1) put(data, 4295, 4301, 6) put(data, 5024, 5109, 1) - put(data, 7312, 7354, 1) + put(data, 7305, 7312, 7) + put(data, 7313, 7354, 1) put(data, 7357, 7359, 1) put(data, 7680, 7828, 2) put(data, 7838, 7934, 2) @@ -469,8 +478,9 @@ module Unicode put(data, 42928, 42932, 1) put(data, 42934, 42948, 2) put(data, 42949, 42951, 1) - put(data, 42953, 42960, 7) - put(data, 42966, 42968, 2) + put(data, 42953, 42955, 2) + put(data, 42956, 42960, 4) + put(data, 42966, 42972, 2) put(data, 42997, 65313, 22316) put(data, 65314, 65338, 1) put(data, 66560, 66599, 1) @@ -480,6 +490,7 @@ module Unicode put(data, 66956, 66962, 1) put(data, 66964, 66965, 1) put(data, 68736, 68786, 1) + put(data, 68944, 68965, 1) put(data, 71840, 71871, 1) put(data, 93760, 93791, 1) put(data, 119808, 119833, 1) @@ -516,7 +527,7 @@ module Unicode data end private class_getter category_Ll : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(163) + data = Array({Int32, Int32, Int32}).new(166) put(data, 97, 122, 1) put(data, 181, 223, 42) put(data, 224, 246, 1) @@ -572,7 +583,8 @@ module Unicode put(data, 4349, 4351, 1) put(data, 5112, 5117, 1) put(data, 7296, 7304, 1) - put(data, 7424, 7467, 1) + put(data, 7306, 7424, 118) + put(data, 7425, 7467, 1) put(data, 7531, 7543, 1) put(data, 7545, 7578, 1) put(data, 7681, 7829, 2) @@ -631,7 +643,8 @@ module Unicode put(data, 42927, 42933, 6) put(data, 42935, 42947, 2) put(data, 42952, 42954, 2) - put(data, 42961, 42969, 2) + put(data, 42957, 42961, 4) + put(data, 42963, 42971, 2) put(data, 42998, 43002, 4) put(data, 43824, 43866, 1) put(data, 43872, 43880, 1) @@ -646,6 +659,7 @@ module Unicode put(data, 66995, 67001, 1) put(data, 67003, 67004, 1) put(data, 68800, 68850, 1) + put(data, 68976, 68997, 1) put(data, 71872, 71903, 1) put(data, 93792, 93823, 1) put(data, 119834, 119859, 1) @@ -694,7 +708,7 @@ module Unicode data end private class_getter category_Lm : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(54) + data = Array({Int32, Int32, Int32}).new(57) put(data, 688, 705, 1) put(data, 710, 721, 1) put(data, 736, 740, 1) @@ -739,7 +753,10 @@ module Unicode put(data, 67456, 67461, 1) put(data, 67463, 67504, 1) put(data, 67506, 67514, 1) + put(data, 68942, 68975, 33) put(data, 92992, 92995, 1) + put(data, 93504, 93506, 1) + put(data, 93547, 93548, 1) put(data, 94099, 94111, 1) put(data, 94176, 94177, 1) put(data, 94179, 110576, 16397) @@ -752,7 +769,7 @@ module Unicode data end private class_getter category_Lo : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(486) + data = Array({Int32, Int32, Int32}).new(502) put(data, 170, 186, 16) put(data, 443, 448, 5) put(data, 449, 451, 1) @@ -1052,6 +1069,7 @@ module Unicode put(data, 66640, 66717, 1) put(data, 66816, 66855, 1) put(data, 66864, 66915, 1) + put(data, 67008, 67059, 1) put(data, 67072, 67382, 1) put(data, 67392, 67413, 1) put(data, 67424, 67431, 1) @@ -1083,8 +1101,11 @@ module Unicode put(data, 68480, 68497, 1) put(data, 68608, 68680, 1) put(data, 68864, 68899, 1) - put(data, 69248, 69289, 1) + put(data, 68938, 68941, 1) + put(data, 68943, 69248, 305) + put(data, 69249, 69289, 1) put(data, 69296, 69297, 1) + put(data, 69314, 69316, 1) put(data, 69376, 69404, 1) put(data, 69415, 69424, 9) put(data, 69425, 69445, 1) @@ -1120,7 +1141,12 @@ module Unicode put(data, 70453, 70457, 1) put(data, 70461, 70480, 19) put(data, 70493, 70497, 1) - put(data, 70656, 70708, 1) + put(data, 70528, 70537, 1) + put(data, 70539, 70542, 3) + put(data, 70544, 70581, 1) + put(data, 70583, 70609, 26) + put(data, 70611, 70656, 45) + put(data, 70657, 70708, 1) put(data, 70727, 70730, 1) put(data, 70751, 70753, 1) put(data, 70784, 70831, 1) @@ -1150,6 +1176,7 @@ module Unicode put(data, 72284, 72329, 1) put(data, 72349, 72368, 19) put(data, 72369, 72440, 1) + put(data, 72640, 72672, 1) put(data, 72704, 72712, 1) put(data, 72714, 72750, 1) put(data, 72768, 72818, 50) @@ -1172,7 +1199,9 @@ module Unicode put(data, 77712, 77808, 1) put(data, 77824, 78895, 1) put(data, 78913, 78918, 1) + put(data, 78944, 82938, 1) put(data, 82944, 83526, 1) + put(data, 90368, 90397, 1) put(data, 92160, 92728, 1) put(data, 92736, 92766, 1) put(data, 92784, 92862, 1) @@ -1180,12 +1209,14 @@ module Unicode put(data, 92928, 92975, 1) put(data, 93027, 93047, 1) put(data, 93053, 93071, 1) + put(data, 93507, 93546, 1) put(data, 93952, 94026, 1) put(data, 94032, 94208, 176) put(data, 100343, 100352, 9) put(data, 100353, 101589, 1) - put(data, 101632, 101640, 1) - put(data, 110592, 110882, 1) + put(data, 101631, 101632, 1) + put(data, 101640, 110592, 8952) + put(data, 110593, 110882, 1) put(data, 110898, 110928, 30) put(data, 110929, 110930, 1) put(data, 110933, 110948, 15) @@ -1201,7 +1232,9 @@ module Unicode put(data, 123537, 123565, 1) put(data, 123584, 123627, 1) put(data, 124112, 124138, 1) - put(data, 124896, 124902, 1) + put(data, 124368, 124397, 1) + put(data, 124400, 124896, 496) + put(data, 124897, 124902, 1) put(data, 124904, 124907, 1) put(data, 124909, 124910, 1) put(data, 124912, 124926, 1) @@ -1242,7 +1275,7 @@ module Unicode data end private class_getter category_Mn : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(308) + data = Array({Int32, Int32, Int32}).new(315) put(data, 768, 879, 1) put(data, 1155, 1159, 1) put(data, 1425, 1469, 1) @@ -1266,7 +1299,7 @@ module Unicode put(data, 2085, 2087, 1) put(data, 2089, 2093, 1) put(data, 2137, 2139, 1) - put(data, 2200, 2207, 1) + put(data, 2199, 2207, 1) put(data, 2250, 2273, 1) put(data, 2275, 2306, 1) put(data, 2362, 2364, 2) @@ -1435,8 +1468,9 @@ module Unicode put(data, 68159, 68325, 166) put(data, 68326, 68900, 574) put(data, 68901, 68903, 1) + put(data, 68969, 68973, 1) put(data, 69291, 69292, 1) - put(data, 69373, 69375, 1) + put(data, 69372, 69375, 1) put(data, 69446, 69456, 1) put(data, 69506, 69509, 1) put(data, 69633, 69688, 55) @@ -1465,6 +1499,9 @@ module Unicode put(data, 70464, 70502, 38) put(data, 70503, 70508, 1) put(data, 70512, 70516, 1) + put(data, 70587, 70592, 1) + put(data, 70606, 70610, 2) + put(data, 70625, 70626, 1) put(data, 70712, 70719, 1) put(data, 70722, 70724, 1) put(data, 70726, 70750, 24) @@ -1482,8 +1519,8 @@ module Unicode put(data, 71341, 71344, 3) put(data, 71345, 71349, 1) put(data, 71351, 71453, 102) - put(data, 71454, 71455, 1) - put(data, 71458, 71461, 1) + put(data, 71455, 71458, 3) + put(data, 71459, 71461, 1) put(data, 71463, 71467, 1) put(data, 71727, 71735, 1) put(data, 71737, 71738, 1) @@ -1518,8 +1555,10 @@ module Unicode put(data, 73473, 73526, 53) put(data, 73527, 73530, 1) put(data, 73536, 73538, 2) - put(data, 78912, 78919, 7) - put(data, 78920, 78933, 1) + put(data, 73562, 78912, 5350) + put(data, 78919, 78933, 1) + put(data, 90398, 90409, 1) + put(data, 90413, 90415, 1) put(data, 92912, 92916, 1) put(data, 92976, 92982, 1) put(data, 94031, 94095, 64) @@ -1548,13 +1587,14 @@ module Unicode put(data, 123566, 123628, 62) put(data, 123629, 123631, 1) put(data, 124140, 124143, 1) + put(data, 124398, 124399, 1) put(data, 125136, 125142, 1) put(data, 125252, 125258, 1) put(data, 917760, 917999, 1) data end private class_getter category_Mc : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(158) + data = Array({Int32, Int32, Int32}).new(165) put(data, 2307, 2363, 56) put(data, 2366, 2368, 1) put(data, 2377, 2380, 1) @@ -1672,7 +1712,12 @@ module Unicode put(data, 70471, 70472, 1) put(data, 70475, 70477, 1) put(data, 70487, 70498, 11) - put(data, 70499, 70709, 210) + put(data, 70499, 70584, 85) + put(data, 70585, 70586, 1) + put(data, 70594, 70597, 3) + put(data, 70599, 70602, 1) + put(data, 70604, 70605, 1) + put(data, 70607, 70709, 102) put(data, 70710, 70711, 1) put(data, 70720, 70721, 1) put(data, 70725, 70832, 107) @@ -1687,9 +1732,10 @@ module Unicode put(data, 71227, 71228, 1) put(data, 71230, 71340, 110) put(data, 71342, 71343, 1) - put(data, 71350, 71456, 106) - put(data, 71457, 71462, 5) - put(data, 71724, 71726, 1) + put(data, 71350, 71454, 104) + put(data, 71456, 71457, 1) + put(data, 71462, 71724, 262) + put(data, 71725, 71726, 1) put(data, 71736, 71984, 248) put(data, 71985, 71989, 1) put(data, 71991, 71992, 1) @@ -1708,8 +1754,9 @@ module Unicode put(data, 73462, 73475, 13) put(data, 73524, 73525, 1) put(data, 73534, 73535, 1) - put(data, 73537, 94033, 20496) - put(data, 94034, 94087, 1) + put(data, 73537, 90410, 16873) + put(data, 90411, 90412, 1) + put(data, 94033, 94087, 1) put(data, 94192, 94193, 1) put(data, 119141, 119142, 1) put(data, 119149, 119154, 1) @@ -1725,7 +1772,7 @@ module Unicode data end private class_getter category_Nd : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(64) + data = Array({Int32, Int32, Int32}).new(71) put(data, 48, 57, 1) put(data, 1632, 1641, 1) put(data, 1776, 1785, 1) @@ -1765,6 +1812,7 @@ module Unicode put(data, 65296, 65305, 1) put(data, 66720, 66729, 1) put(data, 68912, 68921, 1) + put(data, 68928, 68937, 1) put(data, 69734, 69743, 1) put(data, 69872, 69881, 1) put(data, 69942, 69951, 1) @@ -1774,20 +1822,26 @@ module Unicode put(data, 70864, 70873, 1) put(data, 71248, 71257, 1) put(data, 71360, 71369, 1) + put(data, 71376, 71395, 1) put(data, 71472, 71481, 1) put(data, 71904, 71913, 1) put(data, 72016, 72025, 1) + put(data, 72688, 72697, 1) put(data, 72784, 72793, 1) put(data, 73040, 73049, 1) put(data, 73120, 73129, 1) put(data, 73552, 73561, 1) + put(data, 90416, 90425, 1) put(data, 92768, 92777, 1) put(data, 92864, 92873, 1) put(data, 93008, 93017, 1) + put(data, 93552, 93561, 1) + put(data, 118000, 118009, 1) put(data, 120782, 120831, 1) put(data, 123200, 123209, 1) put(data, 123632, 123641, 1) put(data, 124144, 124153, 1) + put(data, 124401, 124410, 1) put(data, 125264, 125273, 1) put(data, 130032, 130041, 1) data @@ -1951,7 +2005,7 @@ module Unicode # Most casefold conversions map a range to another range. # Here we store: {from, to, delta} private class_getter casefold_ranges : Array({Int32, Int32, Int32}) do - data = Array({Int32, Int32, Int32}).new(681) + data = Array({Int32, Int32, Int32}).new(687) put(data, 65, 90, 32) put(data, 181, 181, 775) put(data, 192, 214, 32) @@ -2276,6 +2330,7 @@ module Unicode put(data, 7302, 7302, -6204) put(data, 7303, 7303, -6180) put(data, 7304, 7304, 35267) + put(data, 7305, 7305, 1) put(data, 7312, 7354, -3008) put(data, 7357, 7359, -3008) put(data, 7680, 7680, 1) @@ -2617,9 +2672,13 @@ module Unicode put(data, 42950, 42950, -35384) put(data, 42951, 42951, 1) put(data, 42953, 42953, 1) + put(data, 42955, 42955, -42343) + put(data, 42956, 42956, 1) put(data, 42960, 42960, 1) put(data, 42966, 42966, 1) put(data, 42968, 42968, 1) + put(data, 42970, 42970, 1) + put(data, 42972, 42972, -42561) put(data, 42997, 42997, 1) put(data, 43888, 43967, -38864) put(data, 65313, 65338, 32) @@ -2630,6 +2689,7 @@ module Unicode put(data, 66956, 66962, 39) put(data, 66964, 66965, 39) put(data, 68736, 68786, 64) + put(data, 68944, 68965, 32) put(data, 71840, 71871, 32) put(data, 93760, 93791, 32) put(data, 125184, 125217, 34) @@ -2963,7 +3023,7 @@ module Unicode # guarantees that all class values are within `0..254`. # Here we store: {from, to, class} private class_getter canonical_combining_classes : Array({Int32, Int32, UInt8}) do - data = Array({Int32, Int32, UInt8}).new(392) + data = Array({Int32, Int32, UInt8}).new(399) put(data, 768, 788, 230_u8) put(data, 789, 789, 232_u8) put(data, 790, 793, 220_u8) @@ -3084,7 +3144,7 @@ module Unicode put(data, 2085, 2087, 230_u8) put(data, 2089, 2093, 230_u8) put(data, 2137, 2139, 220_u8) - put(data, 2200, 2200, 230_u8) + put(data, 2199, 2200, 230_u8) put(data, 2201, 2203, 220_u8) put(data, 2204, 2207, 230_u8) put(data, 2250, 2254, 230_u8) @@ -3273,6 +3333,7 @@ module Unicode put(data, 68325, 68325, 230_u8) put(data, 68326, 68326, 220_u8) put(data, 68900, 68903, 230_u8) + put(data, 68969, 68973, 230_u8) put(data, 69291, 69292, 230_u8) put(data, 69373, 69375, 220_u8) put(data, 69446, 69447, 220_u8) @@ -3302,6 +3363,9 @@ module Unicode put(data, 70477, 70477, 9_u8) put(data, 70502, 70508, 230_u8) put(data, 70512, 70516, 230_u8) + put(data, 70606, 70606, 9_u8) + put(data, 70607, 70607, 9_u8) + put(data, 70608, 70608, 9_u8) put(data, 70722, 70722, 9_u8) put(data, 70726, 70726, 7_u8) put(data, 70750, 70750, 230_u8) @@ -3328,6 +3392,7 @@ module Unicode put(data, 73111, 73111, 9_u8) put(data, 73537, 73537, 9_u8) put(data, 73538, 73538, 9_u8) + put(data, 90415, 90415, 9_u8) put(data, 92912, 92916, 1_u8) put(data, 92976, 92982, 230_u8) put(data, 94192, 94193, 6_u8) @@ -3353,6 +3418,8 @@ module Unicode put(data, 124140, 124141, 232_u8) put(data, 124142, 124142, 220_u8) put(data, 124143, 124143, 230_u8) + put(data, 124398, 124398, 230_u8) + put(data, 124399, 124399, 220_u8) put(data, 125136, 125142, 220_u8) put(data, 125252, 125257, 230_u8) put(data, 125258, 125258, 7_u8) @@ -3363,7 +3430,7 @@ module Unicode # transformation is always 2 codepoints, so we store them all as 2 codepoints # and 0 means end. private class_getter canonical_decompositions : Hash(Int32, {Int32, Int32}) do - data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 2061) + data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 2081) put(data, 192, 65, 768) put(data, 193, 65, 769) put(data, 194, 65, 770) @@ -4857,6 +4924,8 @@ module Unicode put(data, 64332, 1489, 1471) put(data, 64333, 1499, 1471) put(data, 64334, 1508, 1471) + put(data, 67017, 67026, 775) + put(data, 67044, 67034, 775) put(data, 69786, 69785, 69818) put(data, 69788, 69787, 69818) put(data, 69803, 69797, 69818) @@ -4864,12 +4933,30 @@ module Unicode put(data, 69935, 69938, 69927) put(data, 70475, 70471, 70462) put(data, 70476, 70471, 70487) + put(data, 70531, 70530, 70601) + put(data, 70533, 70532, 70587) + put(data, 70542, 70539, 70594) + put(data, 70545, 70544, 70601) + put(data, 70597, 70594, 70594) + put(data, 70599, 70594, 70584) + put(data, 70600, 70594, 70601) put(data, 70843, 70841, 70842) put(data, 70844, 70841, 70832) put(data, 70846, 70841, 70845) put(data, 71098, 71096, 71087) put(data, 71099, 71097, 71087) put(data, 71992, 71989, 71984) + put(data, 90401, 90398, 90398) + put(data, 90402, 90398, 90409) + put(data, 90403, 90398, 90399) + put(data, 90404, 90409, 90399) + put(data, 90405, 90398, 90400) + put(data, 90406, 90401, 90399) + put(data, 90407, 90402, 90399) + put(data, 90408, 90401, 90400) + put(data, 93544, 93543, 93543) + put(data, 93545, 93539, 93543) + put(data, 93546, 93545, 93543) put(data, 119134, 119127, 119141) put(data, 119135, 119128, 119141) put(data, 119136, 119135, 119150) @@ -8669,7 +8756,7 @@ module Unicode # codepoints. # Here we store: codepoint => {index, count} private class_getter compatibility_decompositions : Hash(Int32, {Int32, Int32}) do - data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 3796) + data = Hash(Int32, {Int32, Int32}).new(initial_capacity: 3832) put(data, 160, 0, 1) put(data, 168, 1, 2) put(data, 170, 3, 1) @@ -11121,6 +11208,42 @@ module Unicode put(data, 67512, 2953, 1) put(data, 67513, 2954, 1) put(data, 67514, 2955, 1) + put(data, 117974, 119, 1) + put(data, 117975, 121, 1) + put(data, 117976, 248, 1) + put(data, 117977, 35, 1) + put(data, 117978, 122, 1) + put(data, 117979, 259, 1) + put(data, 117980, 124, 1) + put(data, 117981, 125, 1) + put(data, 117982, 24, 1) + put(data, 117983, 25, 1) + put(data, 117984, 126, 1) + put(data, 117985, 28, 1) + put(data, 117986, 127, 1) + put(data, 117987, 47, 1) + put(data, 117988, 128, 1) + put(data, 117989, 130, 1) + put(data, 117990, 263, 1) + put(data, 117991, 131, 1) + put(data, 117992, 264, 1) + put(data, 117993, 132, 1) + put(data, 117994, 133, 1) + put(data, 117995, 333, 1) + put(data, 117996, 134, 1) + put(data, 117997, 277, 1) + put(data, 117998, 606, 1) + put(data, 117999, 54, 1) + put(data, 118000, 229, 1) + put(data, 118001, 13, 1) + put(data, 118002, 6, 1) + put(data, 118003, 7, 1) + put(data, 118004, 17, 1) + put(data, 118005, 230, 1) + put(data, 118006, 231, 1) + put(data, 118007, 232, 1) + put(data, 118008, 233, 1) + put(data, 118009, 234, 1) put(data, 119808, 119, 1) put(data, 119809, 121, 1) put(data, 119810, 248, 1) @@ -12473,7 +12596,7 @@ module Unicode # composition exclusions. # Here we store: (first << 21 | second) => codepoint private class_getter canonical_compositions : Hash(Int64, Int32) do - data = Hash(Int64, Int32).new(initial_capacity: 941) + data = Hash(Int64, Int32).new(initial_capacity: 961) put(data, 136315648_i64, 192) put(data, 136315649_i64, 193) put(data, 136315650_i64, 194) @@ -13402,6 +13525,8 @@ module Unicode put(data, 26275229849_i64, 12537) put(data, 26277327001_i64, 12538) put(data, 26300395673_i64, 12542) + put(data, 140563710727_i64, 67017) + put(data, 140580487943_i64, 67044) put(data, 146349822138_i64, 69786) put(data, 146354016442_i64, 69788) put(data, 146374987962_i64, 69803) @@ -13409,12 +13534,30 @@ module Unicode put(data, 146670686503_i64, 69935) put(data, 147788469054_i64, 70475) put(data, 147788469079_i64, 70476) + put(data, 147912201161_i64, 70531) + put(data, 147916395451_i64, 70533) + put(data, 147931075522_i64, 70542) + put(data, 147941561289_i64, 70545) + put(data, 148046418882_i64, 70597) + put(data, 148046418872_i64, 70599) + put(data, 148046418889_i64, 70600) put(data, 148564415674_i64, 70843) put(data, 148564415664_i64, 70844) put(data, 148564415677_i64, 70846) put(data, 149099189679_i64, 71098) put(data, 149101286831_i64, 71099) put(data, 150971947312_i64, 71992) + put(data, 189578436894_i64, 90401) + put(data, 189578436905_i64, 90402) + put(data, 189578436895_i64, 90403) + put(data, 189601505567_i64, 90404) + put(data, 189578436896_i64, 90405) + put(data, 189584728351_i64, 90406) + put(data, 189586825503_i64, 90407) + put(data, 189584728352_i64, 90408) + put(data, 196173983079_i64, 93544) + put(data, 196165594471_i64, 93545) + put(data, 196178177383_i64, 93546) data end @@ -13422,7 +13565,7 @@ module Unicode # Form C (yes if absent in this table). # Here we store: {low, high, result (no or maybe)} private class_getter nfc_quick_check : Array({Int32, Int32, QuickCheckResult}) do - data = Array({Int32, Int32, QuickCheckResult}).new(117) + data = Array({Int32, Int32, QuickCheckResult}).new(124) put(data, 768, 772, QuickCheckResult::Maybe) put(data, 774, 780, QuickCheckResult::Maybe) put(data, 783, 783, QuickCheckResult::Maybe) @@ -13532,11 +13675,18 @@ module Unicode put(data, 69927, 69927, QuickCheckResult::Maybe) put(data, 70462, 70462, QuickCheckResult::Maybe) put(data, 70487, 70487, QuickCheckResult::Maybe) + put(data, 70584, 70584, QuickCheckResult::Maybe) + put(data, 70587, 70587, QuickCheckResult::Maybe) + put(data, 70594, 70594, QuickCheckResult::Maybe) + put(data, 70597, 70597, QuickCheckResult::Maybe) + put(data, 70599, 70601, QuickCheckResult::Maybe) put(data, 70832, 70832, QuickCheckResult::Maybe) put(data, 70842, 70842, QuickCheckResult::Maybe) put(data, 70845, 70845, QuickCheckResult::Maybe) put(data, 71087, 71087, QuickCheckResult::Maybe) put(data, 71984, 71984, QuickCheckResult::Maybe) + put(data, 90398, 90409, QuickCheckResult::Maybe) + put(data, 93543, 93544, QuickCheckResult::Maybe) put(data, 119134, 119140, QuickCheckResult::No) put(data, 119227, 119232, QuickCheckResult::No) put(data, 194560, 195101, QuickCheckResult::No) @@ -13547,7 +13697,7 @@ module Unicode # Form KC (yes if absent in this table). # Here we store: {low, high, result (no or maybe)} private class_getter nfkc_quick_check : Array({Int32, Int32, QuickCheckResult}) do - data = Array({Int32, Int32, QuickCheckResult}).new(436) + data = Array({Int32, Int32, QuickCheckResult}).new(445) put(data, 160, 160, QuickCheckResult::No) put(data, 168, 168, QuickCheckResult::No) put(data, 170, 170, QuickCheckResult::No) @@ -13891,11 +14041,20 @@ module Unicode put(data, 69927, 69927, QuickCheckResult::Maybe) put(data, 70462, 70462, QuickCheckResult::Maybe) put(data, 70487, 70487, QuickCheckResult::Maybe) + put(data, 70584, 70584, QuickCheckResult::Maybe) + put(data, 70587, 70587, QuickCheckResult::Maybe) + put(data, 70594, 70594, QuickCheckResult::Maybe) + put(data, 70597, 70597, QuickCheckResult::Maybe) + put(data, 70599, 70601, QuickCheckResult::Maybe) put(data, 70832, 70832, QuickCheckResult::Maybe) put(data, 70842, 70842, QuickCheckResult::Maybe) put(data, 70845, 70845, QuickCheckResult::Maybe) put(data, 71087, 71087, QuickCheckResult::Maybe) put(data, 71984, 71984, QuickCheckResult::Maybe) + put(data, 90398, 90409, QuickCheckResult::Maybe) + put(data, 93543, 93544, QuickCheckResult::Maybe) + put(data, 117974, 117999, QuickCheckResult::No) + put(data, 118000, 118009, QuickCheckResult::No) put(data, 119134, 119140, QuickCheckResult::No) put(data, 119227, 119232, QuickCheckResult::No) put(data, 119808, 119892, QuickCheckResult::No) @@ -13992,7 +14151,7 @@ module Unicode # codepoints contained here may not appear under NFD. # Here we store: {low, high} private class_getter nfd_quick_check : Array({Int32, Int32}) do - data = Array({Int32, Int32}).new(243) + data = Array({Int32, Int32}).new(253) put(data, 192, 197) put(data, 199, 207) put(data, 209, 214) @@ -14224,15 +14383,25 @@ module Unicode put(data, 64320, 64321) put(data, 64323, 64324) put(data, 64326, 64334) + put(data, 67017, 67017) + put(data, 67044, 67044) put(data, 69786, 69786) put(data, 69788, 69788) put(data, 69803, 69803) put(data, 69934, 69935) put(data, 70475, 70476) + put(data, 70531, 70531) + put(data, 70533, 70533) + put(data, 70542, 70542) + put(data, 70545, 70545) + put(data, 70597, 70597) + put(data, 70599, 70600) put(data, 70843, 70844) put(data, 70846, 70846) put(data, 71098, 71099) put(data, 71992, 71992) + put(data, 90401, 90408) + put(data, 93544, 93546) put(data, 119134, 119140) put(data, 119227, 119232) put(data, 194560, 195101) @@ -14244,7 +14413,7 @@ module Unicode # codepoints contained here may not appear under NFKD. # Here we store: {low, high} private class_getter nfkd_quick_check : Array({Int32, Int32}) do - data = Array({Int32, Int32}).new(548) + data = Array({Int32, Int32}).new(560) put(data, 160, 160) put(data, 168, 168) put(data, 170, 170) @@ -14693,6 +14862,8 @@ module Unicode put(data, 65512, 65512) put(data, 65513, 65516) put(data, 65517, 65518) + put(data, 67017, 67017) + put(data, 67044, 67044) put(data, 67457, 67461) put(data, 67463, 67504) put(data, 67506, 67514) @@ -14701,10 +14872,20 @@ module Unicode put(data, 69803, 69803) put(data, 69934, 69935) put(data, 70475, 70476) + put(data, 70531, 70531) + put(data, 70533, 70533) + put(data, 70542, 70542) + put(data, 70545, 70545) + put(data, 70597, 70597) + put(data, 70599, 70600) put(data, 70843, 70844) put(data, 70846, 70846) put(data, 71098, 71099) put(data, 71992, 71992) + put(data, 90401, 90408) + put(data, 93544, 93546) + put(data, 117974, 117999) + put(data, 118000, 118009) put(data, 119134, 119140) put(data, 119227, 119232) put(data, 119808, 119892) diff --git a/src/unicode/unicode.cr b/src/unicode/unicode.cr index 1fb4b530686b..ab49ea31368b 100644 --- a/src/unicode/unicode.cr +++ b/src/unicode/unicode.cr @@ -1,7 +1,7 @@ # Provides the `Unicode::CaseOptions` enum for special case conversions like Turkic. module Unicode # The currently supported [Unicode](https://home.unicode.org) version. - VERSION = "15.1.0" + VERSION = "16.0.0" # Case options to pass to various `Char` and `String` methods such as `upcase` or `downcase`. @[Flags] From 37da284f8d9ecab90c51b3f1aa8c125cebd4826d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 19 Sep 2024 21:21:03 +0800 Subject: [PATCH 119/193] Add missing `@[Link(dll:)]` annotation to MPIR (#15003) --- src/big/lib_gmp.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/big/lib_gmp.cr b/src/big/lib_gmp.cr index 5ae18b5a4606..c50b1f7f6e9b 100644 --- a/src/big/lib_gmp.cr +++ b/src/big/lib_gmp.cr @@ -1,5 +1,8 @@ {% if flag?(:win32) %} @[Link("mpir")] + {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} + @[Link(dll: "mpir.dll")] + {% end %} {% else %} @[Link("gmp")] {% end %} From c2488f6e9dea40d3225ed18a1a589db62973e482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 20 Sep 2024 11:54:06 +0200 Subject: [PATCH 120/193] Update previous Crystal release 1.13.3 (#15016) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shell.nix | 12 ++++++------ 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39984bc5aadb..5be7fd2cd388 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index ba32bb2dd2d6..aa28b15f9abc 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: build-interpreter: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build strategy: matrix: part: [0, 1, 2, 3] @@ -67,7 +67,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.2-build + image: crystallang/crystal:1.13.3-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d1128ebdbca8..a729d5f7681d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.2] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 152e2b5294b5..dab5b9bb74cd 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -58,7 +58,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.2" + crystal: "1.13.3" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 4eede5adf78c..30bc74844e2b 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -10,7 +10,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.13.2-alpine + container: crystallang/crystal:1.13.3-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 186192288895..9587c5fae85f 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -10,7 +10,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.13.2-alpine + container: crystallang/crystal:1.13.3-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.13.2-alpine + container: crystallang/crystal:1.13.3-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 7ce32ee2d625..e35a9ad182fd 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -12,7 +12,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-latest - container: crystallang/crystal:1.13.2-build + container: crystallang/crystal:1.13.3-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 98c428ee5bad..8f164ae5ddc8 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -25,7 +25,7 @@ jobs: uses: crystal-lang/install-crystal@v1 id: install-crystal with: - crystal: "1.13.2" + crystal: "1.13.3" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index 4ca0eb96577e..d998373438e8 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.2-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.3-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.2}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.3}" case $ARCH in x86_64) diff --git a/shell.nix b/shell.nix index db69834a8a89..efadd688f0e3 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; - sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; + sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-darwin-universal.tar.gz"; - sha256 = "sha256:046zlsyrj1i769xh4jvv0a81nlqj7kiz0hliq1za86k1749kcmlz"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; + sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.2/crystal-1.13.2-1-linux-x86_64.tar.gz"; - sha256 = "sha256:0186q0y97135kvxa8bmzgqc24idv19jg4vglany0pkpzy8b3qs0s"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-linux-x86_64.tar.gz"; + sha256 = "sha256:1zf9b3njxx0jzn81dy6vyhkml31kjxfk4iskf13w9ysj0kwakbyz"; }; }.${pkgs.stdenv.system}); From d18482290bb5b069ed4852eda508b31f0213a98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 21 Sep 2024 11:24:07 +0200 Subject: [PATCH 121/193] Fix compiler artifact name in WindowsCI (#15021) Before https://github.com/crystal-lang/crystal/pull/15000 we had two different builds of the compiler in every CI run, a release build and a non-release. Now we only have a release build and there's no need to differentiate by name. Going back to `crystal` makes the install-crystal action work again. --- .github/workflows/win.yml | 6 +++--- .github/workflows/win_build_portable.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 89c13959e8cb..9025586fa991 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -252,7 +252,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal-release + name: crystal path: build - name: Restore LLVM @@ -294,7 +294,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal-release + name: crystal path: build - name: Restore LLVM @@ -330,7 +330,7 @@ jobs: - name: Download Crystal executable uses: actions/download-artifact@v4 with: - name: crystal-release + name: crystal path: etc/win-ci/portable - name: Restore LLVM diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 8f164ae5ddc8..d3265239c20c 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -145,5 +145,5 @@ jobs: - name: Upload Crystal binaries uses: actions/upload-artifact@v4 with: - name: ${{ inputs.release && 'crystal-release' || 'crystal' }} + name: crystal path: crystal From 5f018fcafbf020c4e965ffb401df5246d082aa59 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 21 Sep 2024 18:59:53 +0800 Subject: [PATCH 122/193] Reduce calls to `Crystal::Type#remove_indirection` in module dispatch (#14992) Module types are expanded into unions during Crystal's codegen phase when the receiver of a call has a module type. This patch makes this expansion occur once for the entire call, instead of once for each including type. --- src/compiler/crystal/codegen/call.cr | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/codegen/call.cr b/src/compiler/crystal/codegen/call.cr index 1b678232c054..5934ffeb0c14 100644 --- a/src/compiler/crystal/codegen/call.cr +++ b/src/compiler/crystal/codegen/call.cr @@ -340,7 +340,10 @@ class Crystal::CodeGenVisitor # Create self var if available if node_obj - new_vars["%self"] = LLVMVar.new(@last, node_obj.type, true) + # call `#remove_indirection` here so that the downcast call in + # `#visit(Var)` doesn't spend time expanding module types again and again + # (it should be the only use site of `node_obj.type`) + new_vars["%self"] = LLVMVar.new(@last, node_obj.type.remove_indirection, true) end # Get type if of args and create arg vars @@ -359,6 +362,10 @@ class Crystal::CodeGenVisitor is_super = node.super? + # call `#remove_indirection` here so that the `match_type_id` below doesn't + # spend time expanding module types again and again + owner = owner.remove_indirection unless is_super + with_cloned_context do context.vars = new_vars From c74f6bc6523510d9d8a86640b2b180bafa40649d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 21 Sep 2024 13:00:29 +0200 Subject: [PATCH 123/193] Compiler: enable parallel codegen with MT (#14748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements parallel codegen of object files when MT is enabled in the compiler (`-Dpreview_mt`). It only impacts codegen for compilations with more than one compilation unit (module), that is when neither of `--single-module`, `--release` or `--cross-compile` is specified. This behavior is identical to the fork based codegen. **Advantages:** - allows parallel codegen on Windows (untested); - no need to fork many processes; - no repeated GC collections in each forked processes; - a simple Channel to distribute work efficiently (no need for IPC). The main points are increased portability and simpler logic, despite having to take care of LLVM thread safety quirks (see comments). Co-authored-by: Johannes Müller --- src/compiler/crystal/compiler.cr | 251 +++++++++++++++++++------------ src/llvm/lib_llvm/bit_reader.cr | 5 + src/llvm/module.cr | 6 + 3 files changed, 164 insertions(+), 98 deletions(-) create mode 100644 src/llvm/lib_llvm/bit_reader.cr diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index f25713c6385e..0d7ba0ff12f9 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -5,6 +5,9 @@ require "crystal/digest/md5" {% if flag?(:msvc) %} require "./loader" {% end %} +{% if flag?(:preview_mt) %} + require "wait_group" +{% end %} module Crystal @[Flags] @@ -80,7 +83,13 @@ module Crystal property? no_codegen = false # Maximum number of LLVM modules that are compiled in parallel - property n_threads : Int32 = {% if flag?(:preview_mt) || flag?(:win32) %} 1 {% else %} 8 {% end %} + property n_threads : Int32 = {% if flag?(:preview_mt) %} + ENV["CRYSTAL_WORKERS"]?.try(&.to_i?) || 4 + {% elsif flag?(:win32) %} + 1 + {% else %} + 8 + {% end %} # Default prelude file to use. This ends up adding a # `require "prelude"` (or whatever name is set here) to @@ -328,6 +337,12 @@ module Crystal CompilationUnit.new(self, program, type_name, llvm_mod, output_dir, bc_flags_changed) end + {% if LibLLVM::IS_LT_170 %} + # initialize the legacy pass manager once in the main thread/process + # before we start codegen in threads (MT) or processes (fork) + init_llvm_legacy_pass_manager unless optimization_mode.o0? + {% end %} + if @cross_compile cross_compile program, units, output_filename else @@ -391,7 +406,7 @@ module Crystal llvm_mod = unit.llvm_mod @progress_tracker.stage("Codegen (bc+obj)") do - optimize llvm_mod unless @optimization_mode.o0? + optimize llvm_mod, target_machine unless @optimization_mode.o0? unit.emit(@emit_targets, emit_base_filename || output_filename) @@ -529,7 +544,8 @@ module Crystal private def parallel_codegen(units, n_threads) {% if flag?(:preview_mt) %} - raise "Cannot fork compiler in multithread mode." + raise "LLVM isn't multithreaded and cannot fork compiler in multithread mode." unless LLVM.multithreaded? + mt_codegen(units, n_threads) {% elsif LibC.has_method?("fork") %} fork_codegen(units, n_threads) {% else %} @@ -537,6 +553,39 @@ module Crystal {% end %} end + private def mt_codegen(units, n_threads) + channel = Channel(CompilationUnit).new(n_threads * 2) + wg = WaitGroup.new + mutex = Mutex.new + + n_threads.times do + wg.spawn do + while unit = channel.receive? + unit.compile(isolate_context: true) + mutex.synchronize { @progress_tracker.stage_progress += 1 } + end + end + end + + units.each do |unit| + # We generate the bitcode in the main thread because LLVM contexts + # must be unique per compilation unit, but we share different contexts + # across many modules (or rely on the global context); trying to + # codegen in parallel would segfault! + # + # Luckily generating the bitcode is quick and once the bitcode is + # generated we don't need the global LLVM contexts anymore but can + # parse the bitcode in an isolated context and we can parallelize the + # slowest part: the optimization pass & compiling the object file. + unit.generate_bitcode + + channel.send(unit) + end + channel.close + + wg.wait + end + private def fork_codegen(units, n_threads) workers = fork_workers(n_threads) do |input, output| while i = input.gets(chomp: true).presence @@ -677,9 +726,10 @@ module Crystal puts puts "Codegen (bc+obj):" - if units.size == reused + case reused + when units.size puts " - all previous .o files were reused" - elsif reused == 0 + when .zero? puts " - no previous .o files were reused" else puts " - #{reused}/#{units.size} .o files were reused" @@ -706,61 +756,52 @@ module Crystal end {% if LibLLVM::IS_LT_170 %} + property! pass_manager_builder : LLVM::PassManagerBuilder + + private def init_llvm_legacy_pass_manager + registry = LLVM::PassRegistry.instance + registry.initialize_all + + builder = LLVM::PassManagerBuilder.new + builder.size_level = 0 + + case optimization_mode + in .o3? + builder.opt_level = 3 + builder.use_inliner_with_threshold = 275 + in .o2? + builder.opt_level = 2 + builder.use_inliner_with_threshold = 275 + in .o1? + builder.opt_level = 1 + builder.use_inliner_with_threshold = 150 + in .o0? + # default behaviour, no optimizations + in .os? + builder.opt_level = 2 + builder.size_level = 1 + builder.use_inliner_with_threshold = 50 + in .oz? + builder.opt_level = 2 + builder.size_level = 2 + builder.use_inliner_with_threshold = 5 + end + + @pass_manager_builder = builder + end + private def optimize_with_pass_manager(llvm_mod) fun_pass_manager = llvm_mod.new_function_pass_manager pass_manager_builder.populate fun_pass_manager fun_pass_manager.run llvm_mod - module_pass_manager.run llvm_mod - end - - @module_pass_manager : LLVM::ModulePassManager? - - private def module_pass_manager - @module_pass_manager ||= begin - mod_pass_manager = LLVM::ModulePassManager.new - pass_manager_builder.populate mod_pass_manager - mod_pass_manager - end - end - @pass_manager_builder : LLVM::PassManagerBuilder? - - private def pass_manager_builder - @pass_manager_builder ||= begin - registry = LLVM::PassRegistry.instance - registry.initialize_all - - builder = LLVM::PassManagerBuilder.new - builder.size_level = 0 - - case optimization_mode - in .o3? - builder.opt_level = 3 - builder.use_inliner_with_threshold = 275 - in .o2? - builder.opt_level = 2 - builder.use_inliner_with_threshold = 275 - in .o1? - builder.opt_level = 1 - builder.use_inliner_with_threshold = 150 - in .o0? - # default behaviour, no optimizations - in .os? - builder.opt_level = 2 - builder.size_level = 1 - builder.use_inliner_with_threshold = 50 - in .oz? - builder.opt_level = 2 - builder.size_level = 2 - builder.use_inliner_with_threshold = 5 - end - - builder - end + module_pass_manager = LLVM::ModulePassManager.new + pass_manager_builder.populate module_pass_manager + module_pass_manager.run llvm_mod end {% end %} - protected def optimize(llvm_mod) + protected def optimize(llvm_mod, target_machine) {% if LibLLVM::IS_LT_130 %} optimize_with_pass_manager(llvm_mod) {% else %} @@ -836,6 +877,9 @@ module Crystal getter llvm_mod property? reused_previous_compilation = false getter object_extension : String + @memory_buffer : LLVM::MemoryBuffer? + @object_name : String? + @bc_name : String? def initialize(@compiler : Compiler, program : Program, @name : String, @llvm_mod : LLVM::Module, @output_dir : String, @bc_flags_changed : Bool) @@ -865,40 +909,44 @@ module Crystal @object_extension = compiler.codegen_target.object_extension end - def compile - compile_to_object + def generate_bitcode + @memory_buffer ||= llvm_mod.write_bitcode_to_memory_buffer end - private def compile_to_object - bc_name = self.bc_name - object_name = self.object_name - temporary_object_name = self.temporary_object_name + # To compile a file we first generate a `.bc` file and then create an + # object file from it. These `.bc` files are stored in the cache + # directory. + # + # On a next compilation of the same project, and if the compile flags + # didn't change (a combination of the target triple, mcpu and link flags, + # amongst others), we check if the new `.bc` file is exactly the same as + # the old one. In that case the `.o` file will also be the same, so we + # simply reuse the old one. Generating an `.o` file is what takes most + # time. + # + # However, instead of directly generating the final `.o` file from the + # `.bc` file, we generate it to a temporary name (`.o.tmp`) and then we + # rename that file to `.o`. We do this because the compiler could be + # interrupted while the `.o` file is being generated, leading to a + # corrupted file that later would cause compilation issues. Moving a file + # is an atomic operation so no corrupted `.o` file should be generated. + def compile(isolate_context = false) + if must_compile? + isolate_module_context if isolate_context + update_bitcode_cache + compile_to_object + else + @reused_previous_compilation = true + end + dump_llvm_ir + end + + private def must_compile? + memory_buffer = generate_bitcode - # To compile a file we first generate a `.bc` file and then - # create an object file from it. These `.bc` files are stored - # in the cache directory. - # - # On a next compilation of the same project, and if the compile - # flags didn't change (a combination of the target triple, mcpu - # and link flags, amongst others), we check if the new - # `.bc` file is exactly the same as the old one. In that case - # the `.o` file will also be the same, so we simply reuse the - # old one. Generating an `.o` file is what takes most time. - # - # However, instead of directly generating the final `.o` file - # from the `.bc` file, we generate it to a temporary name (`.o.tmp`) - # and then we rename that file to `.o`. We do this because the compiler - # could be interrupted while the `.o` file is being generated, leading - # to a corrupted file that later would cause compilation issues. - # Moving a file is an atomic operation so no corrupted `.o` file should - # be generated. - - must_compile = true can_reuse_previous_compilation = compiler.emit_targets.none? && !@bc_flags_changed && File.exists?(bc_name) && File.exists?(object_name) - memory_buffer = llvm_mod.write_bitcode_to_memory_buffer - if can_reuse_previous_compilation memory_io = IO::Memory.new(memory_buffer.to_slice) changed = File.open(bc_name) { |bc_file| !IO.same_content?(bc_file, memory_io) } @@ -906,32 +954,39 @@ module Crystal # If the user cancelled a previous compilation # it might be that the .o file is empty if !changed && File.size(object_name) > 0 - must_compile = false memory_buffer.dispose - memory_buffer = nil + return false else # We need to compile, so we'll write the memory buffer to file end end - # If there's a memory buffer, it means we must create a .o from it - if memory_buffer - # Delete existing .o file. It cannot be used anymore. - File.delete?(object_name) - # Create the .bc file (for next compilations) - File.write(bc_name, memory_buffer.to_slice) - memory_buffer.dispose - end + true + end - if must_compile - compiler.optimize llvm_mod unless compiler.optimization_mode.o0? - compiler.target_machine.emit_obj_to_file llvm_mod, temporary_object_name - File.rename(temporary_object_name, object_name) - else - @reused_previous_compilation = true - end + # Parse the previously generated bitcode into the LLVM module using a + # dedicated context, so we can safely optimize & compile the module in + # multiple threads (llvm contexts can't be shared across threads). + private def isolate_module_context + @llvm_mod = LLVM::Module.parse(@memory_buffer.not_nil!, LLVM::Context.new) + end - dump_llvm_ir + private def update_bitcode_cache + return unless memory_buffer = @memory_buffer + + # Delete existing .o file. It cannot be used anymore. + File.delete?(object_name) + # Create the .bc file (for next compilations) + File.write(bc_name, memory_buffer.to_slice) + memory_buffer.dispose + end + + private def compile_to_object + temporary_object_name = self.temporary_object_name + target_machine = compiler.create_target_machine + compiler.optimize llvm_mod, target_machine unless compiler.optimization_mode.o0? + target_machine.emit_obj_to_file llvm_mod, temporary_object_name + File.rename(temporary_object_name, object_name) end private def dump_llvm_ir diff --git a/src/llvm/lib_llvm/bit_reader.cr b/src/llvm/lib_llvm/bit_reader.cr new file mode 100644 index 000000000000..9bfd271cbbe2 --- /dev/null +++ b/src/llvm/lib_llvm/bit_reader.cr @@ -0,0 +1,5 @@ +require "./types" + +lib LibLLVM + fun parse_bitcode_in_context2 = LLVMParseBitcodeInContext2(c : ContextRef, mb : MemoryBufferRef, m : ModuleRef*) : Int +end diff --git a/src/llvm/module.cr b/src/llvm/module.cr index f216d485055c..32b025bffee7 100644 --- a/src/llvm/module.cr +++ b/src/llvm/module.cr @@ -6,6 +6,12 @@ class LLVM::Module getter context : Context + def self.parse(memory_buffer : MemoryBuffer, context : Context) : self + LibLLVM.parse_bitcode_in_context2(context, memory_buffer, out module_ref) + raise "BUG: failed to parse LLVM bitcode from memory buffer" unless module_ref + new(module_ref, context) + end + def initialize(@unwrap : LibLLVM::ModuleRef, @context : Context) @owned = false end From b679b563db6cde5b633bd8bf92f32e3e476d2b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 21 Sep 2024 13:00:58 +0200 Subject: [PATCH 124/193] Add `Exception::CallStack.empty` (#15017) This is a follow-up on https://github.com/crystal-lang/crystal/pull/15002 which explicitly assigns a dummy callstack to `RetryLookupWithLiterals` for performance reasons. `CallStack.empty` is intended to make this use case a bit more ergonomical. It doesn't require allocating a dummy instance with fake data. Instead, it's an explicitly empty callstack. This makes this mechanism easier to re-use in other places (ref https://github.com/crystal-lang/crystal/issues/11658#issuecomment-2352649774). --- src/compiler/crystal/semantic/call.cr | 4 +--- src/exception/call_stack.cr | 7 +++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/compiler/crystal/semantic/call.cr b/src/compiler/crystal/semantic/call.cr index e265829a919e..1fa4379d543e 100644 --- a/src/compiler/crystal/semantic/call.cr +++ b/src/compiler/crystal/semantic/call.cr @@ -13,10 +13,8 @@ class Crystal::Call property? uses_with_scope = false class RetryLookupWithLiterals < ::Exception - @@dummy_call_stack = Exception::CallStack.new - def initialize - self.callstack = @@dummy_call_stack + self.callstack = Exception::CallStack.empty end end diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index c80f73a6ce48..09173f2e5500 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -31,8 +31,11 @@ struct Exception::CallStack @callstack : Array(Void*) @backtrace : Array(String)? - def initialize - @callstack = CallStack.unwind + def initialize(@callstack : Array(Void*) = CallStack.unwind) + end + + def self.empty + new([] of Void*) end def printable_backtrace : Array(String) From eda63e656a3f0ceeeed2dee9f0fd3e5e9db245a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 24 Sep 2024 16:14:40 +0200 Subject: [PATCH 125/193] Cache `Exception::CallStack.empty` to avoid repeat `Array` allocation (#15025) --- src/exception/call_stack.cr | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index 09173f2e5500..be631e19cdc7 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -34,9 +34,7 @@ struct Exception::CallStack def initialize(@callstack : Array(Void*) = CallStack.unwind) end - def self.empty - new([] of Void*) - end + class_getter empty = new([] of Void*) def printable_backtrace : Array(String) @backtrace ||= decode_backtrace From 6a81bb0f120ceef75a7f64fa59f81a4cdf153c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 25 Sep 2024 12:56:17 +0200 Subject: [PATCH 126/193] Add documentation about synchronous DNS resolution (#15027) --- src/socket/addrinfo.cr | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index ef76d0e285b6..411c09143411 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -4,6 +4,19 @@ require "crystal/system/addrinfo" class Socket # Domain name resolver. + # + # # Query Concurrency Behaviour + # + # On most platforms, DNS queries are currently resolved synchronously. + # Calling a resolve method blocks the entire thread until it returns. + # This can cause latencies, especially in single-threaded processes. + # + # DNS queries resolve asynchronously on the following platforms: + # + # * Windows 8 and higher + # + # NOTE: Follow the discussion in [Async DNS resolution (#13619)](https://github.com/crystal-lang/crystal/issues/13619) + # for more details. struct Addrinfo include Crystal::System::Addrinfo From cde543a762d56324e1d1261154d767c84db1ebc1 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 25 Sep 2024 18:56:29 +0800 Subject: [PATCH 127/193] Fix `Slice.literal` for multiple calls with identical signature (#15009) --- spec/primitives/slice_spec.cr | 7 ++ src/compiler/crystal/program.cr | 4 +- src/compiler/crystal/semantic/main_visitor.cr | 105 ++++++++++-------- 3 files changed, 69 insertions(+), 47 deletions(-) diff --git a/spec/primitives/slice_spec.cr b/spec/primitives/slice_spec.cr index 546ae0de5ce1..98bea774df8b 100644 --- a/spec/primitives/slice_spec.cr +++ b/spec/primitives/slice_spec.cr @@ -12,6 +12,13 @@ describe "Primitives: Slice" do slice.to_a.should eq([0, 1, 4, 9, 16, 25] of {{ num }}) slice.read_only?.should be_true end + + # TODO: these should probably return the same pointers + pending_interpreted "creates multiple literals" do + slice1 = Slice({{ num }}).literal(1, 2, 3) + slice2 = Slice({{ num }}).literal(1, 2, 3) + slice1.should eq(slice2) + end {% end %} end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index b1cc99f0dfc6..c262a2d9770a 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -205,6 +205,8 @@ module Crystal types["Regex"] = @regex = NonGenericClassType.new self, self, "Regex", reference types["Range"] = range = @range = GenericClassType.new self, self, "Range", struct_t, ["B", "E"] range.struct = true + types["Slice"] = slice = @slice = GenericClassType.new self, self, "Slice", struct_t, ["T"] + slice.struct = true types["Exception"] = @exception = NonGenericClassType.new self, self, "Exception", reference @@ -528,7 +530,7 @@ module Crystal {% for name in %w(object no_return value number reference void nil bool char int int8 int16 int32 int64 int128 uint8 uint16 uint32 uint64 uint128 float float32 float64 string symbol pointer enumerable indexable - array static_array exception tuple named_tuple proc union enum range regex crystal + array static_array exception tuple named_tuple proc union enum range slice regex crystal packed_annotation thread_local_annotation no_inline_annotation always_inline_annotation naked_annotation returns_twice_annotation raises_annotation primitive_annotation call_convention_annotation diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index c33c64e893ff..ea5626c37f94 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -1313,6 +1313,10 @@ module Crystal if check_special_new_call(node, obj.type?) return false end + + if check_slice_literal_call(node, obj.type?) + return false + end end args.each &.accept(self) @@ -1567,6 +1571,60 @@ module Crystal false end + def check_slice_literal_call(node, obj_type) + return false unless obj_type + return false unless obj_type.metaclass? + + instance_type = obj_type.instance_type.remove_typedef + + if node.name == "literal" + case instance_type + when GenericClassType # Slice + return false unless instance_type == @program.slice + node.raise "TODO: implement slice_literal primitive for Slice without generic arguments" + when GenericClassInstanceType # Slice(T) + return false unless instance_type.generic_type == @program.slice + + element_type = instance_type.type_vars["T"].type + kind = case element_type + when IntegerType + element_type.kind + when FloatType + element_type.kind + else + node.raise "Only slice literals of primitive integer or float types can be created" + end + + node.args.each do |arg| + arg.raise "Expected NumberLiteral, got #{arg.class_desc}" unless arg.is_a?(NumberLiteral) + arg.accept self + arg.raise "Argument out of range for a Slice(#{element_type})" unless arg.representable_in?(element_type) + end + + # create the internal constant `$Slice:n` to hold the slice contents + const_name = "$Slice:#{@program.const_slices.size}" + const_value = Nop.new + const_value.type = @program.static_array_of(element_type, node.args.size) + const = Const.new(@program, @program, const_name, const_value) + @program.types[const_name] = const + @program.const_slices << Program::ConstSliceInfo.new(const_name, kind, node.args) + + # ::Slice.new(pointerof($Slice:n.@buffer), {{ args.size }}, read_only: true) + pointer_node = PointerOf.new(ReadInstanceVar.new(Path.new(const_name).at(node), "@buffer").at(node)).at(node) + size_node = NumberLiteral.new(node.args.size.to_s, :i32).at(node) + read_only_node = NamedArgument.new("read_only", BoolLiteral.new(true).at(node)).at(node) + expanded = Call.new(Path.global("Slice").at(node), "new", [pointer_node, size_node], named_args: [read_only_node]).at(node) + + expanded.accept self + node.bind_to expanded + node.expanded = expanded + return true + end + end + + false + end + # Rewrite: # # LibFoo::Struct.new arg0: value0, argN: value0 @@ -2308,7 +2366,7 @@ module Crystal when "pointer_new" visit_pointer_new node when "slice_literal" - visit_slice_literal node + node.raise "BUG: Slice literal should have been expanded" when "argc" # Already typed when "argv" @@ -2466,51 +2524,6 @@ module Crystal node.type = scope.instance_type end - def visit_slice_literal(node) - call = self.call.not_nil! - - case slice_type = scope.instance_type - when GenericClassType # Slice - call.raise "TODO: implement slice_literal primitive for Slice without generic arguments" - when GenericClassInstanceType # Slice(T) - element_type = slice_type.type_vars["T"].type - kind = case element_type - when IntegerType - element_type.kind - when FloatType - element_type.kind - else - call.raise "Only slice literals of primitive integer or float types can be created" - end - - call.args.each do |arg| - arg.raise "Expected NumberLiteral, got #{arg.class_desc}" unless arg.is_a?(NumberLiteral) - arg.raise "Argument out of range for a Slice(#{element_type})" unless arg.representable_in?(element_type) - end - - # create the internal constant `$Slice:n` to hold the slice contents - const_name = "$Slice:#{@program.const_slices.size}" - const_value = Nop.new - const_value.type = @program.static_array_of(element_type, call.args.size) - const = Const.new(@program, @program, const_name, const_value) - @program.types[const_name] = const - @program.const_slices << Program::ConstSliceInfo.new(const_name, kind, call.args) - - # ::Slice.new(pointerof($Slice:n.@buffer), {{ args.size }}, read_only: true) - pointer_node = PointerOf.new(ReadInstanceVar.new(Path.new(const_name).at(node), "@buffer").at(node)).at(node) - size_node = NumberLiteral.new(call.args.size.to_s, :i32).at(node) - read_only_node = NamedArgument.new("read_only", BoolLiteral.new(true).at(node)).at(node) - extra = Call.new(Path.global("Slice").at(node), "new", [pointer_node, size_node], named_args: [read_only_node]).at(node) - - extra.accept self - node.extra = extra - node.type = slice_type - call.expanded = extra - else - node.raise "BUG: Unknown scope for slice_literal primitive" - end - end - def visit_struct_or_union_set(node) scope = @scope.as(NonGenericClassType) From 9df6760292f525d4a35c243f1d76e7ab534cec44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 25 Sep 2024 22:29:28 +0200 Subject: [PATCH 128/193] Restrict CI runners from runs-on to 8GB (#15030) --- .github/workflows/aarch64.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index 85a8af2c8b37..3df14588e0fc 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -43,7 +43,7 @@ jobs: args: make std_spec aarch64-musl-test-compiler: needs: aarch64-musl-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -94,7 +94,7 @@ jobs: args: make std_spec aarch64-gnu-test-compiler: needs: aarch64-gnu-build - runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source From 48883e2428f5931fbc53a43b8366caffe29d1bfa Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 26 Sep 2024 16:12:08 +0800 Subject: [PATCH 129/193] Update `DeallocationStack` for Windows context switch (#15032) Calling `LibC.GetCurrentThreadStackLimits` outside the main fiber of a thread will continue to return the original stack top of that thread: ```crystal LibC.GetCurrentThreadStackLimits(out low_limit, out high_limit) low_limit # => 86767566848 high_limit # => 86775955456 spawn do LibC.GetCurrentThreadStackLimits(out low_limit2, out high_limit2) low_limit2 # => 86767566848 high_limit2 # => 1863570554880 end sleep 1.second ``` This doesn't affect the Crystal runtime because the function is only called once to initialize the main thread, nor the GC because [it probes the stack top using `VirtualQuery`](https://github.com/ivmai/bdwgc/blob/39394464633d64131690b56fca7379924d9a5abe/win32_threads.c#L656-L672). It may nonetheless affect other external libraries because the bad stack will most certainly contain unmapped memory. The correct way is to also update the [Win32-specific, non-NT `DeallocationStack` field](https://en.wikipedia.org/w/index.php?title=Win32_Thread_Information_Block&oldid=1194048970#Stack_information_stored_in_the_TIB), since this is the actual stack top that gets returned (see also [Wine's implementation](https://gitlab.winehq.org/wine/wine/-/blob/a4e77b33f6897d930261634c1b3ba5e4edc209f3/dlls/kernelbase/thread.c#L131-135)). --- src/fiber/context/x86_64-microsoft.cr | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/fiber/context/x86_64-microsoft.cr b/src/fiber/context/x86_64-microsoft.cr index 55d893cb8184..83f95ea7b069 100644 --- a/src/fiber/context/x86_64-microsoft.cr +++ b/src/fiber/context/x86_64-microsoft.cr @@ -4,19 +4,20 @@ class Fiber # :nodoc: def makecontext(stack_ptr, fiber_main) : Nil # A great explanation on stack contexts for win32: - # https://cfsamson.gitbook.io/green-threads-explained-in-200-lines-of-rust/supporting-windows + # https://web.archive.org/web/20220527113808/https://cfsamson.gitbook.io/green-threads-explained-in-200-lines-of-rust/supporting-windows - # 8 registers + 2 qwords for NT_TIB + 1 parameter + 10 128bit XMM registers - @context.stack_top = (stack_ptr - (11 + 10*2)).as(Void*) + # 8 registers + 3 qwords for NT_TIB + 1 parameter + 10 128bit XMM registers + @context.stack_top = (stack_ptr - (12 + 10*2)).as(Void*) @context.resumable = 1 stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address stack_ptr[-1] = self.as(Void*) # %rcx: puts `self` as first argument for `fiber_main` - # The following two values are stored in the Thread Information Block (NT_TIB) + # The following three values are stored in the Thread Information Block (NT_TIB) # and are used by Windows to track the current stack limits - stack_ptr[-2] = @stack # %gs:0x10: Stack Limit - stack_ptr[-3] = @stack_bottom # %gs:0x08: Stack Base + stack_ptr[-2] = @stack # %gs:0x1478: Win32 DeallocationStack + stack_ptr[-3] = @stack # %gs:0x10: Stack Limit + stack_ptr[-4] = @stack_bottom # %gs:0x08: Stack Base end # :nodoc: @@ -27,6 +28,7 @@ class Fiber # %rcx , %rdx asm(" pushq %rcx + pushq %gs:0x1478 // Thread Information Block: Win32 DeallocationStack pushq %gs:0x10 // Thread Information Block: Stack Limit pushq %gs:0x08 // Thread Information Block: Stack Base pushq %rdi // push 1st argument (because of initial resume) @@ -73,6 +75,7 @@ class Fiber popq %rdi // pop 1st argument (for initial resume) popq %gs:0x08 popq %gs:0x10 + popq %gs:0x1478 popq %rcx ") {% else %} @@ -80,6 +83,7 @@ class Fiber # instructions that breaks the context switching. asm(" pushq %rcx + pushq %gs:0x1478 // Thread Information Block: Win32 DeallocationStack pushq %gs:0x10 // Thread Information Block: Stack Limit pushq %gs:0x08 // Thread Information Block: Stack Base pushq %rdi // push 1st argument (because of initial resume) @@ -126,6 +130,7 @@ class Fiber popq %rdi // pop 1st argument (for initial resume) popq %gs:0x08 popq %gs:0x10 + popq %gs:0x1478 popq %rcx " :: "r"(current_context), "r"(new_context)) {% end %} From 7c3bf20d7cba9ea509429be83181eda410f19070 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 27 Sep 2024 04:51:13 +0800 Subject: [PATCH 130/193] Add missing return type of `LibC.VirtualQuery` (#15036) This function is only used when `-Dwin7` is specified and its return value was unused. --- src/crystal/system/win32/thread.cr | 4 +++- src/lib_c/x86_64-windows-msvc/c/memoryapi.cr | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index 1a4f61a41738..652f5487498f 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -76,7 +76,9 @@ module Crystal::System::Thread {% else %} tib = LibC.NtCurrentTeb high_limit = tib.value.stackBase - LibC.VirtualQuery(tib.value.stackLimit, out mbi, sizeof(LibC::MEMORY_BASIC_INFORMATION)) + if LibC.VirtualQuery(tib.value.stackLimit, out mbi, sizeof(LibC::MEMORY_BASIC_INFORMATION)) == 0 + raise RuntimeError.from_winerror("VirtualQuery") + end low_limit = mbi.allocationBase low_limit {% end %} diff --git a/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr b/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr index 7b0103713d8a..0ea28b8262f6 100644 --- a/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/memoryapi.cr @@ -11,5 +11,5 @@ lib LibC fun VirtualFree(lpAddress : Void*, dwSize : SizeT, dwFreeType : DWORD) : BOOL fun VirtualProtect(lpAddress : Void*, dwSize : SizeT, flNewProtect : DWORD, lpfOldProtect : DWORD*) : BOOL - fun VirtualQuery(lpAddress : Void*, lpBuffer : MEMORY_BASIC_INFORMATION*, dwLength : SizeT) + fun VirtualQuery(lpAddress : Void*, lpBuffer : MEMORY_BASIC_INFORMATION*, dwLength : SizeT) : SizeT end From d58ede5bacba23fbefed9040066aacec44ca953d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 27 Sep 2024 04:52:33 +0800 Subject: [PATCH 131/193] Fix Linux `getrandom` failure in interpreted code (#15035) Whereas the syscall returns the negative of the `Errno` value on failure, the `LibC` function returns -1 and then sets `Errno.value`. Crystal always assumes the former: ```cr err = Errno.new(-read_bytes.to_i) if err.in?(Errno::EINTR, Errno::EAGAIN) ::Fiber.yield else raise RuntimeError.from_os_error("getrandom", err) end ``` https://github.com/crystal-lang/crystal/blob/cde543a762d56324e1d1261154d767c84db1ebc1/src/crystal/system/unix/getrandom.cr#L105-L110 As `EPERM` equals 1 on Linux, _all_ failures are treated like `EPERM` in interpreted code, even though `EPERM` itself is not listed as an error for [`getrandom(2)`](https://man7.org/linux/man-pages/man2/getrandom.2.html), hence the "Operation not permitted". The same can probably happen on other Linux distros if you run out of entropy. --- src/crystal/system/unix/getrandom.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/unix/getrandom.cr b/src/crystal/system/unix/getrandom.cr index 229716a3d846..e759ff0406e6 100644 --- a/src/crystal/system/unix/getrandom.cr +++ b/src/crystal/system/unix/getrandom.cr @@ -13,7 +13,10 @@ require "./syscall" # TODO: Implement syscall for interpreter def self.getrandom(buf : UInt8*, buflen : LibC::SizeT, flags : UInt32) : LibC::SSizeT - LibC.getrandom(buf, buflen, flags) + # the syscall returns the negative of errno directly, the C function + # doesn't, so we mimic the syscall behavior + read_bytes = LibC.getrandom(buf, buflen, flags) + read_bytes >= 0 ? read_bytes : LibC::SSizeT.new(-Errno.value.value) end end {% end %} From 347e7d6b95fe3b612ac8447e11c3ff58d2a26f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 27 Sep 2024 19:09:36 +0200 Subject: [PATCH 132/193] Increase memory for stdlib CI runners from runs-on to 16GB (#15044) #15030 reduced the runner instances for all CI jobs to machines with 8GB. This looked fine at first, but the `test-stdlib` jobs have been unstable since merging. This patch increases runners to bigger instances with more memory to bring stability. --- .github/workflows/aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index 3df14588e0fc..aec37c3860e1 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -26,7 +26,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-musl-test-stdlib: needs: aarch64-musl-build - runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source @@ -77,7 +77,7 @@ jobs: src/llvm/ext/llvm_ext.o aarch64-gnu-test-stdlib: needs: aarch64-gnu-build - runs-on: [runs-on, runner=2cpu-linux-arm64, "family=m7g", ram=8, "run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=4cpu-linux-arm64, "family=m7g", ram=16, "run-id=${{ github.run_id }}"] if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source From 7aeba1e508dc44e4bcc1d0c4bf989b7321906f0d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 28 Sep 2024 21:12:12 +0800 Subject: [PATCH 133/193] Fix undefined behavior in interpreter mixed union upcast (#15042) The interpreter upcasts a value to a mixed union by placing it on top of the stack, and then copying the data portion to a higher position to reserve space for the type ID. Hence, when the size of the value exceeds that of the type ID, the `copy_to` here would occur between two overlapping ranges: ```cr tmp_stack = stack stack_grow_by(union_size - from_size) (tmp_stack - from_size).copy_to(tmp_stack - from_size + type_id_bytesize, from_size) (tmp_stack - from_size).as(Int64*).value = type_id.to_i64! ``` This is undefined behavior in both ISO C and POSIX. Instead `move_to` must be used here (and most likely in a few other places too). This patch also changes the `move_to` in the tuple indexers to `move_from`, although in practice these don't exhibit unexpected behavior, because most `memcpy` implementations copy data from lower addresses to higher addresses, and these calls move data to a lower address. --- spec/compiler/interpreter/unions_spec.cr | 7 +++++++ src/compiler/crystal/interpreter/instructions.cr | 8 +++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/spec/compiler/interpreter/unions_spec.cr b/spec/compiler/interpreter/unions_spec.cr index 11bde229b44d..0fa82e8cbddb 100644 --- a/spec/compiler/interpreter/unions_spec.cr +++ b/spec/compiler/interpreter/unions_spec.cr @@ -36,6 +36,13 @@ describe Crystal::Repl::Interpreter do CRYSTAL end + it "returns large union type (#15041)" do + interpret(<<-CRYSTAL).should eq(4_i64) + a = {3_i64, 4_i64} || nil + a.is_a?(Tuple) ? a[1] : 5_i64 + CRYSTAL + end + it "put and remove from union in local var" do interpret(<<-CRYSTAL).should eq(3) a = 1 == 1 ? 2 : true diff --git a/src/compiler/crystal/interpreter/instructions.cr b/src/compiler/crystal/interpreter/instructions.cr index 6a38afd888d3..23428df03b90 100644 --- a/src/compiler/crystal/interpreter/instructions.cr +++ b/src/compiler/crystal/interpreter/instructions.cr @@ -1309,7 +1309,7 @@ require "./repl" code: begin tmp_stack = stack stack_grow_by(union_size - from_size) - (tmp_stack - from_size).copy_to(tmp_stack - from_size + type_id_bytesize, from_size) + (tmp_stack - from_size).move_to(tmp_stack - from_size + type_id_bytesize, from_size) (tmp_stack - from_size).as(Int64*).value = type_id.to_i64! end, disassemble: { @@ -1319,6 +1319,8 @@ require "./repl" put_reference_type_in_union: { operands: [union_size : Int32], code: begin + # `copy_to` here is valid only when `from_size <= type_id_bytesize`, + # which is always true from_size = sizeof(Pointer(UInt8)) reference = (stack - from_size).as(UInt8**).value type_id = @@ -1462,7 +1464,7 @@ require "./repl" tuple_indexer_known_index: { operands: [tuple_size : Int32, offset : Int32, value_size : Int32], code: begin - (stack - tuple_size).copy_from(stack - tuple_size + offset, value_size) + (stack - tuple_size).move_from(stack - tuple_size + offset, value_size) aligned_value_size = align(value_size) stack_shrink_by(tuple_size - value_size) stack_grow_by(aligned_value_size - value_size) @@ -1474,7 +1476,7 @@ require "./repl" }, tuple_copy_element: { operands: [tuple_size : Int32, old_offset : Int32, new_offset : Int32, element_size : Int32], - code: (stack - tuple_size + new_offset).copy_from(stack - tuple_size + old_offset, element_size), + code: (stack - tuple_size + new_offset).move_from(stack - tuple_size + old_offset, element_size), }, # >>> Tuples (3) From 8692740e69108ba5b088d8987a5cc2c7382a7ad5 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 30 Sep 2024 03:33:01 +0800 Subject: [PATCH 134/193] Support LLVM 19.1 (#14842) --- .github/workflows/llvm.yml | 16 +++++++++------- spec/compiler/codegen/debug_spec.cr | 2 -- src/compiler/crystal/codegen/debug.cr | 16 ++++++++++++++++ src/intrinsics.cr | 9 +++++++-- src/llvm/context.cr | 6 +++++- src/llvm/di_builder.cr | 6 +++++- src/llvm/ext/llvm-versions.txt | 2 +- src/llvm/lib_llvm.cr | 1 + src/llvm/lib_llvm/core.cr | 6 +++++- src/llvm/lib_llvm/debug_info.cr | 15 +++++++++++---- src/llvm/lib_llvm/types.cr | 1 + 11 files changed, 61 insertions(+), 19 deletions(-) diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index dab5b9bb74cd..796b26a66c08 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -17,17 +17,19 @@ jobs: matrix: include: - llvm_version: "13.0.0" - llvm_ubuntu_version: "20.04" + llvm_filename: "clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" - llvm_version: "14.0.0" - llvm_ubuntu_version: "18.04" + llvm_filename: "clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - llvm_version: "15.0.6" - llvm_ubuntu_version: "18.04" + llvm_filename: "clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - llvm_version: "16.0.3" - llvm_ubuntu_version: "22.04" + llvm_filename: "clang+llvm-16.0.3-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - llvm_version: "17.0.6" - llvm_ubuntu_version: "22.04" + llvm_filename: "clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - llvm_version: "18.1.4" - llvm_ubuntu_version: "18.04" + llvm_filename: "clang+llvm-18.1.4-x86_64-linux-gnu-ubuntu-18.04.tar.xz" + - llvm_version: "19.1.0" + llvm_filename: "LLVM-19.1.0-Linux-X64.tar.xz" name: "LLVM ${{ matrix.llvm_version }}" steps: - name: Checkout Crystal source @@ -44,7 +46,7 @@ jobs: - name: Install LLVM ${{ matrix.llvm_version }} run: | mkdir -p llvm - curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.llvm_version }}/clang+llvm-${{ matrix.llvm_version }}-x86_64-linux-gnu-ubuntu-${{ matrix.llvm_ubuntu_version }}.tar.xz" > llvm.tar.xz + curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.llvm_version }}/${{ matrix.llvm_filename }}" > llvm.tar.xz tar x --xz -C llvm --strip-components=1 -f llvm.tar.xz if: steps.cache-llvm.outputs.cache-hit != 'true' diff --git a/spec/compiler/codegen/debug_spec.cr b/spec/compiler/codegen/debug_spec.cr index 4a57056fc7a3..0032fcb64b4c 100644 --- a/spec/compiler/codegen/debug_spec.cr +++ b/spec/compiler/codegen/debug_spec.cr @@ -160,8 +160,6 @@ describe "Code gen: debug" do it "has debug info in closure inside if (#5593)" do codegen(%( - require "prelude" - def foo if true && true yield 1 diff --git a/src/compiler/crystal/codegen/debug.cr b/src/compiler/crystal/codegen/debug.cr index 72555d074bb0..9a03420ba203 100644 --- a/src/compiler/crystal/codegen/debug.cr +++ b/src/compiler/crystal/codegen/debug.cr @@ -367,6 +367,16 @@ module Crystal old_debug_location = @current_debug_location set_current_debug_location location if builder.current_debug_location != llvm_nil && (ptr = alloca) + # FIXME: When debug records are used instead of debug intrinsics, it + # seems inserting them into an empty BasicBlock will instead place them + # in a totally different (next?) function where the variable doesn't + # exist, leading to a "function-local metadata used in wrong function" + # validation error. This might happen when e.g. all variables inside a + # block are closured. Ideally every debug record should immediately + # follow the variable it declares. + {% unless LibLLVM::IS_LT_190 %} + call(do_nothing_fun) if block.instructions.empty? + {% end %} di_builder.insert_declare_at_end(ptr, var, expr, builder.current_debug_location_metadata, block) set_current_debug_location old_debug_location true @@ -376,6 +386,12 @@ module Crystal end end + private def do_nothing_fun + fetch_typed_fun(@llvm_mod, "llvm.donothing") do + LLVM::Type.function([] of LLVM::Type, @llvm_context.void) + end + end + # Emit debug info for toplevel variables. Used for the main module and all # required files. def emit_vars_debug_info(vars) diff --git a/src/intrinsics.cr b/src/intrinsics.cr index 7cdc462ce543..dc83ab91c884 100644 --- a/src/intrinsics.cr +++ b/src/intrinsics.cr @@ -163,8 +163,13 @@ lib LibIntrinsics {% if flag?(:interpreted) %} @[Primitive(:interpreter_intrinsics_fshr128)] {% end %} fun fshr128 = "llvm.fshr.i128"(a : UInt128, b : UInt128, count : UInt128) : UInt128 - fun va_start = "llvm.va_start"(ap : Void*) - fun va_end = "llvm.va_end"(ap : Void*) + {% if compare_versions(Crystal::LLVM_VERSION, "19.1.0") < 0 %} + fun va_start = "llvm.va_start"(ap : Void*) + fun va_end = "llvm.va_end"(ap : Void*) + {% else %} + fun va_start = "llvm.va_start.p0"(ap : Void*) + fun va_end = "llvm.va_end.p0"(ap : Void*) + {% end %} {% if flag?(:i386) || flag?(:x86_64) %} {% if flag?(:interpreted) %} @[Primitive(:interpreter_intrinsics_pause)] {% end %} diff --git a/src/llvm/context.cr b/src/llvm/context.cr index 987e8f13ba6b..84c96610a96f 100644 --- a/src/llvm/context.cr +++ b/src/llvm/context.cr @@ -108,7 +108,11 @@ class LLVM::Context end def const_string(string : String) : Value - Value.new LibLLVM.const_string_in_context(self, string, string.bytesize, 0) + {% if LibLLVM::IS_LT_190 %} + Value.new LibLLVM.const_string_in_context(self, string, string.bytesize, 0) + {% else %} + Value.new LibLLVM.const_string_in_context2(self, string, string.bytesize, 0) + {% end %} end def const_struct(values : Array(LLVM::Value), packed = false) : Value diff --git a/src/llvm/di_builder.cr b/src/llvm/di_builder.cr index 37be65ef8cf8..7a06a7041349 100644 --- a/src/llvm/di_builder.cr +++ b/src/llvm/di_builder.cr @@ -96,7 +96,11 @@ struct LLVM::DIBuilder end def insert_declare_at_end(storage, var_info, expr, dl : LibLLVM::MetadataRef, block) - LibLLVM.di_builder_insert_declare_at_end(self, storage, var_info, expr, dl, block) + {% if LibLLVM::IS_LT_190 %} + LibLLVM.di_builder_insert_declare_at_end(self, storage, var_info, expr, dl, block) + {% else %} + LibLLVM.di_builder_insert_declare_record_at_end(self, storage, var_info, expr, dl, block) + {% end %} end def get_or_create_array(elements : Array(LibLLVM::MetadataRef)) diff --git a/src/llvm/ext/llvm-versions.txt b/src/llvm/ext/llvm-versions.txt index 92ae5ecbaa5a..6f4d3d4816d0 100644 --- a/src/llvm/ext/llvm-versions.txt +++ b/src/llvm/ext/llvm-versions.txt @@ -1 +1 @@ -18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 +19.1 18.1 17.0 16.0 15.0 14.0 13.0 12.0 11.1 11.0 10.0 9.0 8.0 diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 976cedc90df5..4c7ed49e7900 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -65,6 +65,7 @@ IS_LT_160 = {{compare_versions(LibLLVM::VERSION, "16.0.0") < 0}} IS_LT_170 = {{compare_versions(LibLLVM::VERSION, "17.0.0") < 0}} IS_LT_180 = {{compare_versions(LibLLVM::VERSION, "18.0.0") < 0}} + IS_LT_190 = {{compare_versions(LibLLVM::VERSION, "19.0.0") < 0}} end {% end %} diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index ff3327a3f78d..1796bd00a0ee 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -116,7 +116,11 @@ lib LibLLVM fun const_int_get_zext_value = LLVMConstIntGetZExtValue(constant_val : ValueRef) : ULongLong fun const_int_get_sext_value = LLVMConstIntGetSExtValue(constant_val : ValueRef) : LongLong - fun const_string_in_context = LLVMConstStringInContext(c : ContextRef, str : Char*, length : UInt, dont_null_terminate : Bool) : ValueRef + {% if LibLLVM::IS_LT_190 %} + fun const_string_in_context = LLVMConstStringInContext(c : ContextRef, str : Char*, length : UInt, dont_null_terminate : Bool) : ValueRef + {% else %} + fun const_string_in_context2 = LLVMConstStringInContext2(c : ContextRef, str : Char*, length : SizeT, dont_null_terminate : Bool) : ValueRef + {% end %} fun const_struct_in_context = LLVMConstStructInContext(c : ContextRef, constant_vals : ValueRef*, count : UInt, packed : Bool) : ValueRef fun const_array = LLVMConstArray(element_ty : TypeRef, constant_vals : ValueRef*, length : UInt) : ValueRef diff --git a/src/llvm/lib_llvm/debug_info.cr b/src/llvm/lib_llvm/debug_info.cr index e97e8c71a177..e6155b317eb5 100644 --- a/src/llvm/lib_llvm/debug_info.cr +++ b/src/llvm/lib_llvm/debug_info.cr @@ -111,10 +111,17 @@ lib LibLLVM fun metadata_replace_all_uses_with = LLVMMetadataReplaceAllUsesWith(target_metadata : MetadataRef, replacement : MetadataRef) - fun di_builder_insert_declare_at_end = LLVMDIBuilderInsertDeclareAtEnd( - builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, - expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef - ) : ValueRef + {% if LibLLVM::IS_LT_190 %} + fun di_builder_insert_declare_at_end = LLVMDIBuilderInsertDeclareAtEnd( + builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + ) : ValueRef + {% else %} + fun di_builder_insert_declare_record_at_end = LLVMDIBuilderInsertDeclareRecordAtEnd( + builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + ) : DbgRecordRef + {% end %} fun di_builder_create_auto_variable = LLVMDIBuilderCreateAutoVariable( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, diff --git a/src/llvm/lib_llvm/types.cr b/src/llvm/lib_llvm/types.cr index a1b374f30219..532078394794 100644 --- a/src/llvm/lib_llvm/types.cr +++ b/src/llvm/lib_llvm/types.cr @@ -17,4 +17,5 @@ lib LibLLVM {% end %} type OperandBundleRef = Void* type AttributeRef = Void* + type DbgRecordRef = Void* end From 2821fbb7283ae1f440532eb076e8b2cfb1b1b8eb Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 30 Sep 2024 03:34:18 +0800 Subject: [PATCH 135/193] Fix race condition in `pthread_create` handle initialization (#15043) The thread routine may be started before `GC.pthread_create` returns; `Thread#start` then calls `Crystal::System::Thread#stack_address`, which may call `LibC.pthread_getattr_np` before `GC.pthread_create` initializes the `@system_handle` instance variable, hence the segmentation fault. (On musl-libc, [the underlying non-atomic store occurs right before `LibC.pthread_create` returns](https://git.musl-libc.org/cgit/musl/tree/src/thread/pthread_create.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n389).) Thus there needs to be some kind of synchronization. --- spec/std/thread/condition_variable_spec.cr | 8 -------- spec/std/thread/mutex_spec.cr | 8 -------- spec/std/thread_spec.cr | 8 -------- src/crystal/system/unix/pthread.cr | 23 ++++++++++++++++++---- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/spec/std/thread/condition_variable_spec.cr b/spec/std/thread/condition_variable_spec.cr index ff9c44204bb6..1bf78f797357 100644 --- a/spec/std/thread/condition_variable_spec.cr +++ b/spec/std/thread/condition_variable_spec.cr @@ -1,11 +1,3 @@ -{% if flag?(:musl) %} - # FIXME: These thread specs occasionally fail on musl/alpine based ci, so - # they're disabled for now to reduce noise. - # See https://github.com/crystal-lang/crystal/issues/8738 - pending Thread::ConditionVariable - {% skip_file %} -{% end %} - require "../spec_helper" # interpreter doesn't support threads yet (#14287) diff --git a/spec/std/thread/mutex_spec.cr b/spec/std/thread/mutex_spec.cr index ff298f318329..99f3c5d385c3 100644 --- a/spec/std/thread/mutex_spec.cr +++ b/spec/std/thread/mutex_spec.cr @@ -1,11 +1,3 @@ -{% if flag?(:musl) %} - # FIXME: These thread specs occasionally fail on musl/alpine based ci, so - # they're disabled for now to reduce noise. - # See https://github.com/crystal-lang/crystal/issues/8738 - pending Thread::Mutex - {% skip_file %} -{% end %} - require "../spec_helper" # interpreter doesn't support threads yet (#14287) diff --git a/spec/std/thread_spec.cr b/spec/std/thread_spec.cr index feb55454b621..5a43c7e429d1 100644 --- a/spec/std/thread_spec.cr +++ b/spec/std/thread_spec.cr @@ -1,13 +1,5 @@ require "./spec_helper" -{% if flag?(:musl) %} - # FIXME: These thread specs occasionally fail on musl/alpine based ci, so - # they're disabled for now to reduce noise. - # See https://github.com/crystal-lang/crystal/issues/8738 - pending Thread - {% skip_file %} -{% end %} - # interpreter doesn't support threads yet (#14287) pending_interpreted describe: Thread do it "allows passing an argumentless fun to execute" do diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index b55839ff2784..952843f4c2b0 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -9,20 +9,35 @@ module Crystal::System::Thread @system_handle end + protected setter system_handle + private def init_handle - # NOTE: the thread may start before `pthread_create` returns, so - # `@system_handle` must be set as soon as possible; we cannot use a separate - # handle and assign it to `@system_handle`, which would have been too late + # NOTE: `@system_handle` needs to be set here too, not just in + # `.thread_proc`, since the current thread might progress first; the value + # of `LibC.pthread_self` inside the new thread must be equal to this + # `@system_handle` after `pthread_create` returns ret = GC.pthread_create( thread: pointerof(@system_handle), attr: Pointer(LibC::PthreadAttrT).null, - start: ->(data : Void*) { data.as(::Thread).start; Pointer(Void).null }, + start: ->Thread.thread_proc(Void*), arg: self.as(Void*), ) raise RuntimeError.from_os_error("pthread_create", Errno.new(ret)) unless ret == 0 end + def self.thread_proc(data : Void*) : Void* + th = data.as(::Thread) + + # `#start` calls `#stack_address`, which might read `@system_handle` before + # `GC.pthread_create` updates it in the original thread that spawned the + # current one, so we also assign to it here + th.system_handle = current_handle + + th.start + Pointer(Void).null + end + def self.current_handle : Handle LibC.pthread_self end From 46ca8fb923b7dbe85c901dfdd573fd4a56223e8e Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 30 Sep 2024 19:34:36 +0800 Subject: [PATCH 136/193] Fix main stack top detection on musl-libc (#15047) Stack overflow detection relies on the current thread's stack bounds. On Linux this is obtained using `LibC.pthread_getattr_np` and `LibC.pthread_attr_getstack`. However, on musl-libc the internal stack size is always [hardcoded](https://git.musl-libc.org/cgit/musl/tree/src/thread/pthread_create.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n267) to a very small [default](https://git.musl-libc.org/cgit/musl/tree/src/internal/pthread_impl.h?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n197) of [128 KiB](https://git.musl-libc.org/cgit/musl/tree/src/thread/default_attr.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n3), and [`pthread_getattr_np` returns this value unmodified](https://git.musl-libc.org/cgit/musl/tree/src/thread/pthread_getattr_np.c?id=dd1e63c3638d5f9afb857fccf6ce1415ca5f1b8b#n13), presumably for the main thread as well. The result is that the main thread has an entirely incorrect `@stack`, as it respects `ulimit -s`, and that non-main threads are really that small, as we don't pass any `attr` to `GC.pthread_create` on thread creation. [This is also mentioned on Ruby's bug tracker](https://bugs.ruby-lang.org/issues/14387#change-70914). --- spec/std/kernel_spec.cr | 23 ++++++--------- src/crystal/system/unix/pthread.cr | 29 +++++++++++++++++-- .../aarch64-linux-musl/c/sys/resource.cr | 11 +++++++ src/lib_c/i386-linux-musl/c/sys/resource.cr | 11 +++++++ src/lib_c/x86_64-linux-musl/c/sys/resource.cr | 11 +++++++ 5 files changed, 68 insertions(+), 17 deletions(-) diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index 149e6385ac97..f41529af901a 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -254,16 +254,12 @@ describe "hardware exception" do error.should_not contain("Stack overflow") end - {% if flag?(:musl) %} - # FIXME: Pending as mitigation for https://github.com/crystal-lang/crystal/issues/7482 - pending "detects stack overflow on the main stack" - {% else %} - it "detects stack overflow on the main stack", tags: %w[slow] do - # This spec can take some time under FreeBSD where - # the default stack size is 0.5G. Setting a - # smaller stack size with `ulimit -s 8192` - # will address this. - status, _, error = compile_and_run_source <<-'CRYSTAL' + it "detects stack overflow on the main stack", tags: %w[slow] do + # This spec can take some time under FreeBSD where + # the default stack size is 0.5G. Setting a + # smaller stack size with `ulimit -s 8192` + # will address this. + status, _, error = compile_and_run_source <<-'CRYSTAL' def foo y = StaticArray(Int8, 512).new(0) foo @@ -271,10 +267,9 @@ describe "hardware exception" do foo CRYSTAL - status.success?.should be_false - error.should contain("Stack overflow") - end - {% end %} + status.success?.should be_false + error.should contain("Stack overflow") + end it "detects stack overflow on a fiber stack", tags: %w[slow] do status, _, error = compile_and_run_source <<-'CRYSTAL' diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 952843f4c2b0..50a0fc56e818 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -131,11 +131,26 @@ module Crystal::System::Thread ret = LibC.pthread_attr_destroy(pointerof(attr)) raise RuntimeError.from_os_error("pthread_attr_destroy", Errno.new(ret)) unless ret == 0 {% elsif flag?(:linux) %} - if LibC.pthread_getattr_np(@system_handle, out attr) == 0 - LibC.pthread_attr_getstack(pointerof(attr), pointerof(address), out _) - end + ret = LibC.pthread_getattr_np(@system_handle, out attr) + raise RuntimeError.from_os_error("pthread_getattr_np", Errno.new(ret)) unless ret == 0 + + LibC.pthread_attr_getstack(pointerof(attr), pointerof(address), out stack_size) + ret = LibC.pthread_attr_destroy(pointerof(attr)) raise RuntimeError.from_os_error("pthread_attr_destroy", Errno.new(ret)) unless ret == 0 + + # with musl-libc, the main thread does not respect `rlimit -Ss` and + # instead returns the same default stack size as non-default threads, so + # we obtain the rlimit to correct the stack address manually + {% if flag?(:musl) %} + if Thread.current_is_main? + if LibC.getrlimit(LibC::RLIMIT_STACK, out rlim) == 0 + address = address + stack_size - rlim.rlim_cur + else + raise RuntimeError.from_errno("getrlimit") + end + end + {% end %} {% elsif flag?(:openbsd) %} ret = LibC.pthread_stackseg_np(@system_handle, out stack) raise RuntimeError.from_os_error("pthread_stackseg_np", Errno.new(ret)) unless ret == 0 @@ -153,6 +168,14 @@ module Crystal::System::Thread address end + {% if flag?(:musl) %} + @@main_handle : Handle = current_handle + + def self.current_is_main? + current_handle == @@main_handle + end + {% end %} + # Warning: must be called from the current thread itself, because Darwin # doesn't allow to set the name of any thread but the current one! private def system_name=(name : String) : String diff --git a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr index 7f550c37a622..daa583ac5895 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr @@ -1,4 +1,15 @@ lib LibC + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + fun getrlimit(Int, Rlimit*) : Int + + RLIMIT_STACK = 3 + struct RUsage ru_utime : Timeval ru_stime : Timeval diff --git a/src/lib_c/i386-linux-musl/c/sys/resource.cr b/src/lib_c/i386-linux-musl/c/sys/resource.cr index 7f550c37a622..daa583ac5895 100644 --- a/src/lib_c/i386-linux-musl/c/sys/resource.cr +++ b/src/lib_c/i386-linux-musl/c/sys/resource.cr @@ -1,4 +1,15 @@ lib LibC + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + fun getrlimit(Int, Rlimit*) : Int + + RLIMIT_STACK = 3 + struct RUsage ru_utime : Timeval ru_stime : Timeval diff --git a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr index 7f550c37a622..daa583ac5895 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr @@ -1,4 +1,15 @@ lib LibC + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + fun getrlimit(Int, Rlimit*) : Int + + RLIMIT_STACK = 3 + struct RUsage ru_utime : Timeval ru_stime : Timeval From 1f6b51d9b6e88b22f29581e817fc4e7c15677a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Fri, 4 Oct 2024 07:50:21 +0200 Subject: [PATCH 137/193] Fix copy-paste-typo in spec describe (#15054) --- spec/compiler/crystal/tools/doc/type_spec.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/compiler/crystal/tools/doc/type_spec.cr b/spec/compiler/crystal/tools/doc/type_spec.cr index c5dde7d4b258..6533955464fe 100644 --- a/spec/compiler/crystal/tools/doc/type_spec.cr +++ b/spec/compiler/crystal/tools/doc/type_spec.cr @@ -312,7 +312,7 @@ describe Doc::Type do end end - describe "#included_modules" do + describe "#extended_modules" do it "only include types with docs" do program = semantic(<<-CRYSTAL, wants_doc: true).program # :nodoc: From a9b26ff381ef91f94b3b35d39babcb891d7219ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 7 Oct 2024 17:03:41 +0200 Subject: [PATCH 138/193] Revert "Add nodoc filter to doc type methods (#14910)" (#15064) * Revert "Fix copy-paste-typo in spec describe (#15054)" This reverts commit 1f6b51d9b6e88b22f29581e817fc4e7c15677a60. * Revert "Add nodoc filter to doc type methods (#14910)" This reverts commit 791b0e451766503e4a8d2b63fd1bc726b9948276. --- spec/compiler/crystal/tools/doc/type_spec.cr | 136 ------------------- src/compiler/crystal/tools/doc/type.cr | 3 - 2 files changed, 139 deletions(-) diff --git a/spec/compiler/crystal/tools/doc/type_spec.cr b/spec/compiler/crystal/tools/doc/type_spec.cr index 6533955464fe..34ab535f6d5e 100644 --- a/spec/compiler/crystal/tools/doc/type_spec.cr +++ b/spec/compiler/crystal/tools/doc/type_spec.cr @@ -212,140 +212,4 @@ describe Doc::Type do type.macros.map(&.name).should eq ["+", "~", "foo"] end end - - describe "#subclasses" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - class Foo - end - - class Bar < Foo - end - - # :nodoc: - class Baz < Foo - end - - module Mod1 - class Bar < ::Foo - end - end - - # :nodoc: - module Mod2 - class Baz < ::Foo - end - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.subclasses.map(&.full_name).should eq ["Bar", "Mod1::Bar"] - end - end - - describe "#ancestors" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - # :nodoc: - module Mod3 - class Baz - end - end - - class Mod2::Baz < Mod3::Baz - end - - module Mod1 - # :nodoc: - class Baz < Mod2::Baz - end - end - - class Baz < Mod1::Baz - end - - class Foo < Baz - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.ancestors.map(&.full_name).should eq ["Baz", "Mod2::Baz"] - end - end - - describe "#included_modules" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - # :nodoc: - module Mod3 - module Baz - end - end - - module Mod2 - # :nodoc: - module Baz - end - end - - module Mod1 - module Baz - end - end - - module Baz - end - - class Foo - include Baz - include Mod1::Baz - include Mod2::Baz - include Mod3::Baz - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.included_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] - end - end - - describe "#extended_modules" do - it "only include types with docs" do - program = semantic(<<-CRYSTAL, wants_doc: true).program - # :nodoc: - module Mod3 - module Baz - end - end - - module Mod2 - # :nodoc: - module Baz - end - end - - module Mod1 - module Baz - end - end - - module Baz - end - - class Foo - extend Baz - extend Mod1::Baz - extend Mod2::Baz - extend Mod3::Baz - end - CRYSTAL - - generator = Doc::Generator.new program, [""] - type = generator.type(program.types["Foo"]) - type.extended_modules.map(&.full_name).should eq ["Baz", "Mod1::Baz"] - end - end end diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 9b1a0a86cf7e..624c8f017fe7 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -117,7 +117,6 @@ class Crystal::Doc::Type unless ast_node? @type.ancestors.each do |ancestor| - next unless @generator.must_include? ancestor doc_type = @generator.type(ancestor) ancestors << doc_type break if ancestor == @generator.program.object || doc_type.ast_node? @@ -259,7 +258,6 @@ class Crystal::Doc::Type included_modules = [] of Type @type.parents.try &.each do |parent| if parent.module? - next unless @generator.must_include? parent included_modules << @generator.type(parent) end end @@ -274,7 +272,6 @@ class Crystal::Doc::Type extended_modules = [] of Type @type.metaclass.parents.try &.each do |parent| if parent.module? - next unless @generator.must_include? parent extended_modules << @generator.type(parent) end end From 401eb47bf373f11b2da6e382c306ead36615e1d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 7 Oct 2024 22:31:17 +0200 Subject: [PATCH 139/193] Fix `crystal tool unreachable` & co visiting circular hierarchies (#15065) Avoid visiting circular hierarchies created by type aliases. --- spec/compiler/crystal/tools/unreachable_spec.cr | 8 ++++++++ src/compiler/crystal/tools/typed_def_processor.cr | 9 +++++++++ src/compiler/crystal/tools/unreachable.cr | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/spec/compiler/crystal/tools/unreachable_spec.cr b/spec/compiler/crystal/tools/unreachable_spec.cr index 12ed82499740..f94277348e6c 100644 --- a/spec/compiler/crystal/tools/unreachable_spec.cr +++ b/spec/compiler/crystal/tools/unreachable_spec.cr @@ -112,6 +112,14 @@ describe "unreachable" do CRYSTAL end + it "handles circular hierarchy references (#14034)" do + assert_unreachable <<-CRYSTAL + class Foo + alias Bar = Foo + end + CRYSTAL + end + it "finds initializer" do assert_unreachable <<-CRYSTAL class Foo diff --git a/src/compiler/crystal/tools/typed_def_processor.cr b/src/compiler/crystal/tools/typed_def_processor.cr index 2ba2441d7902..a0a911a6a618 100644 --- a/src/compiler/crystal/tools/typed_def_processor.cr +++ b/src/compiler/crystal/tools/typed_def_processor.cr @@ -17,6 +17,15 @@ module Crystal::TypedDefProcessor end private def process_type(type : Type) : Nil + # Avoid visiting circular hierarchies. There's no use in processing + # alias types anyway. + # For example: + # + # struct Foo + # alias Bar = Foo + # end + return if type.is_a?(AliasType) || type.is_a?(TypeDefType) + if type.is_a?(NamedType) || type.is_a?(Program) || type.is_a?(FileModule) type.types?.try &.each_value do |inner_type| process_type inner_type diff --git a/src/compiler/crystal/tools/unreachable.cr b/src/compiler/crystal/tools/unreachable.cr index a8886fecf596..733a94518899 100644 --- a/src/compiler/crystal/tools/unreachable.cr +++ b/src/compiler/crystal/tools/unreachable.cr @@ -128,6 +128,15 @@ module Crystal property excludes = [] of String def process_type(type) + # Avoid visiting circular hierarchies. There's no use in processing + # alias types anyway. + # For example: + # + # struct Foo + # alias Bar = Foo + # end + return if type.is_a?(AliasType) || type.is_a?(TypeDefType) + if type.is_a?(ModuleType) track_unused_defs type end From dddc0dc8a0585de82efe3c5e233803c770918a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 8 Oct 2024 17:06:18 +0200 Subject: [PATCH 140/193] Fix `TopLevelVisitor` adding existing `ClassDef` type to current scope (#15067) When a type is reopened, the compiler adds it as nested type to the current scope (i.e. where it's reopened). That's wrong because the reopened type is already scoped to the location of the original location and this could lead to cycles. For example, the following program would crash the compiler with an infinite recursion: ```cr class Foo alias Bar = Foo class Bar end end ``` This fix makes sure to only add newly defined types to the current scope, not reopened ones. All other visitor methods which defined types already do this, only `ClassDef` was unconditionally adding to the current scope. --- spec/compiler/semantic/alias_spec.cr | 16 ++++++++++++++++ .../crystal/semantic/top_level_visitor.cr | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/spec/compiler/semantic/alias_spec.cr b/spec/compiler/semantic/alias_spec.cr index faf3b81b8e92..3af2f24e5e84 100644 --- a/spec/compiler/semantic/alias_spec.cr +++ b/spec/compiler/semantic/alias_spec.cr @@ -178,6 +178,22 @@ describe "Semantic: alias" do Bar.bar )) { int32 } end + + it "reopens #{type} through alias within itself" do + assert_type <<-CRYSTAL { int32 } + #{type} Foo + alias Bar = Foo + + #{type} Bar + def self.bar + 1 + end + end + end + + Foo.bar + CRYSTAL + end end %w(class struct).each do |type| diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 1fc7119b9ffd..3654e24ff7a5 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -193,9 +193,9 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor if superclass.is_a?(GenericClassInstanceType) superclass.generic_type.add_subclass(type) end + scope.types[name] = type end - scope.types[name] = type node.resolved_type = type process_annotations(annotations) do |annotation_type, ann| From c5abee0eb53a770263aa9e1e0bdead7b7904b54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 9 Oct 2024 13:35:05 +0200 Subject: [PATCH 141/193] Add `uri/json` to `docs_main` (#15069) --- src/docs_main.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/docs_main.cr b/src/docs_main.cr index ab3ee2affdbc..1fec70580a04 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -52,6 +52,7 @@ require "./string_pool" require "./string_scanner" require "./unicode/unicode" require "./uri" +require "./uri/json" require "./uri/params/serializable" require "./uuid" require "./uuid/json" From dacd97bccc80b41c7d6c448cfad19d37184766e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 9 Oct 2024 14:09:40 +0200 Subject: [PATCH 142/193] Changelog for 1.14.0 (#14969) --- CHANGELOG.md | 354 ++++++++++++++++++++++++++++++++++++++++++ shard.yml | 2 +- src/SOURCE_DATE_EPOCH | 1 + src/VERSION | 2 +- 4 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 src/SOURCE_DATE_EPOCH diff --git a/CHANGELOG.md b/CHANGELOG.md index 341586a8fb95..76272bb1679b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,359 @@ # Changelog +## [1.14.0] (2024-10-09) + +[1.14.0]: https://github.com/crystal-lang/crystal/releases/1.14.0 + +### Features + +#### lang + +- Allow `^` in constant numeric expressions ([#14951], thanks @HertzDevil) + +[#14951]: https://github.com/crystal-lang/crystal/pull/14951 + +#### stdlib + +- Add support for Windows on aarch64 ([#14911], thanks @HertzDevil) +- *(collection)* **[breaking]** Add support for negative start index in `Slice#[start, count]` ([#14778], thanks @ysbaddaden) +- *(collection)* Add `Slice#same?` ([#14728], thanks @straight-shoota) +- *(concurrency)* Add `WaitGroup.wait` and `WaitGroup#spawn` ([#14837], thanks @jgaskins) +- *(concurrency)* Open non-blocking regular files as overlapped on Windows ([#14921], thanks @HertzDevil) +- *(concurrency)* Support non-blocking `File#read` and `#write` on Windows ([#14940], thanks @HertzDevil) +- *(concurrency)* Support non-blocking `File#read_at` on Windows ([#14958], thanks @HertzDevil) +- *(concurrency)* Support non-blocking `Process.run` standard streams on Windows ([#14941], thanks @HertzDevil) +- *(concurrency)* Support `IO::FileDescriptor#flock_*` on non-blocking files on Windows ([#14943], thanks @HertzDevil) +- *(concurrency)* Emulate non-blocking `STDIN` console on Windows ([#14947], thanks @HertzDevil) +- *(concurrency)* Async DNS resolution on Windows ([#14979], thanks @HertzDevil) +- *(crypto)* Update `LibCrypto` bindings for LibreSSL 3.5+ ([#14872], thanks @straight-shoota) +- *(llvm)* Expose LLVM instruction builder for `neg` and `fneg` ([#14774], thanks @JarnaChao09) +- *(llvm)* **[experimental]** Add minimal LLVM OrcV2 bindings ([#14887], thanks @HertzDevil) +- *(llvm)* Add `LLVM::Builder#finalize` ([#14892], thanks @JarnaChao09) +- *(llvm)* Support LLVM 19.1 ([#14842], thanks @HertzDevil) +- *(macros)* Add `Crystal::Macros::TypeNode#has_inner_pointers?` ([#14847], thanks @HertzDevil) +- *(macros)* Add `HashLiteral#has_key?` and `NamedTupleLiteral#has_key?` ([#14890], thanks @kamil-gwozdz) +- *(numeric)* Implement floating-point manipulation functions for `BigFloat` ([#11007], thanks @HertzDevil) +- *(runtime)* Stop & start the world (undocumented API) ([#14729], thanks @ysbaddaden) +- *(runtime)* Add `Pointer::Appender#to_slice` ([#14874], thanks @straight-shoota) +- *(serialization)* Add `URI.from_json_object_key?` and `URI#to_json_object_key` ([#14834], thanks @nobodywasishere) +- *(serialization)* Add `URI::Params::Serializable` ([#14684], thanks @Blacksmoke16) +- *(system)* Enable full backtrace for exception in process spawn ([#14796], thanks @straight-shoota) +- *(system)* Implement `System::User` on Windows ([#14933], thanks @HertzDevil) +- *(system)* Implement `System::Group` on Windows ([#14945], thanks @HertzDevil) +- *(system)* Add methods to `Crystal::EventLoop` ([#14977], thanks @ysbaddaden) +- *(text)* Add `underscore_to_space` option to `String#titleize` ([#14822], thanks @Blacksmoke16) +- *(text)* Support Unicode 16.0.0 ([#14997], thanks @HertzDevil) + +[#14911]: https://github.com/crystal-lang/crystal/pull/14911 +[#14778]: https://github.com/crystal-lang/crystal/pull/14778 +[#14728]: https://github.com/crystal-lang/crystal/pull/14728 +[#14837]: https://github.com/crystal-lang/crystal/pull/14837 +[#14921]: https://github.com/crystal-lang/crystal/pull/14921 +[#14940]: https://github.com/crystal-lang/crystal/pull/14940 +[#14958]: https://github.com/crystal-lang/crystal/pull/14958 +[#14941]: https://github.com/crystal-lang/crystal/pull/14941 +[#14943]: https://github.com/crystal-lang/crystal/pull/14943 +[#14947]: https://github.com/crystal-lang/crystal/pull/14947 +[#14979]: https://github.com/crystal-lang/crystal/pull/14979 +[#14872]: https://github.com/crystal-lang/crystal/pull/14872 +[#14774]: https://github.com/crystal-lang/crystal/pull/14774 +[#14887]: https://github.com/crystal-lang/crystal/pull/14887 +[#14892]: https://github.com/crystal-lang/crystal/pull/14892 +[#14842]: https://github.com/crystal-lang/crystal/pull/14842 +[#14847]: https://github.com/crystal-lang/crystal/pull/14847 +[#14890]: https://github.com/crystal-lang/crystal/pull/14890 +[#11007]: https://github.com/crystal-lang/crystal/pull/11007 +[#14729]: https://github.com/crystal-lang/crystal/pull/14729 +[#14874]: https://github.com/crystal-lang/crystal/pull/14874 +[#14834]: https://github.com/crystal-lang/crystal/pull/14834 +[#14684]: https://github.com/crystal-lang/crystal/pull/14684 +[#14796]: https://github.com/crystal-lang/crystal/pull/14796 +[#14933]: https://github.com/crystal-lang/crystal/pull/14933 +[#14945]: https://github.com/crystal-lang/crystal/pull/14945 +[#14977]: https://github.com/crystal-lang/crystal/pull/14977 +[#14822]: https://github.com/crystal-lang/crystal/pull/14822 +[#14997]: https://github.com/crystal-lang/crystal/pull/14997 + +#### compiler + +- *(cli)* Adds initial support for external commands ([#14953], thanks @bcardiff) +- *(interpreter)* Add `Crystal::Repl::Value#runtime_type` ([#14156], thanks @bcardiff) +- *(interpreter)* Implement `Reference.pre_initialize` in the interpreter ([#14968], thanks @HertzDevil) +- *(interpreter)* Enable the interpreter on Windows ([#14964], thanks @HertzDevil) + +[#14953]: https://github.com/crystal-lang/crystal/pull/14953 +[#14156]: https://github.com/crystal-lang/crystal/pull/14156 +[#14968]: https://github.com/crystal-lang/crystal/pull/14968 +[#14964]: https://github.com/crystal-lang/crystal/pull/14964 + +### Bugfixes + +#### lang + +- Fix `Slice.literal` for multiple calls with identical signature ([#15009], thanks @HertzDevil) +- *(macros)* Add location info to some `MacroIf` nodes ([#14885], thanks @Blacksmoke16) + +[#15009]: https://github.com/crystal-lang/crystal/pull/15009 +[#14885]: https://github.com/crystal-lang/crystal/pull/14885 + +#### stdlib + +- *(collection)* Fix `Range#size` return type to `Int32` ([#14588], thanks @straight-shoota) +- *(concurrency)* Update `DeallocationStack` for Windows context switch ([#15032], thanks @HertzDevil) +- *(concurrency)* Fix race condition in `pthread_create` handle initialization ([#15043], thanks @HertzDevil) +- *(files)* **[regression]** Fix `File#truncate` and `#lock` for Win32 append-mode files ([#14706], thanks @HertzDevil) +- *(files)* **[breaking]** Avoid flush in finalizers for `Socket` and `IO::FileDescriptor` ([#14882], thanks @straight-shoota) +- *(files)* Make `IO::Buffered#buffer_size=` idempotent ([#14855], thanks @jgaskins) +- *(macros)* Implement `#sort_by` inside macros using `Enumerable#sort_by` ([#14895], thanks @HertzDevil) +- *(macros)* Fix internal error when calling `#is_a?` on `External` nodes ([#14918], thanks @HertzDevil) +- *(networking)* Use correct timeout for `Socket#connect` on Windows ([#14961], thanks @HertzDevil) +- *(numeric)* Fix handle empty string in `String#to_f(whitespace: false)` ([#14902], thanks @Blacksmoke16) +- *(numeric)* Fix exponent wrapping in `Math.frexp(BigFloat)` for very large values ([#14971], thanks @HertzDevil) +- *(numeric)* Fix exponent overflow in `BigFloat#to_s` for very large values ([#14982], thanks @HertzDevil) +- *(numeric)* Add missing `@[Link(dll:)]` annotation to MPIR ([#15003], thanks @HertzDevil) +- *(runtime)* Add missing return type of `LibC.VirtualQuery` ([#15036], thanks @HertzDevil) +- *(runtime)* Fix main stack top detection on musl-libc ([#15047], thanks @HertzDevil) +- *(serialization)* **[breaking]** Remove `XML::Error.errors` ([#14936], thanks @straight-shoota) +- *(specs)* **[regression]** Fix `Expectations::Be` for module type ([#14926], thanks @straight-shoota) +- *(system)* Fix return type restriction for `ENV.fetch` ([#14919], thanks @straight-shoota) +- *(system)* `#file_descriptor_close` should set `@closed` (UNIX) ([#14973], thanks @ysbaddaden) +- *(system)* reinit event loop first after fork (UNIX) ([#14975], thanks @ysbaddaden) +- *(text)* Fix avoid linking `libpcre` when unused ([#14891], thanks @kojix2) +- *(text)* Add type restriction to `String#byte_index` `offset` parameter ([#14981], thanks @straight-shoota) + +[#14588]: https://github.com/crystal-lang/crystal/pull/14588 +[#15032]: https://github.com/crystal-lang/crystal/pull/15032 +[#15043]: https://github.com/crystal-lang/crystal/pull/15043 +[#14706]: https://github.com/crystal-lang/crystal/pull/14706 +[#14882]: https://github.com/crystal-lang/crystal/pull/14882 +[#14855]: https://github.com/crystal-lang/crystal/pull/14855 +[#14895]: https://github.com/crystal-lang/crystal/pull/14895 +[#14918]: https://github.com/crystal-lang/crystal/pull/14918 +[#14961]: https://github.com/crystal-lang/crystal/pull/14961 +[#14902]: https://github.com/crystal-lang/crystal/pull/14902 +[#14971]: https://github.com/crystal-lang/crystal/pull/14971 +[#14982]: https://github.com/crystal-lang/crystal/pull/14982 +[#15003]: https://github.com/crystal-lang/crystal/pull/15003 +[#15036]: https://github.com/crystal-lang/crystal/pull/15036 +[#15047]: https://github.com/crystal-lang/crystal/pull/15047 +[#14936]: https://github.com/crystal-lang/crystal/pull/14936 +[#14926]: https://github.com/crystal-lang/crystal/pull/14926 +[#14919]: https://github.com/crystal-lang/crystal/pull/14919 +[#14973]: https://github.com/crystal-lang/crystal/pull/14973 +[#14975]: https://github.com/crystal-lang/crystal/pull/14975 +[#14891]: https://github.com/crystal-lang/crystal/pull/14891 +[#14981]: https://github.com/crystal-lang/crystal/pull/14981 + +#### compiler + +- *(cli)* Add error handling for linker flag sub commands ([#14932], thanks @straight-shoota) +- *(codegen)* Allow returning `Proc`s from top-level funs ([#14917], thanks @HertzDevil) +- *(codegen)* Fix CRT static-dynamic linking conflict in specs with C sources ([#14970], thanks @HertzDevil) +- *(interpreter)* Fix Linux `getrandom` failure in interpreted code ([#15035], thanks @HertzDevil) +- *(interpreter)* Fix undefined behavior in interpreter mixed union upcast ([#15042], thanks @HertzDevil) +- *(semantic)* Fix `TopLevelVisitor` adding existing `ClassDef` type to current scope ([#15067], thanks @straight-shoota) + +[#14932]: https://github.com/crystal-lang/crystal/pull/14932 +[#14917]: https://github.com/crystal-lang/crystal/pull/14917 +[#14970]: https://github.com/crystal-lang/crystal/pull/14970 +[#15035]: https://github.com/crystal-lang/crystal/pull/15035 +[#15042]: https://github.com/crystal-lang/crystal/pull/15042 +[#15067]: https://github.com/crystal-lang/crystal/pull/15067 + +#### tools + +- *(dependencies)* Fix `crystal tool dependencies` format flat ([#14927], thanks @straight-shoota) +- *(dependencies)* Fix `crystal tool dependencies` filters for Windows paths ([#14928], thanks @straight-shoota) +- *(docs-generator)* Fix doc comment above annotation with macro expansion ([#14849], thanks @Blacksmoke16) +- *(unreachable)* Fix `crystal tool unreachable` & co visiting circular hierarchies ([#15065], thanks @straight-shoota) + +[#14927]: https://github.com/crystal-lang/crystal/pull/14927 +[#14928]: https://github.com/crystal-lang/crystal/pull/14928 +[#14849]: https://github.com/crystal-lang/crystal/pull/14849 +[#15065]: https://github.com/crystal-lang/crystal/pull/15065 + +### Chores + +#### stdlib + +- **[deprecation]** Use `Time::Span` in `Benchmark.ips` ([#14805], thanks @HertzDevil) +- **[deprecation]** Deprecate `::sleep(Number)` ([#14962], thanks @HertzDevil) +- *(runtime)* **[deprecation]** Deprecate `Pointer.new(Int)` ([#14875], thanks @straight-shoota) + +[#14805]: https://github.com/crystal-lang/crystal/pull/14805 +[#14962]: https://github.com/crystal-lang/crystal/pull/14962 +[#14875]: https://github.com/crystal-lang/crystal/pull/14875 + +#### compiler + +- *(interpreter)* Remove TODO in `Crystal::Loader` on Windows ([#14988], thanks @HertzDevil) +- *(interpreter:repl)* Update REPLy version ([#14950], thanks @HertzDevil) + +[#14988]: https://github.com/crystal-lang/crystal/pull/14988 +[#14950]: https://github.com/crystal-lang/crystal/pull/14950 + +### Performance + +#### stdlib + +- *(collection)* Always use unstable sort for simple types ([#14825], thanks @HertzDevil) +- *(collection)* Optimize `Hash#transform_{keys,values}` ([#14502], thanks @jgaskins) +- *(numeric)* Optimize arithmetic between `BigFloat` and integers ([#14944], thanks @HertzDevil) +- *(runtime)* **[regression]** Cache `Exception::CallStack.empty` to avoid repeat `Array` allocation ([#15025], thanks @straight-shoota) + +[#14825]: https://github.com/crystal-lang/crystal/pull/14825 +[#14502]: https://github.com/crystal-lang/crystal/pull/14502 +[#14944]: https://github.com/crystal-lang/crystal/pull/14944 +[#15025]: https://github.com/crystal-lang/crystal/pull/15025 + +#### compiler + +- Avoid unwinding the stack on hot path in method call lookups ([#15002], thanks @ggiraldez) +- *(codegen)* Reduce calls to `Crystal::Type#remove_indirection` in module dispatch ([#14992], thanks @HertzDevil) +- *(codegen)* Compiler: enable parallel codegen with MT ([#14748], thanks @ysbaddaden) + +[#15002]: https://github.com/crystal-lang/crystal/pull/15002 +[#14992]: https://github.com/crystal-lang/crystal/pull/14992 +[#14748]: https://github.com/crystal-lang/crystal/pull/14748 + +### Refactor + +#### stdlib + +- *(concurrency)* Extract `select` from `src/channel.cr` ([#14912], thanks @straight-shoota) +- *(concurrency)* Make `Crystal::IOCP::OverlappedOperation` abstract ([#14987], thanks @HertzDevil) +- *(files)* Move `#evented_read`, `#evented_write` into `Crystal::LibEvent::EventLoop` ([#14883], thanks @straight-shoota) +- *(networking)* Simplify `Socket::Addrinfo.getaddrinfo(&)` ([#14956], thanks @HertzDevil) +- *(networking)* Add `Crystal::System::Addrinfo` ([#14957], thanks @HertzDevil) +- *(runtime)* Add `Exception::CallStack.empty` ([#15017], thanks @straight-shoota) +- *(system)* Refactor cancellation of `IOCP::OverlappedOperation` ([#14754], thanks @straight-shoota) +- *(system)* Include `Crystal::System::Group` instead of extending it ([#14930], thanks @HertzDevil) +- *(system)* Include `Crystal::System::User` instead of extending it ([#14929], thanks @HertzDevil) +- *(system)* Fix: `Crystal::SpinLock` doesn't need to be allocated on the HEAP ([#14972], thanks @ysbaddaden) +- *(system)* Don't involve evloop after fork in `System::Process.spawn` (UNIX) ([#14974], thanks @ysbaddaden) +- *(system)* Refactor `EventLoop` interface for sleeps & select timeouts ([#14980], thanks @ysbaddaden) + +[#14912]: https://github.com/crystal-lang/crystal/pull/14912 +[#14987]: https://github.com/crystal-lang/crystal/pull/14987 +[#14883]: https://github.com/crystal-lang/crystal/pull/14883 +[#14956]: https://github.com/crystal-lang/crystal/pull/14956 +[#14957]: https://github.com/crystal-lang/crystal/pull/14957 +[#15017]: https://github.com/crystal-lang/crystal/pull/15017 +[#14754]: https://github.com/crystal-lang/crystal/pull/14754 +[#14930]: https://github.com/crystal-lang/crystal/pull/14930 +[#14929]: https://github.com/crystal-lang/crystal/pull/14929 +[#14972]: https://github.com/crystal-lang/crystal/pull/14972 +[#14974]: https://github.com/crystal-lang/crystal/pull/14974 +[#14980]: https://github.com/crystal-lang/crystal/pull/14980 + +#### compiler + +- *(codegen)* Compiler: refactor codegen ([#14760], thanks @ysbaddaden) +- *(interpreter)* Refactor interpreter stack code to avoid duplicate macro expansion ([#14876], thanks @straight-shoota) + +[#14760]: https://github.com/crystal-lang/crystal/pull/14760 +[#14876]: https://github.com/crystal-lang/crystal/pull/14876 + +### Documentation + +#### stdlib + +- *(collection)* **[breaking]** Hide `Hash::Entry` from public API docs ([#14881], thanks @Blacksmoke16) +- *(collection)* Fix typos in docs for `Set` and `Hash` ([#14889], thanks @philipp-classen) +- *(llvm)* Add `@[Experimental]` to `LLVM::DIBuilder` ([#14854], thanks @HertzDevil) +- *(networking)* Add documentation about synchronous DNS resolution ([#15027], thanks @straight-shoota) +- *(networking)* Add `uri/json` to `docs_main` ([#15069], thanks @straight-shoota) +- *(runtime)* Add docs about `Pointer`'s alignment requirement ([#14853], thanks @HertzDevil) +- *(runtime)* Reword `Pointer#memcmp`'s documentation ([#14818], thanks @HertzDevil) +- *(runtime)* Add documentation for `NoReturn` and `Void` ([#14817], thanks @nobodywasishere) + +[#14881]: https://github.com/crystal-lang/crystal/pull/14881 +[#14889]: https://github.com/crystal-lang/crystal/pull/14889 +[#14854]: https://github.com/crystal-lang/crystal/pull/14854 +[#15027]: https://github.com/crystal-lang/crystal/pull/15027 +[#15069]: https://github.com/crystal-lang/crystal/pull/15069 +[#14853]: https://github.com/crystal-lang/crystal/pull/14853 +[#14818]: https://github.com/crystal-lang/crystal/pull/14818 +[#14817]: https://github.com/crystal-lang/crystal/pull/14817 + +### Specs + +#### stdlib + +- Remove some uses of deprecated overloads in standard library specs ([#14963], thanks @HertzDevil) +- *(collection)* Disable `Tuple#to_static_array` spec on AArch64 ([#14844], thanks @HertzDevil) +- *(serialization)* Add JSON parsing UTF-8 spec ([#14823], thanks @Blacksmoke16) +- *(text)* Add specs for `String#index`, `#rindex` search for `Char::REPLACEMENT` ([#14946], thanks @straight-shoota) + +[#14963]: https://github.com/crystal-lang/crystal/pull/14963 +[#14844]: https://github.com/crystal-lang/crystal/pull/14844 +[#14823]: https://github.com/crystal-lang/crystal/pull/14823 +[#14946]: https://github.com/crystal-lang/crystal/pull/14946 + +#### compiler + +- *(codegen)* Support return types in codegen specs ([#14888], thanks @HertzDevil) +- *(codegen)* Fix codegen spec for `ProcPointer` of virtual type ([#14903], thanks @HertzDevil) +- *(codegen)* Support LLVM OrcV2 codegen specs ([#14886], thanks @HertzDevil) +- *(codegen)* Don't spawn subprocess if codegen spec uses flags but not the prelude ([#14904], thanks @HertzDevil) + +[#14888]: https://github.com/crystal-lang/crystal/pull/14888 +[#14903]: https://github.com/crystal-lang/crystal/pull/14903 +[#14886]: https://github.com/crystal-lang/crystal/pull/14886 +[#14904]: https://github.com/crystal-lang/crystal/pull/14904 + +### Infrastructure + +- Changelog for 1.14.0 ([#14969], thanks @straight-shoota) +- Update previous Crystal release 1.13.1 ([#14810], thanks @straight-shoota) +- Refactor GitHub changelog generator print special infra ([#14795], thanks @straight-shoota) +- Update distribution-scripts ([#14877], thanks @straight-shoota) +- Update version in `shard.yml` ([#14909], thanks @straight-shoota) +- Merge `release/1.13`@1.13.2 ([#14924], thanks @straight-shoota) +- Update previous Crystal release 1.13.2 ([#14925], thanks @straight-shoota) +- Merge `release/1.13`@1.13.3 ([#15012], thanks @straight-shoota) +- Update previous Crystal release 1.13.3 ([#15016], thanks @straight-shoota) +- **[regression]** Fix `SOURCE_DATE_EPOCH` in `Makefile.win` ([#14922], thanks @HertzDevil) +- *(ci)* Update actions/checkout action to v4 - autoclosed ([#14896], thanks @renovate) +- *(ci)* Update LLVM 18 for `wasm32-test` ([#14821], thanks @straight-shoota) +- *(ci)* Pin package repos for OpenSSL packages ([#14831], thanks @straight-shoota) +- *(ci)* Upgrade XCode 15.4.0 ([#14794], thanks @straight-shoota) +- *(ci)* Update GH Actions ([#14535], thanks @renovate) +- *(ci)* Add test for OpenSSL 3.3 ([#14873], thanks @straight-shoota) +- *(ci)* Update GitHub runner to `macos-14` ([#14833], thanks @straight-shoota) +- *(ci)* Refactor SSL workflow with job matrix ([#14899], thanks @straight-shoota) +- *(ci)* Drop the non-release Windows compiler artifact ([#15000], thanks @HertzDevil) +- *(ci)* Fix compiler artifact name in WindowsCI ([#15021], thanks @straight-shoota) +- *(ci)* Restrict CI runners from runs-on to 8GB ([#15030], thanks @straight-shoota) +- *(ci)* Increase memory for stdlib CI runners from runs-on to 16GB ([#15044], thanks @straight-shoota) +- *(ci)* Use Cygwin to build libiconv on Windows CI ([#14999], thanks @HertzDevil) +- *(ci)* Use our own `libffi` repository on Windows CI ([#14998], thanks @HertzDevil) + +[#14969]: https://github.com/crystal-lang/crystal/pull/14969 +[#14810]: https://github.com/crystal-lang/crystal/pull/14810 +[#14795]: https://github.com/crystal-lang/crystal/pull/14795 +[#14877]: https://github.com/crystal-lang/crystal/pull/14877 +[#14909]: https://github.com/crystal-lang/crystal/pull/14909 +[#14924]: https://github.com/crystal-lang/crystal/pull/14924 +[#14925]: https://github.com/crystal-lang/crystal/pull/14925 +[#15012]: https://github.com/crystal-lang/crystal/pull/15012 +[#15016]: https://github.com/crystal-lang/crystal/pull/15016 +[#14922]: https://github.com/crystal-lang/crystal/pull/14922 +[#14896]: https://github.com/crystal-lang/crystal/pull/14896 +[#14821]: https://github.com/crystal-lang/crystal/pull/14821 +[#14831]: https://github.com/crystal-lang/crystal/pull/14831 +[#14794]: https://github.com/crystal-lang/crystal/pull/14794 +[#14535]: https://github.com/crystal-lang/crystal/pull/14535 +[#14873]: https://github.com/crystal-lang/crystal/pull/14873 +[#14833]: https://github.com/crystal-lang/crystal/pull/14833 +[#14899]: https://github.com/crystal-lang/crystal/pull/14899 +[#15000]: https://github.com/crystal-lang/crystal/pull/15000 +[#15021]: https://github.com/crystal-lang/crystal/pull/15021 +[#15030]: https://github.com/crystal-lang/crystal/pull/15030 +[#15044]: https://github.com/crystal-lang/crystal/pull/15044 +[#14999]: https://github.com/crystal-lang/crystal/pull/14999 +[#14998]: https://github.com/crystal-lang/crystal/pull/14998 + ## [1.13.3] (2024-09-18) [1.13.3]: https://github.com/crystal-lang/crystal/releases/1.13.3 diff --git a/shard.yml b/shard.yml index 1b2835281466..2d43b601771e 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.14.0-dev +version: 1.14.0 authors: - Crystal Core Team diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH new file mode 100644 index 000000000000..e20d25c81b56 --- /dev/null +++ b/src/SOURCE_DATE_EPOCH @@ -0,0 +1 @@ +1728432000 diff --git a/src/VERSION b/src/VERSION index 2f2e08cfa3bf..850e742404bb 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.14.0-dev +1.14.0 From 32b9e6e1c9e984e76548e782d887b3abf6d6df29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 10 Oct 2024 09:39:47 +0200 Subject: [PATCH 143/193] Update previous Crystal release 1.14.0 (#15071) --- .circleci/config.yml | 2 +- .github/workflows/interpreter.yml | 8 ++++---- .github/workflows/linux.yml | 2 +- .github/workflows/llvm.yml | 2 +- .github/workflows/openssl.yml | 2 +- .github/workflows/regex-engine.yml | 4 ++-- .github/workflows/wasm32.yml | 2 +- .github/workflows/win_build_portable.yml | 2 +- bin/ci | 6 +++--- shard.yml | 2 +- shell.nix | 12 ++++++------ src/SOURCE_DATE_EPOCH | 1 - src/VERSION | 2 +- 13 files changed, 23 insertions(+), 24 deletions(-) delete mode 100644 src/SOURCE_DATE_EPOCH diff --git a/.circleci/config.yml b/.circleci/config.yml index 5be7fd2cd388..574d390f3bc3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ parameters: previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string - default: "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1" + default: "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1" defaults: environment: &env diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index aa28b15f9abc..3c74afdd329e 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -13,7 +13,7 @@ jobs: test-interpreter_spec: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build name: "Test Interpreter" steps: - uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: build-interpreter: runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build name: Build interpreter steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build strategy: matrix: part: [0, 1, 2, 3] @@ -67,7 +67,7 @@ jobs: needs: build-interpreter runs-on: ubuntu-22.04 container: - image: crystallang/crystal:1.13.3-build + image: crystallang/crystal:1.14.0-build name: "Test primitives_spec with interpreter" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a729d5f7681d..4bdcc3e0c11e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3] + crystal_bootstrap_version: [1.7.3, 1.8.2, 1.9.2, 1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.0] flags: [""] include: # libffi is only available starting from the 1.2.2 build images diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 796b26a66c08..65d0744575b9 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -60,7 +60,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1 with: - crystal: "1.13.3" + crystal: "1.14.0" - name: Build libllvm_ext run: make -B deps diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 30bc74844e2b..7321abddf788 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -10,7 +10,7 @@ jobs: libssl_test: runs-on: ubuntu-latest name: "${{ matrix.pkg }}" - container: crystallang/crystal:1.13.3-alpine + container: crystallang/crystal:1.14.0-alpine strategy: fail-fast: false matrix: diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 9587c5fae85f..e7ee002103b4 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -10,7 +10,7 @@ jobs: pcre: runs-on: ubuntu-latest name: "PCRE" - container: crystallang/crystal:1.13.3-alpine + container: crystallang/crystal:1.14.0-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: pcre2: runs-on: ubuntu-latest name: "PCRE2" - container: crystallang/crystal:1.13.3-alpine + container: crystallang/crystal:1.14.0-alpine steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index e35a9ad182fd..d0012b67c40f 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -12,7 +12,7 @@ env: jobs: wasm32-test: runs-on: ubuntu-latest - container: crystallang/crystal:1.13.3-build + container: crystallang/crystal:1.14.0-build steps: - name: Download Crystal source uses: actions/checkout@v4 diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index d3265239c20c..889f4d80c629 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -25,7 +25,7 @@ jobs: uses: crystal-lang/install-crystal@v1 id: install-crystal with: - crystal: "1.13.3" + crystal: "1.14.0" - name: Download Crystal source uses: actions/checkout@v4 diff --git a/bin/ci b/bin/ci index d998373438e8..03d8a20a19e4 100755 --- a/bin/ci +++ b/bin/ci @@ -135,8 +135,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.13.3-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-1.14.0-1 crystal;popd' # These commands may take a few minutes to run due to the large size of the repositories. # This restriction has been made on GitHub's request because updating shallow @@ -189,7 +189,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.13.3}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:1.14.0}" case $ARCH in x86_64) diff --git a/shard.yml b/shard.yml index 2d43b601771e..4ddf0dcfb0df 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: crystal -version: 1.14.0 +version: 1.15.0-dev authors: - Crystal Core Team diff --git a/shell.nix b/shell.nix index efadd688f0e3..6501b4a0c577 100644 --- a/shell.nix +++ b/shell.nix @@ -53,18 +53,18 @@ let # Hashes obtained using `nix-prefetch-url --unpack ` latestCrystalBinary = genericBinary ({ x86_64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; - sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz"; + sha256 = "sha256:09mp3mngj4wik4v2bffpms3x8dksmrcy0a7hs4cg8b13hrfdrgww"; }; aarch64-darwin = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-darwin-universal.tar.gz"; - sha256 = "sha256:0iri1hl23kgmlibmm64wc4wdq019z544b7m2h1bl7jxs4dk2wwla"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-darwin-universal.tar.gz"; + sha256 = "sha256:09mp3mngj4wik4v2bffpms3x8dksmrcy0a7hs4cg8b13hrfdrgww"; }; x86_64-linux = { - url = "https://github.com/crystal-lang/crystal/releases/download/1.13.3/crystal-1.13.3-1-linux-x86_64.tar.gz"; - sha256 = "sha256:1zf9b3njxx0jzn81dy6vyhkml31kjxfk4iskf13w9ysj0kwakbyz"; + url = "https://github.com/crystal-lang/crystal/releases/download/1.14.0/crystal-1.14.0-1-linux-x86_64.tar.gz"; + sha256 = "sha256:0p5b22ivggf9xlw91cbhib7n4lzd8is1shd3480jjp14rn1kiy5z"; }; }.${pkgs.stdenv.system}); diff --git a/src/SOURCE_DATE_EPOCH b/src/SOURCE_DATE_EPOCH deleted file mode 100644 index e20d25c81b56..000000000000 --- a/src/SOURCE_DATE_EPOCH +++ /dev/null @@ -1 +0,0 @@ -1728432000 diff --git a/src/VERSION b/src/VERSION index 850e742404bb..9a4866bbcede 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -1.14.0 +1.15.0-dev From f3d49d715fc0b525c0953434096d6146a49fde5b Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 10 Oct 2024 18:14:26 +0800 Subject: [PATCH 144/193] Do not over-commit fiber stacks on Windows (#15037) Whenever a new fiber is spawned on Windows, currently Crystal allocates a fully committed 8 MB range of virtual memory. This commit charge stays until the stack becomes unused and reaped, even when most of the stack goes unused. Thus it is "only" possible to spawn several thousand fibers concurrently before the system runs out of virtual memory, depending on the total size of RAM and page files. With this PR, for every fresh fiber stack, only the guard pages plus one extra initial page are committed. Spawning 100,000 idle fibers now consumes just around 7.4 GB of virtual memory, instead of 800 GB. Committed pages are also reset after a stack is returned to a pool and before it is retrieved again; this should be reasonably first, as decommitting pages doesn't alter the page contents. Note that the guard pages reside immediately above the normal committed pages, not at the top of the whole reserved range. This is required for proper stack overflow detection. --- spec/std/process_spec.cr | 8 ++++ src/crystal/system/fiber.cr | 8 ++-- src/crystal/system/unix/fiber.cr | 3 ++ src/crystal/system/wasi/fiber.cr | 3 ++ src/crystal/system/win32/fiber.cr | 65 ++++++++++++++++++++------- src/fiber/context/x86_64-microsoft.cr | 6 ++- src/fiber/stack_pool.cr | 6 ++- 7 files changed, 78 insertions(+), 21 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 01a154ccb010..8347804cadc5 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -172,6 +172,14 @@ describe Process do error.to_s.should eq("hello#{newline}") end + it "sends long output and error to IO" do + output = IO::Memory.new + error = IO::Memory.new + Process.run(*shell_command("echo #{"." * 8000}"), output: output, error: error) + output.to_s.should eq("." * 8000 + newline) + error.to_s.should be_empty + end + it "controls process in block" do value = Process.run(*stdin_to_stdout_command, error: :inherit) do |proc| proc.input.puts "hello" diff --git a/src/crystal/system/fiber.cr b/src/crystal/system/fiber.cr index 1cc47e2917e1..1f15d2fe5535 100644 --- a/src/crystal/system/fiber.cr +++ b/src/crystal/system/fiber.cr @@ -1,12 +1,12 @@ module Crystal::System::Fiber # Allocates memory for a stack. - # def self.allocate_stack(stack_size : Int) : Void* + # def self.allocate_stack(stack_size : Int, protect : Bool) : Void* + + # Prepares an existing, unused stack for use again. + # def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil # Frees memory of a stack. # def self.free_stack(stack : Void*, stack_size : Int) : Nil - - # Determines location of the top of the main process fiber's stack. - # def self.main_fiber_stack(stack_bottom : Void*) : Void* end {% if flag?(:wasi) %} diff --git a/src/crystal/system/unix/fiber.cr b/src/crystal/system/unix/fiber.cr index 317a3f7fbd41..42153b28bed2 100644 --- a/src/crystal/system/unix/fiber.cr +++ b/src/crystal/system/unix/fiber.cr @@ -21,6 +21,9 @@ module Crystal::System::Fiber pointer end + def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil + end + def self.free_stack(stack : Void*, stack_size) : Nil LibC.munmap(stack, stack_size) end diff --git a/src/crystal/system/wasi/fiber.cr b/src/crystal/system/wasi/fiber.cr index 516fcc10a29a..8461bb15d00c 100644 --- a/src/crystal/system/wasi/fiber.cr +++ b/src/crystal/system/wasi/fiber.cr @@ -3,6 +3,9 @@ module Crystal::System::Fiber LibC.malloc(stack_size) end + def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil + end + def self.free_stack(stack : Void*, stack_size) : Nil LibC.free(stack) end diff --git a/src/crystal/system/win32/fiber.cr b/src/crystal/system/win32/fiber.cr index 9e6495ee594e..05fd230a9cac 100644 --- a/src/crystal/system/win32/fiber.cr +++ b/src/crystal/system/win32/fiber.cr @@ -7,28 +7,63 @@ module Crystal::System::Fiber # overflow RESERVED_STACK_SIZE = LibC::DWORD.new(0x10000) - # the reserved stack size, plus the size of a single page - @@total_reserved_size : LibC::DWORD = begin - LibC.GetNativeSystemInfo(out system_info) - system_info.dwPageSize + RESERVED_STACK_SIZE - end - def self.allocate_stack(stack_size, protect) : Void* - unless memory_pointer = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_COMMIT | LibC::MEM_RESERVE, LibC::PAGE_READWRITE) - raise RuntimeError.from_winerror("VirtualAlloc") + if stack_top = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_RESERVE, LibC::PAGE_READWRITE) + if protect + if commit_and_guard(stack_top, stack_size) + return stack_top + end + else + # for the interpreter, the stack is just ordinary memory so the entire + # range is committed + if LibC.VirtualAlloc(stack_top, stack_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE) + return stack_top + end + end + + # failure + LibC.VirtualFree(stack_top, 0, LibC::MEM_RELEASE) end - # Detects stack overflows by guarding the top of the stack, similar to - # `LibC.mprotect`. Windows will fail to allocate a new guard page for these - # fiber stacks and trigger a stack overflow exception + raise RuntimeError.from_winerror("VirtualAlloc") + end + + def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil if protect - if LibC.VirtualProtect(memory_pointer, @@total_reserved_size, LibC::PAGE_READWRITE | LibC::PAGE_GUARD, out _) == 0 - LibC.VirtualFree(memory_pointer, 0, LibC::MEM_RELEASE) - raise RuntimeError.from_winerror("VirtualProtect") + if LibC.VirtualFree(stack, 0, LibC::MEM_DECOMMIT) == 0 + raise RuntimeError.from_winerror("VirtualFree") + end + unless commit_and_guard(stack, stack_size) + raise RuntimeError.from_winerror("VirtualAlloc") end end + end + + # Commits the bottommost page and sets up the guard pages above it, in the + # same manner as each thread's main stack. When the stack hits a guard page + # for the first time, a page fault is generated, the page's guard status is + # reset, and Windows checks if a reserved page is available above. On success, + # a new guard page is committed, and on failure, a stack overflow exception is + # triggered after the `RESERVED_STACK_SIZE` portion is made available. + private def self.commit_and_guard(stack_top, stack_size) + stack_bottom = stack_top + stack_size + + LibC.GetNativeSystemInfo(out system_info) + stack_commit_size = system_info.dwPageSize + stack_commit_top = stack_bottom - stack_commit_size + unless LibC.VirtualAlloc(stack_commit_top, stack_commit_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE) + return false + end + + # the reserved stack size, plus a final guard page for when the stack + # overflow handler itself overflows the stack + stack_guard_size = system_info.dwPageSize + RESERVED_STACK_SIZE + stack_guard_top = stack_commit_top - stack_guard_size + unless LibC.VirtualAlloc(stack_guard_top, stack_guard_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE | LibC::PAGE_GUARD) + return false + end - memory_pointer + true end def self.free_stack(stack : Void*, stack_size) : Nil diff --git a/src/fiber/context/x86_64-microsoft.cr b/src/fiber/context/x86_64-microsoft.cr index 83f95ea7b069..08576fc348aa 100644 --- a/src/fiber/context/x86_64-microsoft.cr +++ b/src/fiber/context/x86_64-microsoft.cr @@ -10,13 +10,17 @@ class Fiber @context.stack_top = (stack_ptr - (12 + 10*2)).as(Void*) @context.resumable = 1 + # actual stack top, not including guard pages and reserved pages + LibC.GetNativeSystemInfo(out system_info) + stack_top = @stack_bottom - system_info.dwPageSize + stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address stack_ptr[-1] = self.as(Void*) # %rcx: puts `self` as first argument for `fiber_main` # The following three values are stored in the Thread Information Block (NT_TIB) # and are used by Windows to track the current stack limits stack_ptr[-2] = @stack # %gs:0x1478: Win32 DeallocationStack - stack_ptr[-3] = @stack # %gs:0x10: Stack Limit + stack_ptr[-3] = stack_top # %gs:0x10: Stack Limit stack_ptr[-4] = @stack_bottom # %gs:0x08: Stack Base end diff --git a/src/fiber/stack_pool.cr b/src/fiber/stack_pool.cr index c9ea3ceb68e0..8f809335f46c 100644 --- a/src/fiber/stack_pool.cr +++ b/src/fiber/stack_pool.cr @@ -42,7 +42,11 @@ class Fiber # Removes a stack from the bottom of the pool, or allocates a new one. def checkout : {Void*, Void*} - stack = @deque.pop? || Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect) + if stack = @deque.pop? + Crystal::System::Fiber.reset_stack(stack, STACK_SIZE, @protect) + else + stack = Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect) + end {stack, stack + STACK_SIZE} end From 0606cf05bbc2f651296e685ea06eae100c74f394 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 10 Oct 2024 03:15:10 -0700 Subject: [PATCH 145/193] Implement `codecov` format for `unreachable` tool (#15059) --- src/compiler/crystal/tools/unreachable.cr | 29 ++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/tools/unreachable.cr b/src/compiler/crystal/tools/unreachable.cr index 733a94518899..8455a4186882 100644 --- a/src/compiler/crystal/tools/unreachable.cr +++ b/src/compiler/crystal/tools/unreachable.cr @@ -6,7 +6,7 @@ require "csv" module Crystal class Command private def unreachable - config, result = compile_no_codegen "tool unreachable", path_filter: true, unreachable_command: true, allowed_formats: %w[text json csv] + config, result = compile_no_codegen "tool unreachable", path_filter: true, unreachable_command: true, allowed_formats: %w[text json csv codecov] unreachable = UnreachableVisitor.new @@ -42,6 +42,8 @@ module Crystal to_json(STDOUT) when "csv" to_csv(STDOUT) + when "codecov" + to_codecov(STDOUT) else to_text(STDOUT) end @@ -111,6 +113,31 @@ module Crystal end end end + + # https://docs.codecov.com/docs/codecov-custom-coverage-format + def to_codecov(io) + hits = Hash(String, Hash(Int32, Int32)).new { |hash, key| hash[key] = Hash(Int32, Int32).new(0) } + + each do |a_def, location, count| + hits[location.filename][location.line_number] = count + end + + JSON.build io do |builder| + builder.object do + builder.string "coverage" + builder.object do + hits.each do |filename, line_coverage| + builder.string filename + builder.object do + line_coverage.each do |line, count| + builder.field line, count + end + end + end + end + end + end + end end # This visitor walks the entire reachable code tree and collect locations From f237af03d7e39f2ab65627f7e3e44451f124ca63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20B=C3=B6rjesson?= Date: Thu, 10 Oct 2024 12:17:10 +0200 Subject: [PATCH 146/193] Add `Iterator(T).empty` (#15039) --- spec/std/iterator_spec.cr | 7 +++++++ src/iterator.cr | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/spec/std/iterator_spec.cr b/spec/std/iterator_spec.cr index a07b8bedb191..b7f000a871cb 100644 --- a/spec/std/iterator_spec.cr +++ b/spec/std/iterator_spec.cr @@ -33,6 +33,13 @@ private class MockIterator end describe Iterator do + describe "Iterator.empty" do + it "creates empty iterator" do + iter = Iterator(String).empty + iter.next.should be_a(Iterator::Stop) + end + end + describe "Iterator.of" do it "creates singleton" do iter = Iterator.of(42) diff --git a/src/iterator.cr b/src/iterator.cr index a46c813b36b3..6a1513ef2130 100644 --- a/src/iterator.cr +++ b/src/iterator.cr @@ -144,6 +144,19 @@ module Iterator(T) Stop::INSTANCE end + # Returns an empty iterator. + def self.empty + EmptyIterator(T).new + end + + private struct EmptyIterator(T) + include Iterator(T) + + def next + stop + end + end + def self.of(element : T) SingletonIterator(T).new(element) end From b3a052c293e14b27cba241812de78dcb425c38e1 Mon Sep 17 00:00:00 2001 From: Roberto Alsina Date: Thu, 10 Oct 2024 10:13:40 -0300 Subject: [PATCH 147/193] Add `Regex::CompileOptions::MULTILINE_ONLY` (#14870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crystal's `Regex` conflates the pcre options `MULTILINE` and `DOT_ALL` into `CompilerOptions::MULTILINE`. This patch adds an option to use `MULTILINE` without `DOTALL`. To keep backwards compatibility, the behaviour of `MULTILINE` in crystal must not be changed, so the new option is added as `MULTILINE_ONLY`. --------- Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Johannes Müller --- spec/std/regex_spec.cr | 7 +++++++ src/regex.cr | 5 +++++ src/regex/pcre.cr | 3 ++- src/regex/pcre2.cr | 3 ++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/spec/std/regex_spec.cr b/spec/std/regex_spec.cr index 13d301987c56..af03cb2c79b8 100644 --- a/spec/std/regex_spec.cr +++ b/spec/std/regex_spec.cr @@ -250,6 +250,13 @@ describe "Regex" do end end + describe "multiline_only" do + it "anchor" do + ((/^foo.*$/m).match("foo\nbar")).try(&.[](0)).should eq "foo\nbar" + ((Regex.new("^foo.*?", Regex::Options::MULTILINE_ONLY)).match("foo\nbar")).try(&.[](0)).should eq "foo" + end + end + describe "extended" do it "ignores white space" do /foo bar/.matches?("foobar").should be_false diff --git a/src/regex.cr b/src/regex.cr index 69dd500226a9..c71ac9cd673a 100644 --- a/src/regex.cr +++ b/src/regex.cr @@ -240,12 +240,17 @@ class Regex # flag that activates both behaviours, so here we do the same by # mapping `MULTILINE` to `PCRE_MULTILINE | PCRE_DOTALL`. # The same applies for PCRE2 except that the native values are 0x200 and 0x400. + # + # For the behaviour of `PCRE_MULTILINE` use `MULTILINE_ONLY`. # Multiline matching. # # Equivalent to `MULTILINE | DOTALL` in PCRE and PCRE2. MULTILINE = 0x0000_0006 + # Equivalent to `MULTILINE` in PCRE and PCRE2. + MULTILINE_ONLY = 0x0000_0004 + DOTALL = 0x0000_0002 # Ignore white space and `#` comments. diff --git a/src/regex/pcre.cr b/src/regex/pcre.cr index c80714708a0b..19decbb66712 100644 --- a/src/regex/pcre.cr +++ b/src/regex/pcre.cr @@ -36,7 +36,8 @@ module Regex::PCRE if options.includes?(option) flag |= case option when .ignore_case? then LibPCRE::CASELESS - when .multiline? then LibPCRE::DOTALL | LibPCRE::MULTILINE + when .multiline? then LibPCRE::MULTILINE | LibPCRE::DOTALL + when .multiline_only? then LibPCRE::MULTILINE when .dotall? then LibPCRE::DOTALL when .extended? then LibPCRE::EXTENDED when .anchored? then LibPCRE::ANCHORED diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index abbb502eb78c..b56a4ea68839 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -67,7 +67,8 @@ module Regex::PCRE2 if options.includes?(option) flag |= case option when .ignore_case? then LibPCRE2::CASELESS - when .multiline? then LibPCRE2::DOTALL | LibPCRE2::MULTILINE + when .multiline? then LibPCRE2::MULTILINE | LibPCRE2::DOTALL + when .multiline_only? then LibPCRE2::MULTILINE when .dotall? then LibPCRE2::DOTALL when .extended? then LibPCRE2::EXTENDED when .anchored? then LibPCRE2::ANCHORED From 8b4fe412c487cba7cabce8e37c815616a03a0270 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 10 Oct 2024 15:14:41 +0200 Subject: [PATCH 148/193] Assume `getrandom` on Linux (#15040) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `getrandom(2)` syscall was added in 2017 and at the time we couldn't expect the glibc 2.25 to be widely available, but we're in 2024 now, and even Ubuntu 18.04 LTS that is now EOL had a compatible glibc release (2.27). I assume musl-libc also added the symbol at the same time. We can simplify the implementation to assume `getrandom` is available, which avoids the initial check, initialization and fallback to urandom. We still fallback to urandom at compile time when targeting android api level < 28 (we support 24+). An issue is that executables will now expect glibc 2.25+ (for example), though the interpreter already did. We also expect kernel 2.6.18 to be compatible, but `getrandom` was added in 3.17 which means it depends on how the libc symbol is implemented —does it fallback to urandom, does it fail? Related to #15034. Co-authored-by: Johannes Müller --- src/crystal/system/random.cr | 7 +- src/crystal/system/unix/getrandom.cr | 118 +++--------------- src/crystal/system/unix/urandom.cr | 2 - src/lib_c/aarch64-android/c/sys/random.cr | 7 ++ src/lib_c/aarch64-linux-gnu/c/sys/random.cr | 5 + src/lib_c/aarch64-linux-musl/c/sys/random.cr | 5 + src/lib_c/arm-linux-gnueabihf/c/sys/random.cr | 5 + src/lib_c/i386-linux-gnu/c/sys/random.cr | 5 + src/lib_c/i386-linux-musl/c/sys/random.cr | 5 + src/lib_c/x86_64-linux-gnu/c/sys/random.cr | 5 + src/lib_c/x86_64-linux-musl/c/sys/random.cr | 5 + src/random/secure.cr | 2 +- 12 files changed, 68 insertions(+), 103 deletions(-) create mode 100644 src/lib_c/aarch64-android/c/sys/random.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/random.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/random.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/random.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/random.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/random.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/random.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/random.cr diff --git a/src/crystal/system/random.cr b/src/crystal/system/random.cr index 1a5b3c8f4677..ccf9d6dfa344 100644 --- a/src/crystal/system/random.cr +++ b/src/crystal/system/random.cr @@ -13,7 +13,12 @@ end {% if flag?(:wasi) %} require "./wasi/random" {% elsif flag?(:linux) %} - require "./unix/getrandom" + require "c/sys/random" + \{% if LibC.has_method?(:getrandom) %} + require "./unix/getrandom" + \{% else %} + require "./unix/urandom" + \{% end %} {% elsif flag?(:bsd) || flag?(:darwin) %} require "./unix/arc4random" {% elsif flag?(:unix) %} diff --git a/src/crystal/system/unix/getrandom.cr b/src/crystal/system/unix/getrandom.cr index e759ff0406e6..6ad217c7cbf2 100644 --- a/src/crystal/system/unix/getrandom.cr +++ b/src/crystal/system/unix/getrandom.cr @@ -1,119 +1,39 @@ -{% skip_file unless flag?(:linux) %} - -require "c/unistd" -require "./syscall" - -{% if flag?(:interpreted) %} - lib LibC - fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : LibC::SSizeT - end - - module Crystal::System::Syscall - GRND_NONBLOCK = 1u32 - - # TODO: Implement syscall for interpreter - def self.getrandom(buf : UInt8*, buflen : LibC::SizeT, flags : UInt32) : LibC::SSizeT - # the syscall returns the negative of errno directly, the C function - # doesn't, so we mimic the syscall behavior - read_bytes = LibC.getrandom(buf, buflen, flags) - read_bytes >= 0 ? read_bytes : LibC::SSizeT.new(-Errno.value.value) - end - end -{% end %} +require "c/sys/random" module Crystal::System::Random - @@initialized = false - @@getrandom_available = false - @@urandom : ::File? - - private def self.init - @@initialized = true - - if has_sys_getrandom - @@getrandom_available = true - else - urandom = ::File.open("/dev/urandom", "r") - return unless urandom.info.type.character_device? - - urandom.close_on_exec = true - urandom.read_buffering = false # don't buffer bytes - @@urandom = urandom - end - end - - private def self.has_sys_getrandom - sys_getrandom(Bytes.new(16)) - true - rescue - false - end - # Reads n random bytes using the Linux `getrandom(2)` syscall. - def self.random_bytes(buf : Bytes) : Nil - init unless @@initialized - - if @@getrandom_available - getrandom(buf) - elsif urandom = @@urandom - urandom.read_fully(buf) - else - raise "Failed to access secure source to generate random bytes!" - end + def self.random_bytes(buffer : Bytes) : Nil + getrandom(buffer) end def self.next_u : UInt8 - init unless @@initialized - - if @@getrandom_available - buf = uninitialized UInt8 - getrandom(pointerof(buf).to_slice(1)) - buf - elsif urandom = @@urandom - urandom.read_byte.not_nil! - else - raise "Failed to access secure source to generate random bytes!" - end + buffer = uninitialized UInt8 + getrandom(pointerof(buffer).to_slice(1)) + buffer end # Reads n random bytes using the Linux `getrandom(2)` syscall. - private def self.getrandom(buf) + private def self.getrandom(buffer) # getrandom(2) may only read up to 256 bytes at once without being # interrupted or returning early chunk_size = 256 - while buf.size > 0 - if buf.size < chunk_size - chunk_size = buf.size - end + while buffer.size > 0 + read_bytes = 0 - read_bytes = sys_getrandom(buf[0, chunk_size]) + loop do + # pass GRND_NONBLOCK flag so that it fails with EAGAIN if the requested + # entropy was not available + read_bytes = LibC.getrandom(buffer, buffer.size.clamp(..chunk_size), LibC::GRND_NONBLOCK) + break unless read_bytes == -1 - buf += read_bytes - end - end + err = Errno.value + raise RuntimeError.from_os_error("getrandom", err) unless err.in?(Errno::EINTR, Errno::EAGAIN) - # Low-level wrapper for the `getrandom(2)` syscall, returns the number of - # bytes read or the errno as a negative number if an error occurred (or the - # syscall isn't available). The GRND_NONBLOCK=1 flag is passed as last argument, - # so that it returns -EAGAIN if the requested entropy was not available. - # - # We use the kernel syscall instead of the `getrandom` C function so any - # binary compiled for Linux will always use getrandom if the kernel is 3.17+ - # and silently fallback to read from /dev/urandom if not (so it's more - # portable). - private def self.sys_getrandom(buf : Bytes) - loop do - read_bytes = Syscall.getrandom(buf.to_unsafe, LibC::SizeT.new(buf.size), Syscall::GRND_NONBLOCK) - if read_bytes < 0 - err = Errno.new(-read_bytes.to_i) - if err.in?(Errno::EINTR, Errno::EAGAIN) - ::Fiber.yield - else - raise RuntimeError.from_os_error("getrandom", err) - end - else - return read_bytes + ::Fiber.yield end + + buffer += read_bytes end end end diff --git a/src/crystal/system/unix/urandom.cr b/src/crystal/system/unix/urandom.cr index 7ac025f43e6b..fe81129a8ade 100644 --- a/src/crystal/system/unix/urandom.cr +++ b/src/crystal/system/unix/urandom.cr @@ -1,5 +1,3 @@ -{% skip_file unless flag?(:unix) && !flag?(:netbsd) && !flag?(:openbsd) && !flag?(:linux) %} - module Crystal::System::Random @@initialized = false @@urandom : ::File? diff --git a/src/lib_c/aarch64-android/c/sys/random.cr b/src/lib_c/aarch64-android/c/sys/random.cr new file mode 100644 index 000000000000..77e193958ff2 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/random.cr @@ -0,0 +1,7 @@ +lib LibC + {% if ANDROID_API >= 28 %} + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT + {% end %} +end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/random.cr b/src/lib_c/aarch64-linux-gnu/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/random.cr b/src/lib_c/aarch64-linux-musl/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/random.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/i386-linux-gnu/c/sys/random.cr b/src/lib_c/i386-linux-gnu/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/i386-linux-musl/c/sys/random.cr b/src/lib_c/i386-linux-musl/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/random.cr b/src/lib_c/x86_64-linux-gnu/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/random.cr b/src/lib_c/x86_64-linux-musl/c/sys/random.cr new file mode 100644 index 000000000000..2c74de96abfb --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/random.cr @@ -0,0 +1,5 @@ +lib LibC + GRND_NONBLOCK = 1_u32 + + fun getrandom(buf : Void*, buflen : SizeT, flags : UInt32) : SSizeT +end diff --git a/src/random/secure.cr b/src/random/secure.cr index 1722b5e6e884..a6b9df03063f 100644 --- a/src/random/secure.cr +++ b/src/random/secure.cr @@ -12,7 +12,7 @@ require "crystal/system/random" # ``` # # On BSD-based systems and macOS/Darwin, it uses [`arc4random`](https://man.openbsd.org/arc4random), -# on Linux [`getrandom`](http://man7.org/linux/man-pages/man2/getrandom.2.html) (if the kernel supports it), +# on Linux [`getrandom`](http://man7.org/linux/man-pages/man2/getrandom.2.html), # on Windows [`RtlGenRandom`](https://docs.microsoft.com/en-us/windows/win32/api/ntsecapi/nf-ntsecapi-rtlgenrandom), # and falls back to reading from `/dev/urandom` on UNIX systems. module Random::Secure From 991f9d0036e1e80297f2d934a8a3dc6dda1c36ce Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 10 Oct 2024 09:53:25 -0700 Subject: [PATCH 149/193] Enable pending formatter features (#14718) --- spec/compiler/formatter/formatter_spec.cr | 344 +++++------------- spec/llvm-ir/pass-closure-to-c-debug-loc.cr | 2 +- spec/llvm-ir/proc-call-debug-loc.cr | 2 +- spec/std/benchmark_spec.cr | 2 +- spec/std/channel_spec.cr | 100 ++--- spec/std/concurrent/select_spec.cr | 64 ++-- .../http/server/handlers/log_handler_spec.cr | 2 +- spec/std/openssl/ssl/socket_spec.cr | 4 +- spec/std/proc_spec.cr | 16 +- src/compiler/crystal/command/format.cr | 2 +- src/compiler/crystal/compiler.cr | 2 +- src/compiler/crystal/ffi/lib_ffi.cr | 8 +- .../crystal/interpreter/closure_context.cr | 2 +- .../crystal/interpreter/compiled_def.cr | 2 +- src/compiler/crystal/interpreter/compiler.cr | 4 +- .../crystal/interpreter/interpreter.cr | 2 +- .../crystal/interpreter/lib_function.cr | 2 +- src/compiler/crystal/program.cr | 2 +- src/compiler/crystal/semantic/main_visitor.cr | 4 +- src/compiler/crystal/semantic/type_merge.cr | 4 +- src/compiler/crystal/tools/formatter.cr | 12 +- src/compiler/crystal/tools/init.cr | 2 +- src/compiler/crystal/util.cr | 2 +- src/crystal/system/thread.cr | 2 +- src/crystal/system/win32/event_loop_iocp.cr | 2 +- src/crystal/system/win32/file_descriptor.cr | 2 +- src/crystal/system/win32/socket.cr | 2 +- src/gc/boehm.cr | 4 +- src/kernel.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/consoleapi.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/dbghelp.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/fileapi.cr | 4 +- src/lib_c/x86_64-windows-msvc/c/ioapiset.cr | 14 +- .../x86_64-windows-msvc/c/stringapiset.cr | 4 +- src/lib_c/x86_64-windows-msvc/c/winsock2.cr | 24 +- src/llvm/lib_llvm/debug_info.cr | 42 +-- src/llvm/lib_llvm/orc.cr | 2 +- src/proc.cr | 2 +- src/random/isaac.cr | 2 +- src/wait_group.cr | 2 +- 40 files changed, 272 insertions(+), 428 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 7c332aac3b0a..02d140088c2d 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -203,8 +203,8 @@ describe Crystal::Formatter do assert_format "def foo ( x , y , ) \n end", "def foo(x, y)\nend" assert_format "def foo ( x , y ,\n) \n end", "def foo(x, y)\nend" assert_format "def foo ( x ,\n y ) \n end", "def foo(x,\n y)\nend" - assert_format "def foo (\nx ,\n y ) \n end", "def foo(\n x,\n y\n)\nend" - assert_format "class Foo\ndef foo (\nx ,\n y ) \n end\nend", "class Foo\n def foo(\n x,\n y\n )\n end\nend" + assert_format "def foo (\nx ,\n y ) \n end", "def foo(\n x,\n y,\n)\nend" + assert_format "class Foo\ndef foo (\nx ,\n y ) \n end\nend", "class Foo\n def foo(\n x,\n y,\n )\n end\nend" assert_format "def foo ( @x) \n end", "def foo(@x)\nend" assert_format "def foo ( @x, @y) \n end", "def foo(@x, @y)\nend" assert_format "def foo ( @@x) \n end", "def foo(@@x)\nend" @@ -277,7 +277,7 @@ describe Crystal::Formatter do assert_format "def foo(@[AnnOne] @[AnnTwo] &block : Int32 -> ); end", "def foo(@[AnnOne] @[AnnTwo] &block : Int32 ->); end" assert_format <<-CRYSTAL def foo( - @[MyAnn] bar + @[MyAnn] bar, ); end CRYSTAL @@ -321,14 +321,14 @@ describe Crystal::Formatter do ); end CRYSTAL def foo( - @[MyAnn] bar + @[MyAnn] bar, ); end CRYSTAL assert_format <<-CRYSTAL def foo( @[MyAnn] - bar + bar, ); end CRYSTAL @@ -336,7 +336,7 @@ describe Crystal::Formatter do def foo( @[MyAnn] @[MyAnn] - bar + bar, ); end CRYSTAL @@ -345,7 +345,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] bar, - @[MyAnn] baz + @[MyAnn] baz, ); end CRYSTAL @@ -355,7 +355,7 @@ describe Crystal::Formatter do @[MyAnn] bar, - @[MyAnn] baz + @[MyAnn] baz, ); end CRYSTAL @@ -367,7 +367,7 @@ describe Crystal::Formatter do CRYSTAL def foo( @[MyAnn] - bar + bar, ); end CRYSTAL @@ -379,7 +379,7 @@ describe Crystal::Formatter do CRYSTAL def foo( @[MyAnn] - bar + bar, ); end CRYSTAL @@ -391,7 +391,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] baz, @[MyAnn] @[MyAnn] - biz + biz, ); end CRYSTAL @@ -405,7 +405,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] - biz + biz, ); end CRYSTAL @@ -433,7 +433,7 @@ describe Crystal::Formatter do @[MyAnn] @[MyAnn] - biz + biz, ); end CRYSTAL @@ -568,7 +568,7 @@ describe Crystal::Formatter do assert_format "with foo yield bar" context "adds `&` to yielding methods that don't have a block parameter (#8764)" do - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo yield end @@ -578,7 +578,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo() yield end @@ -588,7 +588,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( ) yield @@ -600,7 +600,7 @@ describe Crystal::Formatter do CRYSTAL # #13091 - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo # bar yield end @@ -610,7 +610,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x) yield end @@ -620,7 +620,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x ,) yield end @@ -630,7 +630,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x, y) yield @@ -642,7 +642,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x, y,) yield @@ -654,7 +654,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x ) yield @@ -666,7 +666,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(x, ) yield @@ -678,7 +678,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x) yield @@ -691,7 +691,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x, y) yield @@ -704,7 +704,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x, y) @@ -719,7 +719,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( x, ) @@ -734,7 +734,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[method_signature_yield] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo(a, **b) yield end @@ -744,172 +744,9 @@ describe Crystal::Formatter do end CRYSTAL - assert_format "macro f\n yield\n {{ yield }}\nend", flags: %w[method_signature_yield] + assert_format "macro f\n yield\n {{ yield }}\nend" end - context "does not add `&` without flag `method_signature_yield`" do - assert_format <<-CRYSTAL - def foo - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo() - yield - end - CRYSTAL - def foo - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - ) - yield - end - CRYSTAL - def foo - yield - end - CRYSTAL - - # #13091 - assert_format <<-CRYSTAL - def foo # bar - yield - end - CRYSTAL - - assert_format <<-CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x ,) - yield - end - CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL - def foo(x, - y) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x, - y,) - yield - end - CRYSTAL - def foo(x, - y,) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x - ) - yield - end - CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo(x, - ) - yield - end - CRYSTAL - def foo(x) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x) - yield - end - CRYSTAL - def foo( - x - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x, y) - yield - end - CRYSTAL - def foo( - x, y - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x, - y) - yield - end - CRYSTAL - def foo( - x, - y - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL, <<-CRYSTAL - def foo( - x, - ) - yield - end - CRYSTAL - def foo( - x, - ) - yield - end - CRYSTAL - - assert_format <<-CRYSTAL - def foo(a, **b) - yield - end - CRYSTAL - end - - # Allows trailing commas, but doesn't enforce them - assert_format <<-CRYSTAL - def foo( - a, - b - ) - end - CRYSTAL - assert_format <<-CRYSTAL def foo( a, @@ -935,7 +772,7 @@ describe Crystal::Formatter do CRYSTAL context "adds trailing comma to def multi-line normal, splat, and double splat parameters" do - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL macro foo( a, b @@ -949,7 +786,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL macro foo( a, *b @@ -963,7 +800,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL fun foo( a : Int32, b : Int32 @@ -977,7 +814,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL fun foo( a : Int32, ... @@ -985,7 +822,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b @@ -999,7 +836,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a : Int32, b : Int32 @@ -1013,7 +850,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a : Int32, b : Int32 = 1 @@ -1027,7 +864,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b c @@ -1041,7 +878,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, @[Ann] b @@ -1055,7 +892,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, @[Ann] @@ -1071,7 +908,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b ) @@ -1083,7 +920,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, b, c, d @@ -1097,7 +934,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, # Foo b # Bar @@ -1111,7 +948,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, *b @@ -1125,7 +962,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL, <<-CRYSTAL def foo( a, **b @@ -1139,7 +976,7 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo( a, &block @@ -1147,44 +984,44 @@ describe Crystal::Formatter do end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo( a, ) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, b) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, *args) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, *args, &block) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, **kwargs) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, **kwargs, &block) end CRYSTAL - assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + assert_format <<-CRYSTAL def foo(a, &block) end CRYSTAL @@ -1709,22 +1546,23 @@ describe Crystal::Formatter do assert_format "foo = 1\n->foo.[](Int32)" assert_format "foo = 1\n->foo.[]=(Int32)" - assert_format "->{ x }" - assert_format "->{\nx\n}", "->{\n x\n}" - assert_format "->do\nx\nend", "->do\n x\nend" - assert_format "->( ){ x }", "->{ x }" - assert_format "->() do x end", "->do x end" + assert_format "->{ x }", "-> { x }" + assert_format "->{\nx\n}", "-> {\n x\n}" + assert_format "->do\nx\nend", "-> do\n x\nend" + assert_format "->( ){ x }", "-> { x }" + assert_format "->() do x end", "-> do x end" assert_format "->( x , y ) { x }", "->(x, y) { x }" assert_format "->( x : Int32 , y ) { x }", "->(x : Int32, y) { x }" - assert_format "->{}" + assert_format "->{ x }", "-> { x }" # #13232 - assert_format "->{}", "-> { }", flags: %w[proc_literal_whitespace] - assert_format "->(){}", "-> { }", flags: %w[proc_literal_whitespace] - assert_format "->{1}", "-> { 1 }", flags: %w[proc_literal_whitespace] - assert_format "->(x : Int32) {}", "->(x : Int32) { }", flags: %w[proc_literal_whitespace] - assert_format "-> : Int32 {}", "-> : Int32 { }", flags: %w[proc_literal_whitespace] - assert_format "->do\nend", "-> do\nend", flags: %w[proc_literal_whitespace] + assert_format "->{}", "-> { }" + assert_format "->(){}", "-> { }" + assert_format "->{1}", "-> { 1 }" + assert_format "->(x : Int32) {}", "->(x : Int32) { }" + assert_format "-> : Int32 {}", "-> : Int32 { }" + assert_format "->do\nend", "-> do\nend" + assert_format "-> : Int32 {}", "-> : Int32 { }" # Allows whitespace around proc literal, but doesn't enforce them assert_format "-> { }" @@ -1733,15 +1571,15 @@ describe Crystal::Formatter do assert_format "-> : Int32 { }" assert_format "-> do\nend" - assert_format "-> : Int32 {}" + assert_format "-> : Int32 { }" assert_format "-> : Int32 | String { 1 }" - assert_format "-> : Array(Int32) {}" - assert_format "-> : Int32? {}" - assert_format "-> : Int32* {}" - assert_format "-> : Int32[1] {}" - assert_format "-> : {Int32, String} {}" + assert_format "-> : Array(Int32) {}", "-> : Array(Int32) { }" + assert_format "-> : Int32? {}", "-> : Int32? { }" + assert_format "-> : Int32* {}", "-> : Int32* { }" + assert_format "-> : Int32[1] {}", "-> : Int32[1] { }" + assert_format "-> : {Int32, String} {}", "-> : {Int32, String} { }" assert_format "-> : {Int32} { String }" - assert_format "-> : {x: Int32, y: String} {}" + assert_format "-> : {x: Int32, y: String} {}", "-> : {x: Int32, y: String} { }" assert_format "->\n:\nInt32\n{\n}", "-> : Int32 {\n}" assert_format "->( x )\n:\nInt32 { }", "->(x) : Int32 { }" assert_format "->: Int32 do\nx\nend", "-> : Int32 do\n x\nend" @@ -1929,18 +1767,18 @@ describe Crystal::Formatter do assert_format "foo((1..3))" assert_format "foo ()" assert_format "foo ( )", "foo ()" - assert_format "def foo(\n\n#foo\nx,\n\n#bar\nz\n)\nend", "def foo(\n # foo\n x,\n\n # bar\n z\n)\nend" - assert_format "def foo(\nx, #foo\nz #bar\n)\nend", "def foo(\n x, # foo\n z # bar\n)\nend" + assert_format "def foo(\n\n#foo\nx,\n\n#bar\nz\n)\nend", "def foo(\n # foo\n x,\n\n # bar\n z,\n)\nend" + assert_format "def foo(\nx, #foo\nz #bar\n)\nend", "def foo(\n x, # foo\n z, # bar\n)\nend" assert_format "a = 1;;; b = 2", "a = 1; b = 2" assert_format "a = 1\n;\nb = 2", "a = 1\nb = 2" assert_format "foo do\n # bar\nend" assert_format "abstract def foo\nabstract def bar" - assert_format "if 1\n ->{ 1 }\nend" + assert_format "if 1\n ->{ 1 }\nend", "if 1\n -> { 1 }\nend" assert_format "foo.bar do\n baz\n .b\nend" assert_format "coco.lala\nfoo\n .bar" assert_format "foo.bar = \n1", "foo.bar =\n 1" assert_format "foo.bar += \n1", "foo.bar +=\n 1" - assert_format "->{}" + assert_format "->{}", "-> { }" assert_format "foo &.[a] = 1" assert_format "[\n # foo\n 1,\n\n # bar\n 2,\n]" assert_format "[c.x]\n .foo" @@ -1948,11 +1786,11 @@ describe Crystal::Formatter do assert_format "bar = foo([\n 1,\n 2,\n 3,\n])" assert_format "foo({\n 1 => 2,\n 3 => 4,\n 5 => 6,\n})" assert_format "bar = foo({\n 1 => 2,\n 3 => 4,\n 5 => 6,\n })", "bar = foo({\n 1 => 2,\n 3 => 4,\n 5 => 6,\n})" - assert_format "foo(->{\n 1 + 2\n})" - assert_format "bar = foo(->{\n 1 + 2\n})" - assert_format "foo(->do\n 1 + 2\nend)" - assert_format "bar = foo(->do\n 1 + 2\nend)" - assert_format "bar = foo(->{\n 1 + 2\n})" + assert_format "foo(->{\n 1 + 2\n})", "foo(-> {\n 1 + 2\n})" + assert_format "bar = foo(->{\n 1 + 2\n})", "bar = foo(-> {\n 1 + 2\n})" + assert_format "foo(->do\n 1 + 2\nend)", "foo(-> do\n 1 + 2\nend)" + assert_format "bar = foo(->do\n 1 + 2\nend)", "bar = foo(-> do\n 1 + 2\nend)" + assert_format "bar = foo(->{\n 1 + 2\n})", "bar = foo(-> {\n 1 + 2\n})" assert_format "case 1\nwhen 2\n 3\n # foo\nelse\n 4\n # bar\nend" assert_format "1 #=> 2", "1 # => 2" assert_format "1 #=>2", "1 # => 2" @@ -2273,11 +2111,11 @@ describe Crystal::Formatter do assert_format "def foo(a,\n *b)\nend" assert_format "def foo(a, # comment\n *b)\nend", "def foo(a, # comment\n *b)\nend" assert_format "def foo(a,\n **b)\nend" - assert_format "def foo(\n **a\n)\n 1\nend" + assert_format "def foo(\n **a\n)\n 1\nend", "def foo(\n **a,\n)\n 1\nend" assert_format "def foo(**a,)\n 1\nend", "def foo(**a)\n 1\nend" - assert_format "def foo(\n **a # comment\n)\n 1\nend" - assert_format "def foo(\n **a\n # comment\n)\n 1\nend" - assert_format "def foo(\n **a\n\n # comment\n)\n 1\nend" + assert_format "def foo(\n **a # comment\n)\n 1\nend", "def foo(\n **a, # comment\n)\n 1\nend" + assert_format "def foo(\n **a\n # comment\n)\n 1\nend", "def foo(\n **a,\n # comment\n)\n 1\nend" + assert_format "def foo(\n **a\n\n # comment\n)\n 1\nend", "def foo(\n **a,\n\n # comment\n)\n 1\nend" assert_format "def foo(**b, # comment\n &block)\nend" assert_format "def foo(a, **b, # comment\n &block)\nend" @@ -2332,7 +2170,7 @@ describe Crystal::Formatter do assert_format "alias X = ((Y, Z) ->)" - assert_format "def x(@y = ->(z) {})\nend" + assert_format "def x(@y = ->(z) {})\nend", "def x(@y = ->(z) { })\nend" assert_format "class X; annotation FooAnnotation ; end ; end", "class X\n annotation FooAnnotation; end\nend" assert_format "class X\n annotation FooAnnotation \n end \n end", "class X\n annotation FooAnnotation\n end\nend" @@ -2742,13 +2580,19 @@ describe Crystal::Formatter do assert_format "a &.a.!" assert_format "a &.!.!" - assert_format <<-CRYSTAL + assert_format <<-CRYSTAL, <<-CRYSTAL ->{ # first comment puts "hi" # second comment } CRYSTAL + -> { + # first comment + puts "hi" + # second comment + } + CRYSTAL # #9014 assert_format <<-CRYSTAL diff --git a/spec/llvm-ir/pass-closure-to-c-debug-loc.cr b/spec/llvm-ir/pass-closure-to-c-debug-loc.cr index a6031798b607..6891ae6ae92f 100644 --- a/spec/llvm-ir/pass-closure-to-c-debug-loc.cr +++ b/spec/llvm-ir/pass-closure-to-c-debug-loc.cr @@ -8,7 +8,7 @@ def raise(msg) end x = 1 -f = ->{ x } +f = -> { x } Foo.foo(f) # CHECK: define internal i8* @"~check_proc_is_not_closure"(%"->" %0) diff --git a/spec/llvm-ir/proc-call-debug-loc.cr b/spec/llvm-ir/proc-call-debug-loc.cr index e83c814f723b..61f02249a9a9 100644 --- a/spec/llvm-ir/proc-call-debug-loc.cr +++ b/spec/llvm-ir/proc-call-debug-loc.cr @@ -1,4 +1,4 @@ -x = ->{} +x = -> { } x.call # CHECK: extractvalue %"->" %{{[0-9]+}}, 0 # CHECK-SAME: !dbg [[LOC:![0-9]+]] diff --git a/spec/std/benchmark_spec.cr b/spec/std/benchmark_spec.cr index 4a46798b2436..63124881c262 100644 --- a/spec/std/benchmark_spec.cr +++ b/spec/std/benchmark_spec.cr @@ -31,7 +31,7 @@ describe Benchmark::IPS::Job do end private def create_entry - Benchmark::IPS::Entry.new("label", ->{ 1 + 1 }) + Benchmark::IPS::Entry.new("label", -> { 1 + 1 }) end private def h_mean(mean) diff --git a/spec/std/channel_spec.cr b/spec/std/channel_spec.cr index 69161dd96e01..a24790dd8dea 100644 --- a/spec/std/channel_spec.cr +++ b/spec/std/channel_spec.cr @@ -82,7 +82,7 @@ describe Channel do context "receive raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String) @@ -92,7 +92,7 @@ describe Channel do it "types nilable channel" do # Yes, although it is discouraged ch = Channel(Nil).new - spawn_and_wait(->{ ch.send nil }) do + spawn_and_wait(-> { ch.send nil }) do i, m = Channel.select(ch.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -101,7 +101,7 @@ describe Channel do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action) end @@ -110,7 +110,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action) end @@ -120,7 +120,7 @@ describe Channel do it "awakes all waiting selects" do ch = Channel(String).new - p = ->{ + p = -> { begin Channel.select(ch.receive_select_action) 0 @@ -129,7 +129,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end @@ -140,7 +140,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, ch2.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String | Bool) @@ -151,7 +151,7 @@ describe Channel do context "receive nil-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(String | Nil) @@ -161,7 +161,7 @@ describe Channel do it "types nilable channel" do # Yes, although it is discouraged ch = Channel(Nil).new - spawn_and_wait(->{ ch.send nil }) do + spawn_and_wait(-> { ch.send nil }) do i, m = Channel.select(ch.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -170,7 +170,7 @@ describe Channel do it "returns nil if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.select(ch.receive_select_action?) m.should be_nil end @@ -178,7 +178,7 @@ describe Channel do it "returns nil channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do i, m = Channel.select(ch.receive_select_action?) m.should be_nil end @@ -187,11 +187,11 @@ describe Channel do it "awakes all waiting selects" do ch = Channel(String).new - p = ->{ + p = -> { Channel.select(ch.receive_select_action?) } - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({ {0, nil}, {0, nil}, {0, nil}, {0, nil} }) end @@ -202,7 +202,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action?, ch2.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(String | Bool | Nil) @@ -212,7 +212,7 @@ describe Channel do it "returns index of closed channel" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch2.close }) do + spawn_and_wait(-> { ch2.close }) do i, m = Channel.select(ch.receive_select_action?, ch2.receive_select_action?) i.should eq(1) m.should eq(nil) @@ -224,7 +224,7 @@ describe Channel do it "raises if receive channel was closed and receive? channel was not ready" do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action, ch2.receive_select_action?) end @@ -234,7 +234,7 @@ describe Channel do it "returns nil if receive channel was not ready and receive? channel was closed" do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch2.close }) do + spawn_and_wait(-> { ch2.close }) do i, m = Channel.select(ch.receive_select_action, ch2.receive_select_action?) i.should eq(1) m.should eq(nil) @@ -245,7 +245,7 @@ describe Channel do context "send raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.select(ch.send_select_action("foo")) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -255,7 +255,7 @@ describe Channel do it "types nilable channel" do # Yes, although it is discouraged ch = Channel(Nil).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.select(ch.send_select_action(nil)) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -264,7 +264,7 @@ describe Channel do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo")) end @@ -273,7 +273,7 @@ describe Channel do it "raises if channel is closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo")) end @@ -283,7 +283,7 @@ describe Channel do it "awakes all waiting selects" do ch = Channel(String).new - p = ->{ + p = -> { begin Channel.select(ch.send_select_action("foo")) 0 @@ -292,7 +292,7 @@ describe Channel do end } - spawn_and_wait(->{ sleep 0.2.seconds; ch.close }) do + spawn_and_wait(-> { sleep 0.2.seconds; ch.close }) do r = parallel p.call, p.call, p.call, p.call r.should eq({1, 1, 1, 1}) end @@ -303,7 +303,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.select(ch.send_select_action("foo"), ch2.send_select_action(true)) typeof(i).should eq(Int32) typeof(m).should eq(Nil) @@ -314,7 +314,7 @@ describe Channel do context "timeout" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) typeof(i).should eq(Int32) typeof(m).should eq(String?) @@ -323,7 +323,7 @@ describe Channel do it "triggers timeout" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) i.should eq(1) @@ -333,7 +333,7 @@ describe Channel do it "triggers timeout (reverse order)" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.select(timeout_select_action(0.1.seconds), ch.receive_select_action) i.should eq(0) @@ -343,7 +343,7 @@ describe Channel do it "triggers timeout (same fiber multiple times)" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do 3.times do i, m = Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) @@ -355,7 +355,7 @@ describe Channel do it "allows receiving while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(1.seconds)) i.should eq(0) m.should eq("foo") @@ -364,7 +364,7 @@ describe Channel do it "allows receiving while waiting (reverse order)" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(timeout_select_action(1.seconds), ch.receive_select_action) i.should eq(1) m.should eq("foo") @@ -373,7 +373,7 @@ describe Channel do it "allows receiving while waiting (same fiber multiple times)" do ch = Channel(String).new - spawn_and_wait(->{ 3.times { ch.send "foo" } }) do + spawn_and_wait(-> { 3.times { ch.send "foo" } }) do 3.times do i, m = Channel.select(ch.receive_select_action, timeout_select_action(1.seconds)) i.should eq(0) @@ -384,7 +384,7 @@ describe Channel do it "negative amounts should not trigger timeout" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.select(ch.receive_select_action, timeout_select_action(-1.seconds)) i.should eq(0) @@ -394,7 +394,7 @@ describe Channel do it "send raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.send_select_action("foo"), timeout_select_action(0.1.seconds)) end @@ -403,7 +403,7 @@ describe Channel do it "receive raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.select(ch.receive_select_action, timeout_select_action(0.1.seconds)) end @@ -412,7 +412,7 @@ describe Channel do it "receive nil-on-close returns index of closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.select(ch.receive_select_action?, timeout_select_action(0.1.seconds)) i.should eq(0) @@ -426,7 +426,7 @@ describe Channel do context "receive raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String | Channel::NotReady) @@ -438,7 +438,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action, ch2.receive_select_action) typeof(i).should eq(Int32) typeof(m).should eq(String | Bool | Channel::NotReady) @@ -449,7 +449,7 @@ describe Channel do context "receive nil-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action?) typeof(i).should eq(Int32) typeof(m).should eq(String | Nil | Channel::NotReady) @@ -458,7 +458,7 @@ describe Channel do it "returns nil if channel was closed" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.non_blocking_select(ch.receive_select_action?) m.should be_nil end @@ -470,7 +470,7 @@ describe Channel do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.non_blocking_select(ch.receive_select_action, ch2.receive_select_action?) end @@ -480,7 +480,7 @@ describe Channel do it "returns nil if receive channel was not ready and receive? channel was closed" do ch = Channel(String).new ch2 = Channel(String).new - spawn_and_wait(->{ ch2.close }) do + spawn_and_wait(-> { ch2.close }) do i, m = Channel.non_blocking_select(ch.receive_select_action, ch2.receive_select_action?) i.should eq(1) m.should eq(nil) @@ -491,7 +491,7 @@ describe Channel do context "send raise-on-close single-channel" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.non_blocking_select(ch.send_select_action("foo")) typeof(i).should eq(Int32) typeof(m).should eq(Nil | Channel::NotReady) @@ -503,7 +503,7 @@ describe Channel do it "types" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_wait(->{ ch.receive }) do + spawn_and_wait(-> { ch.receive }) do i, m = Channel.non_blocking_select(ch.send_select_action("foo"), ch2.send_select_action(true)) typeof(i).should eq(Int32) typeof(m).should eq(Nil | Channel::NotReady) @@ -514,7 +514,7 @@ describe Channel do context "timeout" do it "types" do ch = Channel(String).new - spawn_and_wait(->{ ch.send "foo" }) do + spawn_and_wait(-> { ch.send "foo" }) do i, m = Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(0.1.seconds)) typeof(i).should eq(Int32) typeof(m).should eq(String | Nil | Channel::NotReady) @@ -523,7 +523,7 @@ describe Channel do it "should not trigger timeout" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(0.1.seconds)) i.should eq(2) @@ -533,7 +533,7 @@ describe Channel do it "negative amounts should not trigger timeout" do ch = Channel(String).new - spawn_and_wait(->{}) do + spawn_and_wait(-> { }) do i, m = Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(-1.seconds)) i.should eq(2) @@ -543,7 +543,7 @@ describe Channel do it "send raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.non_blocking_select(ch.send_select_action("foo"), timeout_select_action(0.1.seconds)) end @@ -552,7 +552,7 @@ describe Channel do it "receive raise-on-close raises if channel was closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do expect_raises Channel::ClosedError do Channel.non_blocking_select(ch.receive_select_action, timeout_select_action(0.1.seconds)) end @@ -561,7 +561,7 @@ describe Channel do it "receive nil-on-close returns index of closed while waiting" do ch = Channel(String).new - spawn_and_wait(->{ ch.close }) do + spawn_and_wait(-> { ch.close }) do i, m = Channel.non_blocking_select(ch.receive_select_action?, timeout_select_action(0.1.seconds)) i.should eq(0) @@ -573,7 +573,7 @@ describe Channel do it "returns correct index for array argument" do ch = [Channel(String).new, Channel(String).new, Channel(String).new] channels = [ch[0], ch[2], ch[1]] # shuffle around to get non-sequential lock_object_ids - spawn_and_wait(->{ channels[0].send "foo" }) do + spawn_and_wait(-> { channels[0].send "foo" }) do i, m = Channel.non_blocking_select(channels.map(&.receive_select_action)) i.should eq(0) diff --git a/spec/std/concurrent/select_spec.cr b/spec/std/concurrent/select_spec.cr index 5285e3dd070c..4f84734a20ad 100644 --- a/spec/std/concurrent/select_spec.cr +++ b/spec/std/concurrent/select_spec.cr @@ -243,7 +243,7 @@ describe "select" do it "types and exec when" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -259,7 +259,7 @@ describe "select" do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -276,7 +276,7 @@ describe "select" do it "types and exec when if message was ready" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -290,7 +290,7 @@ describe "select" do it "exec else if no message was ready" do ch = Channel(String).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive else @@ -305,7 +305,7 @@ describe "select" do it "raises if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -324,7 +324,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -339,7 +339,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive when m = ch2.receive @@ -357,7 +357,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -373,7 +373,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| begin select when m = ch.receive @@ -392,7 +392,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive w.check @@ -408,7 +408,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive when m = ch2.receive @@ -424,7 +424,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive when m = ch2.receive @@ -441,7 +441,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| begin select when m = ch.receive @@ -458,7 +458,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| begin select when m = ch.receive @@ -477,7 +477,7 @@ describe "select" do it "types and exec when" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -490,7 +490,7 @@ describe "select" do it "types and exec when with nil if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -506,7 +506,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -521,7 +521,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -536,7 +536,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -551,7 +551,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -566,7 +566,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -581,7 +581,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -597,7 +597,7 @@ describe "select" do it "types and exec when" do ch = Channel(String).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -611,7 +611,7 @@ describe "select" do it "exec else if no message was ready" do ch = Channel(String).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive? else @@ -623,7 +623,7 @@ describe "select" do it "types and exec when with nil if channel was closed" do ch = Channel(String).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -640,7 +640,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.send "foo" }) do |w| + spawn_and_check(-> { ch.send "foo" }) do |w| select when m = ch.receive? w.check @@ -656,7 +656,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.send true }) do |w| + spawn_and_check(-> { ch2.send true }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -672,7 +672,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -688,7 +688,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -704,7 +704,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch.close }) do |w| + spawn_and_check(-> { ch.close }) do |w| select when m = ch.receive? w.check @@ -720,7 +720,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ ch2.close }) do |w| + spawn_and_check(-> { ch2.close }) do |w| select when m = ch.receive? when m = ch2.receive? @@ -736,7 +736,7 @@ describe "select" do ch = Channel(String).new ch2 = Channel(Bool).new - spawn_and_check(->{ nil }) do |w| + spawn_and_check(-> { nil }) do |w| select when m = ch.receive? when m = ch2.receive? diff --git a/spec/std/http/server/handlers/log_handler_spec.cr b/spec/std/http/server/handlers/log_handler_spec.cr index 1f94649f09a8..3f33120e03d6 100644 --- a/spec/std/http/server/handlers/log_handler_spec.cr +++ b/spec/std/http/server/handlers/log_handler_spec.cr @@ -28,7 +28,7 @@ describe HTTP::LogHandler do backend = Log::MemoryBackend.new log = Log.new("custom", backend, :info) handler = HTTP::LogHandler.new(log) - handler.next = ->(ctx : HTTP::Server::Context) {} + handler.next = ->(ctx : HTTP::Server::Context) { } handler.call(context) logs = Log::EntriesChecker.new(backend.entries) diff --git a/spec/std/openssl/ssl/socket_spec.cr b/spec/std/openssl/ssl/socket_spec.cr index 47374ce28cca..ed1150407122 100644 --- a/spec/std/openssl/ssl/socket_spec.cr +++ b/spec/std/openssl/ssl/socket_spec.cr @@ -75,7 +75,7 @@ describe OpenSSL::SSL::Socket do server_tests: ->(client : Server) { client.cipher.should_not be_empty }, - client_tests: ->(client : Client) {} + client_tests: ->(client : Client) { } ) end @@ -84,7 +84,7 @@ describe OpenSSL::SSL::Socket do server_tests: ->(client : Server) { client.tls_version.should contain "TLS" }, - client_tests: ->(client : Client) {} + client_tests: ->(client : Client) { } ) end diff --git a/spec/std/proc_spec.cr b/spec/std/proc_spec.cr index 87bea44c0422..f378d768fbef 100644 --- a/spec/std/proc_spec.cr +++ b/spec/std/proc_spec.cr @@ -28,19 +28,19 @@ describe "Proc" do end it "gets pointer" do - f = ->{ 1 } + f = -> { 1 } f.pointer.address.should be > 0 end it "gets closure data for non-closure" do - f = ->{ 1 } + f = -> { 1 } f.closure_data.address.should eq(0) f.closure?.should be_false end it "gets closure data for closure" do a = 1 - f = ->{ a } + f = -> { a } f.closure_data.address.should be > 0 f.closure?.should be_true end @@ -53,19 +53,19 @@ describe "Proc" do end it "does ==" do - func = ->{ 1 } + func = -> { 1 } func.should eq(func) - func2 = ->{ 1 } + func2 = -> { 1 } func2.should_not eq(func) end it "clones" do - func = ->{ 1 } + func = -> { 1 } func.clone.should eq(func) end it "#arity" do - f = ->(x : Int32, y : Int32) {} + f = ->(x : Int32, y : Int32) { } f.arity.should eq(2) end @@ -89,5 +89,5 @@ describe "Proc" do f2.call('r').should eq(2) end - typeof(->{ 1 }.hash) + typeof(-> { 1 }.hash) end diff --git a/src/compiler/crystal/command/format.cr b/src/compiler/crystal/command/format.cr index ed63a26796f9..9d0431b3e3bb 100644 --- a/src/compiler/crystal/command/format.cr +++ b/src/compiler/crystal/command/format.cr @@ -78,7 +78,7 @@ class Crystal::Command @show_backtrace : Bool = false, @color : Bool = true, # stdio is injectable for testing - @stdin : IO = STDIN, @stdout : IO = STDOUT, @stderr : IO = STDERR + @stdin : IO = STDIN, @stdout : IO = STDOUT, @stderr : IO = STDERR, ) @format_stdin = files.size == 1 && files[0] == "-" diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 0d7ba0ff12f9..f620fe2fb312 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -669,7 +669,7 @@ module Crystal end end - private def fork_workers(n_threads) + private def fork_workers(n_threads, &) workers = [] of {Int32, IO::FileDescriptor, IO::FileDescriptor} n_threads.times do diff --git a/src/compiler/crystal/ffi/lib_ffi.cr b/src/compiler/crystal/ffi/lib_ffi.cr index 97163c989ee5..2d08cf4e18dd 100644 --- a/src/compiler/crystal/ffi/lib_ffi.cr +++ b/src/compiler/crystal/ffi/lib_ffi.cr @@ -147,7 +147,7 @@ module Crystal abi : ABI, nargs : LibC::UInt, rtype : Type*, - atypes : Type** + atypes : Type**, ) : Status fun prep_cif_var = ffi_prep_cif_var( @@ -156,7 +156,7 @@ module Crystal nfixedargs : LibC::UInt, varntotalargs : LibC::UInt, rtype : Type*, - atypes : Type** + atypes : Type**, ) : Status @[Raises] @@ -164,7 +164,7 @@ module Crystal cif : Cif*, fn : Void*, rvalue : Void*, - avalue : Void** + avalue : Void**, ) : Void fun closure_alloc = ffi_closure_alloc(size : LibC::SizeT, code : Void**) : Closure* @@ -174,7 +174,7 @@ module Crystal cif : Cif*, fun : ClosureFun, user_data : Void*, - code_loc : Void* + code_loc : Void*, ) : Status end end diff --git a/src/compiler/crystal/interpreter/closure_context.cr b/src/compiler/crystal/interpreter/closure_context.cr index 5df87d884363..4e633ae104b4 100644 --- a/src/compiler/crystal/interpreter/closure_context.cr +++ b/src/compiler/crystal/interpreter/closure_context.cr @@ -20,7 +20,7 @@ class Crystal::Repl @vars : Hash(String, {Int32, Type}), @self_type : Type?, @parent : ClosureContext?, - @bytesize : Int32 + @bytesize : Int32, ) end end diff --git a/src/compiler/crystal/interpreter/compiled_def.cr b/src/compiler/crystal/interpreter/compiled_def.cr index 8bfc3252fcb9..f9d3d48088bd 100644 --- a/src/compiler/crystal/interpreter/compiled_def.cr +++ b/src/compiler/crystal/interpreter/compiled_def.cr @@ -26,7 +26,7 @@ class Crystal::Repl @owner : Type, @args_bytesize : Int32, @instructions : CompiledInstructions = CompiledInstructions.new, - @local_vars = LocalVars.new(context) + @local_vars = LocalVars.new(context), ) end end diff --git a/src/compiler/crystal/interpreter/compiler.cr b/src/compiler/crystal/interpreter/compiler.cr index 50024d8b65e3..ea278876c44f 100644 --- a/src/compiler/crystal/interpreter/compiler.cr +++ b/src/compiler/crystal/interpreter/compiler.cr @@ -103,7 +103,7 @@ class Crystal::Repl::Compiler < Crystal::Visitor @instructions : CompiledInstructions = CompiledInstructions.new, scope : Type? = nil, @def = nil, - @top_level = true + @top_level = true, ) @scope = scope || @context.program @@ -138,7 +138,7 @@ class Crystal::Repl::Compiler < Crystal::Visitor context : Context, compiled_def : CompiledDef, top_level : Bool, - scope : Type = compiled_def.owner + scope : Type = compiled_def.owner, ) new( context: context, diff --git a/src/compiler/crystal/interpreter/interpreter.cr b/src/compiler/crystal/interpreter/interpreter.cr index f73cba958851..e26a6751c176 100644 --- a/src/compiler/crystal/interpreter/interpreter.cr +++ b/src/compiler/crystal/interpreter/interpreter.cr @@ -113,7 +113,7 @@ class Crystal::Repl::Interpreter def initialize( @context : Context, # TODO: what if the stack is exhausted? - @stack : UInt8* = Pointer(Void).malloc(8 * 1024 * 1024).as(UInt8*) + @stack : UInt8* = Pointer(Void).malloc(8 * 1024 * 1024).as(UInt8*), ) @local_vars = LocalVars.new(@context) @argv = [] of String diff --git a/src/compiler/crystal/interpreter/lib_function.cr b/src/compiler/crystal/interpreter/lib_function.cr index 54ac2ac297cf..e1898869227e 100644 --- a/src/compiler/crystal/interpreter/lib_function.cr +++ b/src/compiler/crystal/interpreter/lib_function.cr @@ -19,7 +19,7 @@ class Crystal::Repl::LibFunction @def : External, @symbol : Void*, @call_interface : FFI::CallInterface, - @args_bytesizes : Array(Int32) + @args_bytesizes : Array(Int32), ) end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index c262a2d9770a..bab4e22b9fba 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -506,7 +506,7 @@ module Crystal recorded_requires << RecordedRequire.new(filename, relative_to) end - def run_requires(node : Require, filenames) : Nil + def run_requires(node : Require, filenames, &) : Nil dependency_printer = compiler.try(&.dependency_printer) filenames.each do |filename| diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index ea5626c37f94..905d5bac8cb1 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -2672,7 +2672,7 @@ module Crystal end end - private def visit_size_or_align_of(node) + private def visit_size_or_align_of(node, &) @in_type_args += 1 node.exp.accept self @in_type_args -= 1 @@ -2698,7 +2698,7 @@ module Crystal false end - private def visit_instance_size_or_align_of(node) + private def visit_instance_size_or_align_of(node, &) @in_type_args += 1 node.exp.accept self @in_type_args -= 1 diff --git a/src/compiler/crystal/semantic/type_merge.cr b/src/compiler/crystal/semantic/type_merge.cr index d68cdeb38a99..874949dd516d 100644 --- a/src/compiler/crystal/semantic/type_merge.cr +++ b/src/compiler/crystal/semantic/type_merge.cr @@ -207,7 +207,7 @@ module Crystal def self.least_common_ancestor( type1 : MetaclassType | GenericClassInstanceMetaclassType, - type2 : MetaclassType | GenericClassInstanceMetaclassType + type2 : MetaclassType | GenericClassInstanceMetaclassType, ) return nil unless unifiable_metaclass?(type1) && unifiable_metaclass?(type2) @@ -225,7 +225,7 @@ module Crystal def self.least_common_ancestor( type1 : NonGenericModuleType | GenericModuleInstanceType | GenericClassType, - type2 : NonGenericModuleType | GenericModuleInstanceType | GenericClassType + type2 : NonGenericModuleType | GenericModuleInstanceType | GenericClassType, ) return type2 if type1.implements?(type2) return type1 if type2.implements?(type1) diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 796afe0730de..7ea32627078e 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -1476,7 +1476,7 @@ module Crystal # this formats `def foo # ...` to `def foo(&) # ...` for yielding # methods before consuming the comment line if node.block_arity && node.args.empty? && !node.block_arg && !node.double_splat - write "(&)" if flag?("method_signature_yield") + write "(&)" end skip_space consume_newline: false @@ -1523,7 +1523,7 @@ module Crystal end def format_def_args(node : Def | Macro) - yields = node.is_a?(Def) && !node.block_arity.nil? && flag?("method_signature_yield") + yields = node.is_a?(Def) && !node.block_arity.nil? format_def_args node.args, node.block_arg, node.splat_index, false, node.double_splat, yields end @@ -1651,7 +1651,7 @@ module Crystal yield # Write "," before skipping spaces to prevent inserting comment between argument and comma. - write "," if has_more || (wrote_newline && @token.type.op_comma?) || (write_trailing_comma && flag?("def_trailing_comma")) + write "," if has_more || (wrote_newline && @token.type.op_comma?) || write_trailing_comma just_wrote_newline = skip_space if @token.type.newline? @@ -1681,7 +1681,7 @@ module Crystal elsif @token.type.op_rparen? && has_more && !just_wrote_newline # if we found a `)` and there are still more parameters to write, it # must have been a missing `&` for a def that yields - write " " if flag?("method_signature_yield") + write " " end just_wrote_newline @@ -4273,7 +4273,7 @@ module Crystal skip_space_or_newline end - write " " if a_def.args.present? || return_type || flag?("proc_literal_whitespace") || whitespace_after_op_minus_gt + write " " is_do = false if @token.keyword?(:do) @@ -4281,7 +4281,7 @@ module Crystal is_do = true else write_token :OP_LCURLY - write " " if a_def.body.is_a?(Nop) && (flag?("proc_literal_whitespace") || @token.type.space?) + write " " if a_def.body.is_a?(Nop) end skip_space diff --git a/src/compiler/crystal/tools/init.cr b/src/compiler/crystal/tools/init.cr index 96b004eec2fd..01b2e137c578 100644 --- a/src/compiler/crystal/tools/init.cr +++ b/src/compiler/crystal/tools/init.cr @@ -157,7 +157,7 @@ module Crystal @github_name = "none", @silent = false, @force = false, - @skip_existing = false + @skip_existing = false, ) end diff --git a/src/compiler/crystal/util.cr b/src/compiler/crystal/util.cr index c33bfa5d0d42..d0de6f226f36 100644 --- a/src/compiler/crystal/util.cr +++ b/src/compiler/crystal/util.cr @@ -41,7 +41,7 @@ module Crystal source : String | Array(String), highlight_line_number = nil, color = false, - line_number_start = 1 + line_number_start = 1, ) source = source.lines if source.is_a? String line_number_padding = (source.size + line_number_start).to_s.chars.size diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index 431708c5cc11..0d6f5077633a 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -91,7 +91,7 @@ class Thread # Used once to initialize the thread object representing the main thread of # the process (that already exists). def initialize - @func = ->(t : Thread) {} + @func = ->(t : Thread) { } @system_handle = Crystal::System::Thread.current_handle @current_fiber = @main_fiber = Fiber.new(stack_address, self) diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index d3655fdb5861..ade1862a780c 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -121,7 +121,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop return unless thread # alert the thread to interrupt GetQueuedCompletionStatusEx - LibC.QueueUserAPC(->(ptr : LibC::ULONG_PTR) {}, thread, LibC::ULONG_PTR.new(0)) + LibC.QueueUserAPC(->(ptr : LibC::ULONG_PTR) { }, thread, LibC::ULONG_PTR.new(0)) end def enqueue(event : Crystal::IOCP::Event) diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index cdd23e3ed54d..1f277505302a 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -194,7 +194,7 @@ module Crystal::System::FileDescriptor file_descriptor_close end - def file_descriptor_close + def file_descriptor_close(&) if LibC.CloseHandle(windows_handle) == 0 yield end diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 5ed235e24574..bfb82581204b 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -369,7 +369,7 @@ module Crystal::System::Socket socket_close end - private def socket_close + private def socket_close(&) handle = @volatile_fd.swap(LibC::INVALID_SOCKET) ret = LibC.closesocket(handle) diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 0ce6a1366b6d..41c0f43f2a8c 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -198,7 +198,7 @@ module GC {% end %} LibGC.init - LibGC.set_start_callback ->do + LibGC.set_start_callback -> do GC.lock_write end @@ -449,7 +449,7 @@ module GC @@curr_push_other_roots = block @@prev_push_other_roots = LibGC.get_push_other_roots - LibGC.set_push_other_roots ->do + LibGC.set_push_other_roots -> do @@curr_push_other_roots.try(&.call) @@prev_push_other_roots.try(&.call) end diff --git a/src/kernel.cr b/src/kernel.cr index 16c4a770309a..ac241161c16d 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -585,7 +585,7 @@ end def self.after_fork_child_callbacks @@after_fork_child_callbacks ||= [ # reinit event loop first: - ->{ Crystal::EventLoop.current.after_fork }, + -> { Crystal::EventLoop.current.after_fork }, # reinit signal handling: ->Crystal::System::Signal.after_fork, diff --git a/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr b/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr index fe2fbe381d03..7f7160a6448b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr @@ -19,7 +19,7 @@ lib LibC lpBuffer : Void*, nNumberOfCharsToRead : DWORD, lpNumberOfCharsRead : DWORD*, - pInputControl : Void* + pInputControl : Void*, ) : BOOL CTRL_C_EVENT = 0 diff --git a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr index 2c62d07d3ad8..abd9e0b36104 100644 --- a/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr +++ b/src/lib_c/x86_64-windows-msvc/c/dbghelp.cr @@ -132,6 +132,6 @@ lib LibC fun StackWalk64( machineType : DWORD, hProcess : HANDLE, hThread : HANDLE, stackFrame : STACKFRAME64*, contextRecord : Void*, readMemoryRoutine : PREAD_PROCESS_MEMORY_ROUTINE64, functionTableAccessRoutine : PFUNCTION_TABLE_ACCESS_ROUTINE64, - getModuleBaseRoutine : PGET_MODULE_BASE_ROUTINE64, translateAddress : PTRANSLATE_ADDRESS_ROUTINE64 + getModuleBaseRoutine : PGET_MODULE_BASE_ROUTINE64, translateAddress : PTRANSLATE_ADDRESS_ROUTINE64, ) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/fileapi.cr b/src/lib_c/x86_64-windows-msvc/c/fileapi.cr index c17c0fb48a9a..94714b557cbe 100644 --- a/src/lib_c/x86_64-windows-msvc/c/fileapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/fileapi.cr @@ -107,14 +107,14 @@ lib LibC dwReserved : DWORD, nNumberOfBytesToLockLow : DWORD, nNumberOfBytesToLockHigh : DWORD, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun UnlockFileEx( hFile : HANDLE, dwReserved : DWORD, nNumberOfBytesToUnlockLow : DWORD, nNumberOfBytesToUnlockHigh : DWORD, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun SetFileTime(hFile : HANDLE, lpCreationTime : FILETIME*, lpLastAccessTime : FILETIME*, lpLastWriteTime : FILETIME*) : BOOL diff --git a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr index f6d56ef5a0e6..d6632e329f6b 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr @@ -3,14 +3,14 @@ lib LibC hFile : HANDLE, lpOverlapped : OVERLAPPED*, lpNumberOfBytesTransferred : DWORD*, - bWait : BOOL + bWait : BOOL, ) : BOOL fun CreateIoCompletionPort( fileHandle : HANDLE, existingCompletionPort : HANDLE, completionKey : ULong*, - numberOfConcurrentThreads : DWORD + numberOfConcurrentThreads : DWORD, ) : HANDLE fun GetQueuedCompletionStatusEx( @@ -19,22 +19,22 @@ lib LibC ulCount : ULong, ulNumEntriesRemoved : ULong*, dwMilliseconds : DWORD, - fAlertable : BOOL + fAlertable : BOOL, ) : BOOL fun PostQueuedCompletionStatus( completionPort : HANDLE, dwNumberOfBytesTransferred : DWORD, dwCompletionKey : ULONG_PTR, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun CancelIoEx( hFile : HANDLE, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL fun CancelIo( - hFile : HANDLE + hFile : HANDLE, ) : BOOL fun DeviceIoControl( @@ -45,6 +45,6 @@ lib LibC lpOutBuffer : Void*, nOutBufferSize : DWORD, lpBytesReturned : DWORD*, - lpOverlapped : OVERLAPPED* + lpOverlapped : OVERLAPPED*, ) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr b/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr index f60e80a59328..c22bd1dfab31 100644 --- a/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr @@ -8,13 +8,13 @@ lib LibC fun WideCharToMultiByte( codePage : UInt, dwFlags : DWORD, lpWideCharStr : LPWSTR, cchWideChar : Int, lpMultiByteStr : LPSTR, cbMultiByte : Int, - lpDefaultChar : CHAR*, lpUsedDefaultChar : BOOL* + lpDefaultChar : CHAR*, lpUsedDefaultChar : BOOL*, ) : Int # this was for the now removed delay-load helper, all other code should use # `String#to_utf16` instead fun MultiByteToWideChar( codePage : UInt, dwFlags : DWORD, lpMultiByteStr : LPSTR, - cbMultiByte : Int, lpWideCharStr : LPWSTR, cchWideChar : Int + cbMultiByte : Int, lpWideCharStr : LPWSTR, cchWideChar : Int, ) : Int end diff --git a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr index 68ce6f9ef421..21ae8baba852 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winsock2.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winsock2.cr @@ -154,7 +154,7 @@ lib LibC addr : Sockaddr*, addrlen : Int*, lpfnCondition : LPCONDITIONPROC, - dwCallbackData : DWORD* + dwCallbackData : DWORD*, ) : SOCKET fun WSAConnect( @@ -164,21 +164,21 @@ lib LibC lpCallerData : WSABUF*, lpCalleeData : WSABUF*, lpSQOS : LPQOS, - lpGQOS : LPQOS + lpGQOS : LPQOS, ) fun WSACreateEvent : WSAEVENT fun WSAEventSelect( s : SOCKET, hEventObject : WSAEVENT, - lNetworkEvents : Long + lNetworkEvents : Long, ) : Int fun WSAGetOverlappedResult( s : SOCKET, lpOverlapped : WSAOVERLAPPED*, lpcbTransfer : DWORD*, fWait : BOOL, - lpdwFlags : DWORD* + lpdwFlags : DWORD*, ) : BOOL fun WSAIoctl( s : SOCKET, @@ -189,7 +189,7 @@ lib LibC cbOutBuffer : DWORD, lpcbBytesReturned : DWORD*, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSARecv( s : SOCKET, @@ -198,7 +198,7 @@ lib LibC lpNumberOfBytesRecvd : DWORD*, lpFlags : DWORD*, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSARecvFrom( s : SOCKET, @@ -209,10 +209,10 @@ lib LibC lpFrom : Sockaddr*, lpFromlen : Int*, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSAResetEvent( - hEvent : WSAEVENT + hEvent : WSAEVENT, ) : BOOL fun WSASend( s : SOCKET, @@ -221,7 +221,7 @@ lib LibC lpNumberOfBytesSent : DWORD*, dwFlags : DWORD, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSASendTo( s : SOCKET, @@ -232,7 +232,7 @@ lib LibC lpTo : Sockaddr*, iTolen : Int, lpOverlapped : WSAOVERLAPPED*, - lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE* + lpCompletionRoutine : WSAOVERLAPPED_COMPLETION_ROUTINE*, ) : Int fun WSASocketW( af : Int, @@ -240,13 +240,13 @@ lib LibC protocol : Int, lpProtocolInfo : WSAPROTOCOL_INFOW*, g : GROUP, - dwFlags : DWORD + dwFlags : DWORD, ) : SOCKET fun WSAWaitForMultipleEvents( cEvents : DWORD, lphEvents : WSAEVENT*, fWaitAll : BOOL, dwTimeout : DWORD, - fAlertable : BOOL + fAlertable : BOOL, ) : DWORD end diff --git a/src/llvm/lib_llvm/debug_info.cr b/src/llvm/lib_llvm/debug_info.cr index e6155b317eb5..15d2eca3ebd6 100644 --- a/src/llvm/lib_llvm/debug_info.cr +++ b/src/llvm/lib_llvm/debug_info.cr @@ -14,7 +14,7 @@ lib LibLLVM builder : DIBuilderRef, lang : LLVM::DwarfSourceLanguage, file_ref : MetadataRef, producer : Char*, producer_len : SizeT, is_optimized : Bool, flags : Char*, flags_len : SizeT, runtime_ver : UInt, split_name : Char*, split_name_len : SizeT, kind : DWARFEmissionKind, dwo_id : UInt, - split_debug_inlining : Bool, debug_info_for_profiling : Bool + split_debug_inlining : Bool, debug_info_for_profiling : Bool, ) : MetadataRef {% else %} fun di_builder_create_compile_unit = LLVMDIBuilderCreateCompileUnit( @@ -22,82 +22,82 @@ lib LibLLVM producer_len : SizeT, is_optimized : Bool, flags : Char*, flags_len : SizeT, runtime_ver : UInt, split_name : Char*, split_name_len : SizeT, kind : DWARFEmissionKind, dwo_id : UInt, split_debug_inlining : Bool, debug_info_for_profiling : Bool, sys_root : Char*, - sys_root_len : SizeT, sdk : Char*, sdk_len : SizeT + sys_root_len : SizeT, sdk : Char*, sdk_len : SizeT, ) : MetadataRef {% end %} fun di_builder_create_file = LLVMDIBuilderCreateFile( builder : DIBuilderRef, filename : Char*, filename_len : SizeT, - directory : Char*, directory_len : SizeT + directory : Char*, directory_len : SizeT, ) : MetadataRef fun di_builder_create_function = LLVMDIBuilderCreateFunction( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, linkage_name : Char*, linkage_name_len : SizeT, file : MetadataRef, line_no : UInt, ty : MetadataRef, is_local_to_unit : Bool, is_definition : Bool, scope_line : UInt, - flags : LLVM::DIFlags, is_optimized : Bool + flags : LLVM::DIFlags, is_optimized : Bool, ) : MetadataRef fun di_builder_create_lexical_block = LLVMDIBuilderCreateLexicalBlock( - builder : DIBuilderRef, scope : MetadataRef, file : MetadataRef, line : UInt, column : UInt + builder : DIBuilderRef, scope : MetadataRef, file : MetadataRef, line : UInt, column : UInt, ) : MetadataRef fun di_builder_create_lexical_block_file = LLVMDIBuilderCreateLexicalBlockFile( - builder : DIBuilderRef, scope : MetadataRef, file_scope : MetadataRef, discriminator : UInt + builder : DIBuilderRef, scope : MetadataRef, file_scope : MetadataRef, discriminator : UInt, ) : MetadataRef fun di_builder_create_debug_location = LLVMDIBuilderCreateDebugLocation( - ctx : ContextRef, line : UInt, column : UInt, scope : MetadataRef, inlined_at : MetadataRef + ctx : ContextRef, line : UInt, column : UInt, scope : MetadataRef, inlined_at : MetadataRef, ) : MetadataRef fun di_builder_get_or_create_type_array = LLVMDIBuilderGetOrCreateTypeArray(builder : DIBuilderRef, types : MetadataRef*, length : SizeT) : MetadataRef fun di_builder_create_subroutine_type = LLVMDIBuilderCreateSubroutineType( builder : DIBuilderRef, file : MetadataRef, parameter_types : MetadataRef*, - num_parameter_types : UInt, flags : LLVM::DIFlags + num_parameter_types : UInt, flags : LLVM::DIFlags, ) : MetadataRef {% unless LibLLVM::IS_LT_90 %} fun di_builder_create_enumerator = LLVMDIBuilderCreateEnumerator( - builder : DIBuilderRef, name : Char*, name_len : SizeT, value : Int64, is_unsigned : Bool + builder : DIBuilderRef, name : Char*, name_len : SizeT, value : Int64, is_unsigned : Bool, ) : MetadataRef {% end %} fun di_builder_create_enumeration_type = LLVMDIBuilderCreateEnumerationType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_number : UInt, size_in_bits : UInt64, align_in_bits : UInt32, - elements : MetadataRef*, num_elements : UInt, class_ty : MetadataRef + elements : MetadataRef*, num_elements : UInt, class_ty : MetadataRef, ) : MetadataRef fun di_builder_create_union_type = LLVMDIBuilderCreateUnionType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_number : UInt, size_in_bits : UInt64, align_in_bits : UInt32, flags : LLVM::DIFlags, - elements : MetadataRef*, num_elements : UInt, run_time_lang : UInt, unique_id : Char*, unique_id_len : SizeT + elements : MetadataRef*, num_elements : UInt, run_time_lang : UInt, unique_id : Char*, unique_id_len : SizeT, ) : MetadataRef fun di_builder_create_array_type = LLVMDIBuilderCreateArrayType( builder : DIBuilderRef, size : UInt64, align_in_bits : UInt32, - ty : MetadataRef, subscripts : MetadataRef*, num_subscripts : UInt + ty : MetadataRef, subscripts : MetadataRef*, num_subscripts : UInt, ) : MetadataRef fun di_builder_create_unspecified_type = LLVMDIBuilderCreateUnspecifiedType(builder : DIBuilderRef, name : Char*, name_len : SizeT) : MetadataRef fun di_builder_create_basic_type = LLVMDIBuilderCreateBasicType( builder : DIBuilderRef, name : Char*, name_len : SizeT, size_in_bits : UInt64, - encoding : UInt, flags : LLVM::DIFlags + encoding : UInt, flags : LLVM::DIFlags, ) : MetadataRef fun di_builder_create_pointer_type = LLVMDIBuilderCreatePointerType( builder : DIBuilderRef, pointee_ty : MetadataRef, size_in_bits : UInt64, align_in_bits : UInt32, - address_space : UInt, name : Char*, name_len : SizeT + address_space : UInt, name : Char*, name_len : SizeT, ) : MetadataRef fun di_builder_create_struct_type = LLVMDIBuilderCreateStructType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_number : UInt, size_in_bits : UInt64, align_in_bits : UInt32, flags : LLVM::DIFlags, derived_from : MetadataRef, elements : MetadataRef*, num_elements : UInt, - run_time_lang : UInt, v_table_holder : MetadataRef, unique_id : Char*, unique_id_len : SizeT + run_time_lang : UInt, v_table_holder : MetadataRef, unique_id : Char*, unique_id_len : SizeT, ) : MetadataRef fun di_builder_create_member_type = LLVMDIBuilderCreateMemberType( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, line_no : UInt, size_in_bits : UInt64, align_in_bits : UInt32, offset_in_bits : UInt64, - flags : LLVM::DIFlags, ty : MetadataRef + flags : LLVM::DIFlags, ty : MetadataRef, ) : MetadataRef fun di_builder_create_replaceable_composite_type = LLVMDIBuilderCreateReplaceableCompositeType( builder : DIBuilderRef, tag : UInt, name : Char*, name_len : SizeT, scope : MetadataRef, file : MetadataRef, line : UInt, runtime_lang : UInt, size_in_bits : UInt64, align_in_bits : UInt32, - flags : LLVM::DIFlags, unique_identifier : Char*, unique_identifier_len : SizeT + flags : LLVM::DIFlags, unique_identifier : Char*, unique_identifier_len : SizeT, ) : MetadataRef fun di_builder_get_or_create_subrange = LLVMDIBuilderGetOrCreateSubrange(builder : DIBuilderRef, lo : Int64, count : Int64) : MetadataRef @@ -114,22 +114,22 @@ lib LibLLVM {% if LibLLVM::IS_LT_190 %} fun di_builder_insert_declare_at_end = LLVMDIBuilderInsertDeclareAtEnd( builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, - expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef, ) : ValueRef {% else %} fun di_builder_insert_declare_record_at_end = LLVMDIBuilderInsertDeclareRecordAtEnd( builder : DIBuilderRef, storage : ValueRef, var_info : MetadataRef, - expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef + expr : MetadataRef, debug_loc : MetadataRef, block : BasicBlockRef, ) : DbgRecordRef {% end %} fun di_builder_create_auto_variable = LLVMDIBuilderCreateAutoVariable( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, file : MetadataRef, - line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags, align_in_bits : UInt32 + line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags, align_in_bits : UInt32, ) : MetadataRef fun di_builder_create_parameter_variable = LLVMDIBuilderCreateParameterVariable( builder : DIBuilderRef, scope : MetadataRef, name : Char*, name_len : SizeT, arg_no : UInt, - file : MetadataRef, line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags + file : MetadataRef, line_no : UInt, ty : MetadataRef, always_preserve : Bool, flags : LLVM::DIFlags, ) : MetadataRef fun set_subprogram = LLVMSetSubprogram(func : ValueRef, sp : MetadataRef) diff --git a/src/llvm/lib_llvm/orc.cr b/src/llvm/lib_llvm/orc.cr index a1650b3dfb96..278a9c4aab5d 100644 --- a/src/llvm/lib_llvm/orc.cr +++ b/src/llvm/lib_llvm/orc.cr @@ -12,7 +12,7 @@ lib LibLLVM fun orc_create_dynamic_library_search_generator_for_process = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( result : OrcDefinitionGeneratorRef*, global_prefx : Char, - filter : OrcSymbolPredicate, filter_ctx : Void* + filter : OrcSymbolPredicate, filter_ctx : Void*, ) : ErrorRef fun orc_jit_dylib_add_generator = LLVMOrcJITDylibAddGenerator(jd : OrcJITDylibRef, dg : OrcDefinitionGeneratorRef) diff --git a/src/proc.cr b/src/proc.cr index fca714517dbf..69c0ebf5cd0e 100644 --- a/src/proc.cr +++ b/src/proc.cr @@ -3,7 +3,7 @@ # # ``` # # A proc without arguments -# ->{ 1 } # Proc(Int32) +# -> { 1 } # Proc(Int32) # # # A proc with one argument # ->(x : Int32) { x.to_s } # Proc(Int32, String) diff --git a/src/random/isaac.cr b/src/random/isaac.cr index c877cb9dbae9..294d439fb82d 100644 --- a/src/random/isaac.cr +++ b/src/random/isaac.cr @@ -61,7 +61,7 @@ class Random::ISAAC a = b = c = d = e = f = g = h = 0x9e3779b9_u32 - mix = ->{ + mix = -> { a ^= b << 11; d &+= a; b &+= c b ^= c >> 2; e &+= b; c &+= d c ^= d << 8; f &+= c; d &+= e diff --git a/src/wait_group.cr b/src/wait_group.cr index 89510714c727..c1ebe67bf508 100644 --- a/src/wait_group.cr +++ b/src/wait_group.cr @@ -52,7 +52,7 @@ class WaitGroup # end # end # ``` - def self.wait : Nil + def self.wait(&) : Nil instance = new yield instance instance.wait From d5600eb6dcc2ec67d1a7ac53cc3c0154221edcb8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 11 Oct 2024 19:55:29 +0800 Subject: [PATCH 150/193] Basic MinGW-w64 cross-compilation support (#15070) Resolves part of #6170. These series of patches allow `--cross-compile --target=x86_64-windows-gnu` to mostly work: * The `@[ThreadLocal]` annotation and its corresponding LLVM attribute seem to break when targetting `x86_64-windows-gnu`, so Win32 TLS is used instead. This is only needed for `Thread.current`. * Since MinGW uses `libgcc`, and Crystal relies on the underlying C++ ABI to raise exceptions, we use the Itanium ABI's `_Unwind_*` functions, along with a thin personality function wrapper. ([GCC itself does the same](https://github.com/gcc-mirror/gcc/blob/68afc7acf609be2b19ec05c8393c2ffc7f4adb4a/libgcc/unwind-c.c#L238-L246). See also [Language-specific handler](https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64?view=msvc-170#language-specific-handler) from the Windows x64 ABI docs.) * MinGW binaries now come with DWARF debug information, so they work under GDB, maybe under LLDB, and probably poorly under the Visual Studio Debugger. * There is no need to mangle symbol names the same way MSVC binaries do. * `--cross-compile` now prints ld-style linker flags, rather than MSVC ones. This is still incomplete and includes remnants of the MSVC toolchain like the `/ENTRY` flag; they will be fixed later once we get to native compiler builds. * `src/lib_c/x86_64-windows-gnu` is now a symlink to `src/lib_c/x86_64-windows-msvc`, since both toolchains are targetting the same Win32 APIs. (This is not Cygwin nor MSYS2's MSYS environment.) * Lib funs now use the Win32 C ABI, instead of the SysV ABI. * On MinGW we use GMP proper, and there is no need for MPIR. After building a local compiler, `bin\crystal build --cross-compile --target=x86_64-windows-gnu` will generate an object file suitable for linking under MinGW-w64. At a minimum, this object file depends on Boehm GC and libiconv, although they can be skipped using `-Dgc_none` and `-Dwithout_iconv` respectively. Then we could use MSYS2's UCRT64 environment to link the final executable: ``` $ pacman -Sy mingw-w64-ucrt-x86_64-gc mingw-w64-ucrt-x86_64-pcre2 mingw-w64-ucrt-x86_64-libiconv mingw-w64-ucrt-x86_64-gmp $ cc test.obj `pkg-config bdw-gc iconv libpcre2-8 gmp --libs` -lDbgHelp -lole32 ``` Stack traces do not work correctly yet. Also note that MSYS2's DLL names are different from the ones distributed with MSVC Crystal, and that cross-compilation never copies the DLL dependencies to the output directory. To make the executable run outside MSYS2, use `dumpbin /dependents` from the MSVC developer prompt to obtain the dependencies, then copy them manually from the MSYS2 `/ucrt64/bin` folder. --- spec/std/big/big_float_spec.cr | 44 ++++++++++++----- src/big/big_int.cr | 4 +- src/big/lib_gmp.cr | 29 +++++++----- src/compiler/crystal/codegen/codegen.cr | 4 +- src/compiler/crystal/codegen/debug.cr | 2 +- src/compiler/crystal/codegen/exception.cr | 17 +++---- src/compiler/crystal/codegen/link.cr | 2 +- src/crystal/system/win32/thread.cr | 47 +++++++++++++++++-- src/exception/call_stack.cr | 3 ++ src/exception/call_stack/stackwalk.cr | 27 +++++++++++ src/exception/lib_unwind.cr | 8 +++- src/lib_c/x86_64-windows-gnu | 1 + .../c/processthreadsapi.cr | 6 +++ src/llvm/target_machine.cr | 2 +- src/raise.cr | 29 ++++++++++-- 15 files changed, 173 insertions(+), 52 deletions(-) create mode 120000 src/lib_c/x86_64-windows-gnu diff --git a/spec/std/big/big_float_spec.cr b/spec/std/big/big_float_spec.cr index 23c782aa3de8..4aee9eee51e8 100644 --- a/spec/std/big/big_float_spec.cr +++ b/spec/std/big/big_float_spec.cr @@ -350,8 +350,11 @@ describe "BigFloat" do it { (2.to_big_f ** -7133786264).to_s.should end_with("e-2147483649") } # least power of two with a base-10 exponent less than Int32::MIN it { (10.to_big_f ** 3000000000 * 1.5).to_s.should end_with("e+3000000000") } it { (10.to_big_f ** -3000000000 * 1.5).to_s.should end_with("e-3000000000") } - it { (10.to_big_f ** 10000000000 * 1.5).to_s.should end_with("e+10000000000") } - it { (10.to_big_f ** -10000000000 * 1.5).to_s.should end_with("e-10000000000") } + + {% unless flag?(:win32) && flag?(:gnu) %} + it { (10.to_big_f ** 10000000000 * 1.5).to_s.should end_with("e+10000000000") } + it { (10.to_big_f ** -10000000000 * 1.5).to_s.should end_with("e-10000000000") } + {% end %} end describe "#inspect" do @@ -558,8 +561,12 @@ describe "BigFloat Math" do Math.ilogb(0.2.to_big_f).should eq(-3) Math.ilogb(123.45.to_big_f).should eq(6) Math.ilogb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000) - Math.ilogb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000) - Math.ilogb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.ilogb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000) + Math.ilogb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000) + {% end %} + expect_raises(ArgumentError) { Math.ilogb(0.to_big_f) } end @@ -567,8 +574,12 @@ describe "BigFloat Math" do Math.logb(0.2.to_big_f).should eq(-3.to_big_f) Math.logb(123.45.to_big_f).should eq(6.to_big_f) Math.logb(2.to_big_f ** 1_000_000_000).should eq(1_000_000_000.to_big_f) - Math.logb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000.to_big_f) - Math.logb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000.to_big_f) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.logb(2.to_big_f ** 100_000_000_000).should eq(100_000_000_000.to_big_f) + Math.logb(2.to_big_f ** -100_000_000_000).should eq(-100_000_000_000.to_big_f) + {% end %} + expect_raises(ArgumentError) { Math.logb(0.to_big_f) } end @@ -576,24 +587,33 @@ describe "BigFloat Math" do Math.ldexp(0.2.to_big_f, 2).should eq(0.8.to_big_f) Math.ldexp(0.2.to_big_f, -2).should eq(0.05.to_big_f) Math.ldexp(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) - Math.ldexp(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) - Math.ldexp(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.ldexp(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.ldexp(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + {% end %} end it ".scalbn" do Math.scalbn(0.2.to_big_f, 2).should eq(0.8.to_big_f) Math.scalbn(0.2.to_big_f, -2).should eq(0.05.to_big_f) Math.scalbn(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) - Math.scalbn(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) - Math.scalbn(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.scalbn(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbn(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + {% end %} end it ".scalbln" do Math.scalbln(0.2.to_big_f, 2).should eq(0.8.to_big_f) Math.scalbln(0.2.to_big_f, -2).should eq(0.05.to_big_f) Math.scalbln(1.to_big_f, 1_000_000_000).should eq(2.to_big_f ** 1_000_000_000) - Math.scalbln(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) - Math.scalbln(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + + {% unless flag?(:win32) && flag?(:gnu) %} + Math.scalbln(1.to_big_f, 100_000_000_000).should eq(2.to_big_f ** 100_000_000_000) + Math.scalbln(1.to_big_f, -100_000_000_000).should eq(0.5.to_big_f ** 100_000_000_000) + {% end %} end it ".frexp" do diff --git a/src/big/big_int.cr b/src/big/big_int.cr index 49738cb8bfbc..c306a490a412 100644 --- a/src/big/big_int.cr +++ b/src/big/big_int.cr @@ -659,7 +659,7 @@ struct BigInt < Int {% for n in [8, 16, 32, 64, 128] %} def to_i{{n}} : Int{{n}} \{% if Int{{n}} == LibGMP::SI %} - LibGMP.{{ flag?(:win32) ? "fits_si_p".id : "fits_slong_p".id }}(self) != 0 ? LibGMP.get_si(self) : raise OverflowError.new + LibGMP.{{ flag?(:win32) && !flag?(:gnu) ? "fits_si_p".id : "fits_slong_p".id }}(self) != 0 ? LibGMP.get_si(self) : raise OverflowError.new \{% elsif Int{{n}}::MAX.is_a?(NumberLiteral) && Int{{n}}::MAX < LibGMP::SI::MAX %} LibGMP::SI.new(self).to_i{{n}} \{% else %} @@ -669,7 +669,7 @@ struct BigInt < Int def to_u{{n}} : UInt{{n}} \{% if UInt{{n}} == LibGMP::UI %} - LibGMP.{{ flag?(:win32) ? "fits_ui_p".id : "fits_ulong_p".id }}(self) != 0 ? LibGMP.get_ui(self) : raise OverflowError.new + LibGMP.{{ flag?(:win32) && !flag?(:gnu) ? "fits_ui_p".id : "fits_ulong_p".id }}(self) != 0 ? LibGMP.get_ui(self) : raise OverflowError.new \{% elsif UInt{{n}}::MAX.is_a?(NumberLiteral) && UInt{{n}}::MAX < LibGMP::UI::MAX %} LibGMP::UI.new(self).to_u{{n}} \{% else %} diff --git a/src/big/lib_gmp.cr b/src/big/lib_gmp.cr index c50b1f7f6e9b..7368cb0e9fb6 100644 --- a/src/big/lib_gmp.cr +++ b/src/big/lib_gmp.cr @@ -1,4 +1,4 @@ -{% if flag?(:win32) %} +{% if flag?(:win32) && !flag?(:gnu) %} @[Link("mpir")] {% if compare_versions(Crystal::VERSION, "1.11.0-dev") >= 0 %} @[Link(dll: "mpir.dll")] @@ -14,7 +14,7 @@ lib LibGMP # MPIR uses its own `mpir_si` and `mpir_ui` typedefs in places where GMP uses # the LibC types, when the function name has `si` or `ui`; we follow this # distinction - {% if flag?(:win32) && flag?(:bits64) %} + {% if flag?(:win32) && !flag?(:gnu) && flag?(:bits64) %} alias SI = LibC::LongLong alias UI = LibC::ULongLong {% else %} @@ -26,17 +26,19 @@ lib LibGMP alias Double = LibC::Double alias BitcntT = UI - {% if flag?(:win32) && flag?(:bits64) %} - alias MpExp = LibC::Long + alias MpExp = LibC::Long + + {% if flag?(:win32) && !flag?(:gnu) %} alias MpSize = LibC::LongLong - alias MpLimb = LibC::ULongLong - {% elsif flag?(:bits64) %} - alias MpExp = Int64 - alias MpSize = LibC::Long - alias MpLimb = LibC::ULong {% else %} - alias MpExp = Int32 alias MpSize = LibC::Long + {% end %} + + # NOTE: this assumes GMP is configured by build time to define + # `_LONG_LONG_LIMB=1` on Windows + {% if flag?(:win32) %} + alias MpLimb = LibC::ULongLong + {% else %} alias MpLimb = LibC::ULong {% end %} @@ -149,11 +151,12 @@ lib LibGMP # # Miscellaneous Functions - fun fits_ulong_p = __gmpz_fits_ulong_p(op : MPZ*) : Int - fun fits_slong_p = __gmpz_fits_slong_p(op : MPZ*) : Int - {% if flag?(:win32) %} + {% if flag?(:win32) && !flag?(:gnu) %} fun fits_ui_p = __gmpz_fits_ui_p(op : MPZ*) : Int fun fits_si_p = __gmpz_fits_si_p(op : MPZ*) : Int + {% else %} + fun fits_ulong_p = __gmpz_fits_ulong_p(op : MPZ*) : Int + fun fits_slong_p = __gmpz_fits_slong_p(op : MPZ*) : Int {% end %} # # Special Functions diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index f040f87e17f5..c4844df9a5e8 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -285,7 +285,7 @@ module Crystal @main = @llvm_mod.functions.add(MAIN_NAME, main_type) @fun_types = { {@llvm_mod, MAIN_NAME} => main_type } - if @program.has_flag? "windows" + if @program.has_flag?("msvc") @personality_name = "__CxxFrameHandler3" @main.personality_function = windows_personality_fun.func else @@ -2488,7 +2488,7 @@ module Crystal end def self.safe_mangling(program, name) - if program.has_flag?("windows") + if program.has_flag?("msvc") String.build do |str| name.each_char do |char| if char.ascii_alphanumeric? || char == '_' diff --git a/src/compiler/crystal/codegen/debug.cr b/src/compiler/crystal/codegen/debug.cr index 9a03420ba203..dd4b6c361905 100644 --- a/src/compiler/crystal/codegen/debug.cr +++ b/src/compiler/crystal/codegen/debug.cr @@ -40,7 +40,7 @@ module Crystal def push_debug_info_metadata(mod) di_builder(mod).end - if @program.has_flag?("windows") + if @program.has_flag?("msvc") # Windows uses CodeView instead of DWARF mod.add_flag( LibLLVM::ModuleFlagBehavior::Warning, diff --git a/src/compiler/crystal/codegen/exception.cr b/src/compiler/crystal/codegen/exception.cr index 9a33e1337550..944ac99fce7d 100644 --- a/src/compiler/crystal/codegen/exception.cr +++ b/src/compiler/crystal/codegen/exception.cr @@ -60,9 +60,9 @@ class Crystal::CodeGenVisitor # # Note we codegen the ensure body three times! In practice this isn't a big deal, since ensure bodies are typically small. - windows = @program.has_flag? "windows" + msvc = @program.has_flag?("msvc") - context.fun.personality_function = windows_personality_fun.func if windows + context.fun.personality_function = windows_personality_fun.func if msvc # This is the block which is entered when the body raises an exception rescue_block = new_block "rescue" @@ -109,7 +109,7 @@ class Crystal::CodeGenVisitor old_catch_pad = @catch_pad - if windows + if msvc # Windows structured exception handling must enter a catch_switch instruction # which decides which catch body block to enter. Crystal only ever generates one catch body # which is used for all exceptions. For more information on how structured exception handling works in LLVM, @@ -138,7 +138,8 @@ class Crystal::CodeGenVisitor caught_exception = load exception_llvm_type, caught_exception_ptr exception_type_id = type_id(caught_exception, exception_type) else - # Unwind exception handling code - used on non-windows platforms - is a lot simpler. + # Unwind exception handling code - used on non-MSVC platforms (essentially the Itanium + # C++ ABI) - is a lot simpler. # First we generate the landing pad instruction, this returns a tuple of the libunwind # exception object and the type ID of the exception. This tuple is set up in the crystal # personality function in raise.cr @@ -188,7 +189,7 @@ class Crystal::CodeGenVisitor # If the rescue restriction matches, codegen the rescue block. position_at_end this_rescue_block - # On windows, we are "inside" the catchpad block. It's difficult to track when to catch_ret when + # On MSVC, we are "inside" the catchpad block. It's difficult to track when to catch_ret when # codegenning the entire rescue body, so we catch_ret early and execute the rescue bodies "outside" the # rescue block. if catch_pad = @catch_pad @@ -248,7 +249,7 @@ class Crystal::CodeGenVisitor # Codegen catchswitch+pad or landing pad as described above. # This code is simpler because we never need to extract the exception type - if windows + if msvc rescue_ensure_body = new_block "rescue_ensure_body" catch_switch = builder.catch_switch(old_catch_pad || LLVM::Value.null, @rescue_block || LLVM::BasicBlock.null, 1) builder.add_handler catch_switch, rescue_ensure_body @@ -283,8 +284,8 @@ class Crystal::CodeGenVisitor end def codegen_re_raise(node, unwind_ex_obj) - if @program.has_flag? "windows" - # On windows we can re-raise by calling _CxxThrowException with two null arguments + if @program.has_flag?("msvc") + # On the MSVC C++ ABI we can re-raise by calling _CxxThrowException with two null arguments call windows_throw_fun, [llvm_context.void_pointer.null, llvm_context.void_pointer.null] unreachable else diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 3601aa0fd870..81a1a96f4445 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -121,7 +121,7 @@ module Crystal class Program def lib_flags - has_flag?("windows") ? lib_flags_windows : lib_flags_posix + has_flag?("msvc") ? lib_flags_windows : lib_flags_posix end private def lib_flags_windows diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index 652f5487498f..9cb60f01ced8 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -45,12 +45,49 @@ module Crystal::System::Thread LibC.SwitchToThread end - @[ThreadLocal] - class_property current_thread : ::Thread { ::Thread.new } + # MinGW does not support TLS correctly + {% if flag?(:gnu) %} + @@current_key : LibC::DWORD = begin + current_key = LibC.TlsAlloc + if current_key == LibC::TLS_OUT_OF_INDEXES + Crystal::System.panic("TlsAlloc()", WinError.value) + end + current_key + end - def self.current_thread? : ::Thread? - @@current_thread - end + def self.current_thread : ::Thread + th = current_thread? + return th if th + + # Thread#start sets `Thread.current` as soon it starts. Thus we know + # that if `Thread.current` is not set then we are in the main thread + self.current_thread = ::Thread.new + end + + def self.current_thread? : ::Thread? + ptr = LibC.TlsGetValue(@@current_key) + err = WinError.value + unless err == WinError::ERROR_SUCCESS + Crystal::System.panic("TlsGetValue()", err) + end + + ptr.as(::Thread?) + end + + def self.current_thread=(thread : ::Thread) + if LibC.TlsSetValue(@@current_key, thread.as(Void*)) == 0 + Crystal::System.panic("TlsSetValue()", WinError.value) + end + thread + end + {% else %} + @[ThreadLocal] + class_property current_thread : ::Thread { ::Thread.new } + + def self.current_thread? : ::Thread? + @@current_thread + end + {% end %} def self.sleep(time : ::Time::Span) : Nil LibC.Sleep(time.total_milliseconds.to_i.clamp(1..)) diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index be631e19cdc7..44a281570c1c 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -2,6 +2,9 @@ require "./call_stack/interpreter" {% elsif flag?(:win32) %} require "./call_stack/stackwalk" + {% if flag?(:gnu) %} + require "./lib_unwind" + {% end %} {% elsif flag?(:wasm32) %} require "./call_stack/null" {% else %} diff --git a/src/exception/call_stack/stackwalk.cr b/src/exception/call_stack/stackwalk.cr index b3e2ed8f479c..6ac59fa6db48 100644 --- a/src/exception/call_stack/stackwalk.cr +++ b/src/exception/call_stack/stackwalk.cr @@ -168,6 +168,33 @@ struct Exception::CallStack end end + # TODO: needed only if `__crystal_raise` fails, check if this actually works + {% if flag?(:gnu) %} + def self.print_backtrace : Nil + backtrace_fn = ->(context : LibUnwind::Context, data : Void*) do + last_frame = data.as(RepeatedFrame*) + + ip = {% if flag?(:arm) %} + Pointer(Void).new(__crystal_unwind_get_ip(context)) + {% else %} + Pointer(Void).new(LibUnwind.get_ip(context)) + {% end %} + + if last_frame.value.ip == ip + last_frame.value.incr + else + print_frame(last_frame.value) unless last_frame.value.ip.address == 0 + last_frame.value = RepeatedFrame.new ip + end + LibUnwind::ReasonCode::NO_REASON + end + + rf = RepeatedFrame.new(Pointer(Void).null) + LibUnwind.backtrace(backtrace_fn, pointerof(rf).as(Void*)) + print_frame(rf) + end + {% end %} + private def self.print_frame(repeated_frame) Crystal::System.print_error "[%p] ", repeated_frame.ip print_frame_location(repeated_frame) diff --git a/src/exception/lib_unwind.cr b/src/exception/lib_unwind.cr index 7c9c6fd75ec5..83350c12fe3a 100644 --- a/src/exception/lib_unwind.cr +++ b/src/exception/lib_unwind.cr @@ -113,8 +113,12 @@ lib LibUnwind struct Exception exception_class : LibC::SizeT exception_cleanup : LibC::SizeT - private1 : UInt64 - private2 : UInt64 + {% if flag?(:win32) && flag?(:gnu) %} + private_ : UInt64[6] + {% else %} + private1 : UInt64 + private2 : UInt64 + {% end %} exception_object : Void* exception_type_id : Int32 end diff --git a/src/lib_c/x86_64-windows-gnu b/src/lib_c/x86_64-windows-gnu new file mode 120000 index 000000000000..072348f65d09 --- /dev/null +++ b/src/lib_c/x86_64-windows-gnu @@ -0,0 +1 @@ +x86_64-windows-msvc \ No newline at end of file diff --git a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr index 1fcaee65a01c..22001cfc1632 100644 --- a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr @@ -63,5 +63,11 @@ lib LibC fun ResumeThread(hThread : HANDLE) : DWORD fun SuspendThread(hThread : HANDLE) : DWORD + TLS_OUT_OF_INDEXES = 0xFFFFFFFF_u32 + + fun TlsAlloc : DWORD + fun TlsGetValue(dwTlsIndex : DWORD) : Void* + fun TlsSetValue(dwTlsIndex : DWORD, lpTlsValue : Void*) : BOOL + PROCESS_QUERY_INFORMATION = 0x0400 end diff --git a/src/llvm/target_machine.cr b/src/llvm/target_machine.cr index b9de8296d5c8..6e31836ef7f2 100644 --- a/src/llvm/target_machine.cr +++ b/src/llvm/target_machine.cr @@ -48,7 +48,7 @@ class LLVM::TargetMachine def abi triple = self.triple case triple - when /x86_64.+windows-msvc/ + when /x86_64.+windows-(?:msvc|gnu)/ ABI::X86_Win64.new(self) when /x86_64|amd64/ ABI::X86_64.new(self) diff --git a/src/raise.cr b/src/raise.cr index ff8684795e77..a8e06a3c3930 100644 --- a/src/raise.cr +++ b/src/raise.cr @@ -91,7 +91,7 @@ end {% if flag?(:interpreted) %} # interpreter does not need `__crystal_personality` -{% elsif flag?(:win32) %} +{% elsif flag?(:win32) && !flag?(:gnu) %} require "exception/lib_unwind" {% begin %} @@ -181,8 +181,10 @@ end 0u64 end {% else %} - # :nodoc: - fun __crystal_personality(version : Int32, actions : LibUnwind::Action, exception_class : UInt64, exception_object : LibUnwind::Exception*, context : Void*) : LibUnwind::ReasonCode + {% mingw = flag?(:windows) && flag?(:gnu) %} + fun {{ mingw ? "__crystal_personality_imp".id : "__crystal_personality".id }}( + version : Int32, actions : LibUnwind::Action, exception_class : UInt64, exception_object : LibUnwind::Exception*, context : Void*, + ) : LibUnwind::ReasonCode start = LibUnwind.get_region_start(context) ip = LibUnwind.get_ip(context) lsd = LibUnwind.get_language_specific_data(context) @@ -197,9 +199,26 @@ end return LibUnwind::ReasonCode::CONTINUE_UNWIND end + + {% if mingw %} + lib LibC + alias EXCEPTION_DISPOSITION = Int + alias DISPATCHER_CONTEXT = Void + end + + lib LibUnwind + alias PersonalityFn = Int32, Action, UInt64, Exception*, Void* -> ReasonCode + + fun _GCC_specific_handler(ms_exc : LibC::EXCEPTION_RECORD64*, this_frame : Void*, ms_orig_context : LibC::CONTEXT*, ms_disp : LibC::DISPATCHER_CONTEXT*, gcc_per : PersonalityFn) : LibC::EXCEPTION_DISPOSITION + end + + fun __crystal_personality(ms_exc : LibC::EXCEPTION_RECORD64*, this_frame : Void*, ms_orig_context : LibC::CONTEXT*, ms_disp : LibC::DISPATCHER_CONTEXT*) : LibC::EXCEPTION_DISPOSITION + LibUnwind._GCC_specific_handler(ms_exc, this_frame, ms_orig_context, ms_disp, ->__crystal_personality_imp) + end + {% end %} {% end %} -{% unless flag?(:interpreted) || flag?(:win32) || flag?(:wasm32) %} +{% unless flag?(:interpreted) || (flag?(:win32) && !flag?(:gnu)) || flag?(:wasm32) %} # :nodoc: @[Raises] fun __crystal_raise(unwind_ex : LibUnwind::Exception*) : NoReturn @@ -244,7 +263,7 @@ def raise(message : String) : NoReturn raise Exception.new(message) end -{% if flag?(:win32) %} +{% if flag?(:win32) && !flag?(:gnu) %} # :nodoc: {% if flag?(:interpreted) %} @[Primitive(:interpreter_raise_without_backtrace)] {% end %} def raise_without_backtrace(exception : Exception) : NoReturn From d1c5072f09274c78443f74f60bb2e96565bb0a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 11 Oct 2024 13:55:40 +0200 Subject: [PATCH 151/193] Extract `deploy_api_docs` job into its own Workflow (#15022) --- .github/workflows/docs.yml | 43 +++++++++++++++++++++++++++++++++++++ .github/workflows/linux.yml | 33 ---------------------------- 2 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000000..57147238552e --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,43 @@ +name: Docs + +on: + push: + branches: + - master + +env: + TRAVIS_OS_NAME: linux + +jobs: + deploy_api_docs: + if: github.repository_owner == 'crystal-lang' + env: + ARCH: x86_64 + ARCH_CMD: linux64 + runs-on: ubuntu-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Build docs + run: bin/ci with_build_env 'make crystal docs threads=1' + + - name: Set revision + run: echo $GITHUB_SHA > ./docs/revision.txt + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Deploy API docs to S3 + run: | + aws s3 sync ./docs s3://crystal-api/api/master --delete diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4bdcc3e0c11e..ad65fa005259 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -106,36 +106,3 @@ jobs: - name: Check Format run: bin/ci format - - deploy_api_docs: - if: github.repository_owner == 'crystal-lang' && github.event_name == 'push' && github.ref == 'refs/heads/master' - env: - ARCH: x86_64 - ARCH_CMD: linux64 - runs-on: ubuntu-latest - steps: - - name: Download Crystal source - uses: actions/checkout@v4 - - - name: Prepare System - run: bin/ci prepare_system - - - name: Prepare Build - run: bin/ci prepare_build - - - name: Build docs - run: bin/ci with_build_env 'make crystal docs threads=1' - - - name: Set revision - run: echo $GITHUB_SHA > ./docs/revision.txt - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Deploy API docs to S3 - run: | - aws s3 sync ./docs s3://crystal-api/api/master --delete From 4daf48631ed48133f724c4501fa71d0223a4c49a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 14 Oct 2024 11:10:47 +0200 Subject: [PATCH 152/193] Replace uses of `AliasType#types?` by `Type#lookup_name` (#15068) Introduces `Type#lookup_name` as a dedicated mechanism for looking up a name in a namespace. This allows us to drop the override of `AliasType#types?` which delegated to the aliased type. Thus cyclic hierarchies are no longer possible and we can drop all code (as far as I could find) that was necessary only for handling that. --- spec/compiler/semantic/did_you_mean_spec.cr | 13 +++++++++++++ src/compiler/crystal/codegen/link.cr | 2 -- .../crystal/semantic/abstract_def_checker.cr | 4 ---- src/compiler/crystal/semantic/new.cr | 2 -- src/compiler/crystal/semantic/path_lookup.cr | 2 +- .../crystal/semantic/recursive_struct_checker.cr | 5 ----- src/compiler/crystal/semantic/suggestions.cr | 4 ++-- .../semantic/type_declaration_processor.cr | 10 +++------- src/compiler/crystal/tools/doc/generator.cr | 7 ------- .../crystal/tools/typed_def_processor.cr | 9 --------- src/compiler/crystal/tools/unreachable.cr | 9 --------- src/compiler/crystal/types.cr | 16 ++++++---------- 12 files changed, 25 insertions(+), 58 deletions(-) diff --git a/spec/compiler/semantic/did_you_mean_spec.cr b/spec/compiler/semantic/did_you_mean_spec.cr index cd3f0856ebcb..1c74ebf74c2f 100644 --- a/spec/compiler/semantic/did_you_mean_spec.cr +++ b/spec/compiler/semantic/did_you_mean_spec.cr @@ -75,6 +75,19 @@ describe "Semantic: did you mean" do "Did you mean 'Foo::Bar'?" end + it "says did you mean for nested class via alias" do + assert_error <<-CRYSTAL, "Did you mean 'Boo::Bar'?" + class Foo + class Bar + end + end + + alias Boo = Foo + + Boo::Baz.new + CRYSTAL + end + it "says did you mean finds most similar in def" do assert_error " def barbaza diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 81a1a96f4445..33248e93e19b 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -292,8 +292,6 @@ module Crystal private def add_link_annotations(types, annotations) types.try &.each_value do |type| - next if type.is_a?(AliasType) || type.is_a?(TypeDefType) - if type.is_a?(LibType) && type.used? && (link_annotations = type.link_annotations) annotations.concat link_annotations end diff --git a/src/compiler/crystal/semantic/abstract_def_checker.cr b/src/compiler/crystal/semantic/abstract_def_checker.cr index 2a7ccdc05d2a..6d1aa58447a1 100644 --- a/src/compiler/crystal/semantic/abstract_def_checker.cr +++ b/src/compiler/crystal/semantic/abstract_def_checker.cr @@ -24,7 +24,6 @@ # ``` class Crystal::AbstractDefChecker def initialize(@program : Program) - @all_checked = Set(Type).new end def run @@ -41,9 +40,6 @@ class Crystal::AbstractDefChecker end def check_single(type) - return if @all_checked.includes?(type) - @all_checked << type - if type.abstract? || type.module? type.defs.try &.each_value do |defs_with_metadata| defs_with_metadata.each do |def_with_metadata| diff --git a/src/compiler/crystal/semantic/new.cr b/src/compiler/crystal/semantic/new.cr index de8ae55312a0..43a0a631e2c6 100644 --- a/src/compiler/crystal/semantic/new.cr +++ b/src/compiler/crystal/semantic/new.cr @@ -22,8 +22,6 @@ module Crystal end def define_default_new(type) - return if type.is_a?(AliasType) || type.is_a?(TypeDefType) - type.types?.try &.each_value do |type| define_default_new_single(type) end diff --git a/src/compiler/crystal/semantic/path_lookup.cr b/src/compiler/crystal/semantic/path_lookup.cr index b2d66879d253..72cab053984b 100644 --- a/src/compiler/crystal/semantic/path_lookup.cr +++ b/src/compiler/crystal/semantic/path_lookup.cr @@ -71,7 +71,7 @@ module Crystal # precedence than ancestors and the enclosing namespace. def lookup_path_item(name : String, lookup_self, lookup_in_namespace, include_private, location) : Type | ASTNode | Nil # First search in our types - type = types?.try &.[name]? + type = lookup_name(name) if type if type.private? && !include_private return nil diff --git a/src/compiler/crystal/semantic/recursive_struct_checker.cr b/src/compiler/crystal/semantic/recursive_struct_checker.cr index e7f64913789f..888730e342bb 100644 --- a/src/compiler/crystal/semantic/recursive_struct_checker.cr +++ b/src/compiler/crystal/semantic/recursive_struct_checker.cr @@ -14,10 +14,8 @@ # Because the type of `Test.@test` would be: `Test | Nil`. class Crystal::RecursiveStructChecker @program : Program - @all_checked : Set(Type) def initialize(@program) - @all_checked = Set(Type).new end def run @@ -34,9 +32,6 @@ class Crystal::RecursiveStructChecker end def check_single(type) - has_not_been_checked = @all_checked.add?(type) - return unless has_not_been_checked - if struct?(type) target = type checked = Set(Type).new diff --git a/src/compiler/crystal/semantic/suggestions.cr b/src/compiler/crystal/semantic/suggestions.cr index 8f4a69d963bc..e9e05612007f 100644 --- a/src/compiler/crystal/semantic/suggestions.cr +++ b/src/compiler/crystal/semantic/suggestions.cr @@ -13,10 +13,10 @@ module Crystal type = self names.each_with_index do |name, idx| previous_type = type - type = previous_type.types?.try &.[name]? + type = previous_type.lookup_name(name) unless type best_match = Levenshtein.find(name.downcase) do |finder| - previous_type.types?.try &.each_key do |type_name| + previous_type.remove_alias.types?.try &.each_key do |type_name| finder.test(type_name.downcase, type_name) end end diff --git a/src/compiler/crystal/semantic/type_declaration_processor.cr b/src/compiler/crystal/semantic/type_declaration_processor.cr index 65451741fac3..0e6008b2fa78 100644 --- a/src/compiler/crystal/semantic/type_declaration_processor.cr +++ b/src/compiler/crystal/semantic/type_declaration_processor.cr @@ -621,14 +621,10 @@ struct Crystal::TypeDeclarationProcessor end private def remove_duplicate_instance_vars_declarations - # All the types that we checked for duplicate variables - duplicates_checked = Set(Type).new - remove_duplicate_instance_vars_declarations(@program, duplicates_checked) + remove_duplicate_instance_vars_declarations(@program) end - private def remove_duplicate_instance_vars_declarations(type : Type, duplicates_checked : Set(Type)) - return unless duplicates_checked.add?(type) - + private def remove_duplicate_instance_vars_declarations(type : Type) # If a class has an instance variable that already exists in a superclass, remove it. # Ideally we should process instance variables in a top-down fashion, but it's tricky # with modules and multiple-inheritance. Removing duplicates at the end is maybe @@ -650,7 +646,7 @@ struct Crystal::TypeDeclarationProcessor end type.types?.try &.each_value do |nested_type| - remove_duplicate_instance_vars_declarations(nested_type, duplicates_checked) + remove_duplicate_instance_vars_declarations(nested_type) end end diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index a2f4db47dee0..4c5988cccae5 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -251,13 +251,6 @@ class Crystal::Doc::Generator def collect_subtypes(parent) types = [] of Type - # AliasType has defined `types?` to be the types - # of the aliased type, but for docs we don't want - # to list the nested types for aliases. - if parent.is_a?(AliasType) - return types - end - parent.types?.try &.each_value do |type| case type when Const, LibType diff --git a/src/compiler/crystal/tools/typed_def_processor.cr b/src/compiler/crystal/tools/typed_def_processor.cr index a0a911a6a618..2ba2441d7902 100644 --- a/src/compiler/crystal/tools/typed_def_processor.cr +++ b/src/compiler/crystal/tools/typed_def_processor.cr @@ -17,15 +17,6 @@ module Crystal::TypedDefProcessor end private def process_type(type : Type) : Nil - # Avoid visiting circular hierarchies. There's no use in processing - # alias types anyway. - # For example: - # - # struct Foo - # alias Bar = Foo - # end - return if type.is_a?(AliasType) || type.is_a?(TypeDefType) - if type.is_a?(NamedType) || type.is_a?(Program) || type.is_a?(FileModule) type.types?.try &.each_value do |inner_type| process_type inner_type diff --git a/src/compiler/crystal/tools/unreachable.cr b/src/compiler/crystal/tools/unreachable.cr index 8455a4186882..4ba681240385 100644 --- a/src/compiler/crystal/tools/unreachable.cr +++ b/src/compiler/crystal/tools/unreachable.cr @@ -155,15 +155,6 @@ module Crystal property excludes = [] of String def process_type(type) - # Avoid visiting circular hierarchies. There's no use in processing - # alias types anyway. - # For example: - # - # struct Foo - # alias Bar = Foo - # end - return if type.is_a?(AliasType) || type.is_a?(TypeDefType) - if type.is_a?(ModuleType) track_unused_defs type end diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 5d903b763050..053f6c9d8ac3 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -373,6 +373,10 @@ module Crystal nil end + def lookup_name(name) + types?.try(&.[name]?) + end + def parents nil end @@ -2756,17 +2760,9 @@ module Crystal delegate lookup_defs, lookup_defs_with_modules, lookup_first_def, lookup_macro, lookup_macros, to: aliased_type - def types? + def lookup_name(name) process_value - if aliased_type = @aliased_type - aliased_type.types? - else - nil - end - end - - def types - types?.not_nil! + @aliased_type.try(&.lookup_name(name)) end def remove_alias From 59fdcf419c308e036a353c4e2ec7fe0d2496250d Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 14 Oct 2024 17:11:30 +0800 Subject: [PATCH 153/193] Fix `find-llvm-config` to ignore `LLVM_CONFIG`'s escape sequences (#15076) If the `LLVM_CONFIG` environment variable is already set and points to a valid executable, percent signs and backslashes will be specially interpreted according to `printf`'s rules. This is most likely to happen on MSYS2, since the Windows host might have set `LLVM_CONFIG` to support static LLVM builds: ```sh $ LLVM_CONFIG='C:\Users\nicet\llvm\18\bin\llvm-config.exe' src/llvm/ext/find-llvm-config src/llvm/ext/find-llvm-config: line 19: printf: missing unicode digit for \U C:\Users icet\llvmin\llvm-config.exe ``` --- src/llvm/ext/find-llvm-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm/ext/find-llvm-config b/src/llvm/ext/find-llvm-config index 40be636e1b23..5f40a1f2e6a3 100755 --- a/src/llvm/ext/find-llvm-config +++ b/src/llvm/ext/find-llvm-config @@ -16,7 +16,7 @@ if ! LLVM_CONFIG=$(command -v "$LLVM_CONFIG"); then fi if [ "$LLVM_CONFIG" ]; then - printf "$LLVM_CONFIG" + printf %s "$LLVM_CONFIG" else printf "Error: Could not find location of llvm-config. Please specify path in environment variable LLVM_CONFIG.\n" >&2 printf "Supported LLVM versions: $(cat "$(dirname $0)/llvm-versions.txt" | sed 's/\.0//g')\n" >&2 From 9f3dba5c75e1ae463abf69c4090e5b83cf013104 Mon Sep 17 00:00:00 2001 From: kojix2 <2xijok@gmail.com> Date: Tue, 15 Oct 2024 18:58:58 +0900 Subject: [PATCH 154/193] Fix various typos (#15080) --- spec/std/http/request_spec.cr | 2 +- spec/std/io/delimited_spec.cr | 2 +- spec/std/regex_spec.cr | 2 +- spec/std/time/span_spec.cr | 4 ++-- spec/std/yaml/serializable_spec.cr | 2 +- src/channel/select.cr | 2 +- src/compiler/crystal/types.cr | 4 ++-- src/crystal/system/win32/event_loop_iocp.cr | 2 +- src/docs_pseudo_methods.cr | 2 +- src/file.cr | 2 +- src/gc/none.cr | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/spec/std/http/request_spec.cr b/spec/std/http/request_spec.cr index f997ca8998bc..1a378a39d20a 100644 --- a/spec/std/http/request_spec.cr +++ b/spec/std/http/request_spec.cr @@ -454,7 +454,7 @@ module HTTP request.form_params["test"].should eq("foobar") end - it "returns ignors invalid content-type" do + it "ignores invalid content-type" do request = Request.new("POST", "/form", nil, HTTP::Params.encode({"test" => "foobar"})) request.form_params?.should eq(nil) request.form_params.size.should eq(0) diff --git a/spec/std/io/delimited_spec.cr b/spec/std/io/delimited_spec.cr index b41af9ee5fdb..c1e06bf40dc0 100644 --- a/spec/std/io/delimited_spec.cr +++ b/spec/std/io/delimited_spec.cr @@ -259,7 +259,7 @@ describe "IO::Delimited" do io.gets_to_end.should eq("hello") end - it "handles the case of peek matching first byte, not having enough room, but later not matching (limted slice)" do + it "handles the case of peek matching first byte, not having enough room, but later not matching (limited slice)" do # not a delimiter # --- io = MemoryIOWithFixedPeek.new("abcdefgwijkfghhello") diff --git a/spec/std/regex_spec.cr b/spec/std/regex_spec.cr index af03cb2c79b8..230976d6ad3e 100644 --- a/spec/std/regex_spec.cr +++ b/spec/std/regex_spec.cr @@ -433,7 +433,7 @@ describe "Regex" do }) end - it "alpanumeric" do + it "alphanumeric" do /(?)/.name_table.should eq({1 => "f1"}) end diff --git a/spec/std/time/span_spec.cr b/spec/std/time/span_spec.cr index f9c1dd83f04f..ec49e38651cc 100644 --- a/spec/std/time/span_spec.cr +++ b/spec/std/time/span_spec.cr @@ -360,7 +360,7 @@ describe Time::Span do 1.1.weeks.should eq(7.7.days) end - it "can substract big amount using microseconds" do + it "can subtract big amount using microseconds" do jan_1_2k = Time.utc(2000, 1, 1) past = Time.utc(5, 2, 3, 0, 0, 0) delta = (past - jan_1_2k).total_microseconds.to_i64 @@ -368,7 +368,7 @@ describe Time::Span do past2.should eq(past) end - it "can substract big amount using milliseconds" do + it "can subtract big amount using milliseconds" do jan_1_2k = Time.utc(2000, 1, 1) past = Time.utc(5, 2, 3, 0, 0, 0) delta = (past - jan_1_2k).total_milliseconds.to_i64 diff --git a/spec/std/yaml/serializable_spec.cr b/spec/std/yaml/serializable_spec.cr index 7d13f4318350..a48f0c754425 100644 --- a/spec/std/yaml/serializable_spec.cr +++ b/spec/std/yaml/serializable_spec.cr @@ -1001,7 +1001,7 @@ describe "YAML::Serializable" do yaml = YAMLAttrWithPresenceAndIgnoreSerialize.from_yaml(%({"last_name": null})) yaml.last_name_present?.should be_true - # libyaml 0.2.5 removes traling space for empty scalar nodes + # libyaml 0.2.5 removes trailing space for empty scalar nodes if YAML.libyaml_version >= SemanticVersion.new(0, 2, 5) yaml.to_yaml.should eq("---\nlast_name:\n") else diff --git a/src/channel/select.cr b/src/channel/select.cr index 5628fd460e6e..05db47a79a4c 100644 --- a/src/channel/select.cr +++ b/src/channel/select.cr @@ -142,7 +142,7 @@ class Channel(T) private def self.each_skip_duplicates(ops_locks, &) # Avoid deadlocks from trying to lock the same lock twice. - # `ops_lock` is sorted by `lock_object_id`, so identical onces will be in + # `ops_lock` is sorted by `lock_object_id`, so identical ones will be in # a row and we skip repeats while iterating. last_lock_id = nil ops_locks.each do |op| diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 053f6c9d8ac3..3a2a759b3158 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -1393,10 +1393,10 @@ module Crystal # Float64 mantissa has 52 bits case kind when .i8?, .u8?, .i16?, .u16? - # Less than 23 bits, so convertable to Float32 and Float64 without precision loss + # Less than 23 bits, so convertible to Float32 and Float64 without precision loss true when .i32?, .u32? - # Less than 52 bits, so convertable to Float64 without precision loss + # Less than 52 bits, so convertible to Float64 without precision loss other_type.kind.f64? else false diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index ade1862a780c..3089e36edfeb 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -76,7 +76,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop # Wait for completion timed out but it may have been interrupted or we ask # for immediate timeout (nonblocking), so we check for the next event - # readyness again: + # readiness again: return false if next_event.wake_at > Time.monotonic end diff --git a/src/docs_pseudo_methods.cr b/src/docs_pseudo_methods.cr index 36eb1f09eaff..d789f4a9ecc8 100644 --- a/src/docs_pseudo_methods.cr +++ b/src/docs_pseudo_methods.cr @@ -227,6 +227,6 @@ end struct CRYSTAL_PSEUDO__NoReturn end -# Similar in usage to `Nil`. `Void` is prefered for C lib bindings. +# Similar in usage to `Nil`. `Void` is preferred for C lib bindings. struct CRYSTAL_PSEUDO__Void end diff --git a/src/file.cr b/src/file.cr index 5169a6dc703d..1d12a01f4209 100644 --- a/src/file.cr +++ b/src/file.cr @@ -165,7 +165,7 @@ class File < IO::FileDescriptor # *blocking* must be set to `false` on POSIX targets when the file to open # isn't a regular file but a character device (e.g. `/dev/tty`) or fifo. These # files depend on another process or thread to also be reading or writing, and - # system event queues will properly report readyness. + # system event queues will properly report readiness. # # *blocking* may also be set to `nil` in which case the blocking or # non-blocking flag will be determined automatically, at the expense of an diff --git a/src/gc/none.cr b/src/gc/none.cr index 3943bd265ed9..ce84027e6e69 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -154,7 +154,7 @@ module GC # will block until it can acquire the lock). # # In both cases there can't be a deadlock since we won't suspend another - # thread until it has successfuly added (or removed) itself to (from) the + # thread until it has successfully added (or removed) itself to (from) the # linked list and released the lock, and the other thread won't progress until # it can add (or remove) itself from the list. # From 5fd469cc2467a2ac466da2d2a2985b0ffca9a339 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Tue, 15 Oct 2024 13:25:22 -0400 Subject: [PATCH 155/193] Improve man and shell completion for tools (#15082) --- etc/completion.bash | 2 +- etc/completion.fish | 17 ++++++++++++++- etc/completion.zsh | 38 ++++++++++++++++++++++++++++----- man/crystal.1 | 4 ++-- src/compiler/crystal/command.cr | 4 ++-- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/etc/completion.bash b/etc/completion.bash index 9263289b5b4e..b64bd110a205 100644 --- a/etc/completion.bash +++ b/etc/completion.bash @@ -66,7 +66,7 @@ _crystal() _crystal_compgen_options "${opts}" "${cur}" else if [[ "${prev}" == "tool" ]] ; then - local subcommands="context dependencies flags format hierarchy implementations types" + local subcommands="context dependencies expand flags format hierarchy implementations types unreachable" _crystal_compgen_options "${subcommands}" "${cur}" else _crystal_compgen_sources "${cur}" diff --git a/etc/completion.fish b/etc/completion.fish index 64fc6a97b45a..a74d6ecf3cac 100644 --- a/etc/completion.fish +++ b/etc/completion.fish @@ -1,5 +1,5 @@ set -l crystal_commands init build clear_cache docs env eval i interactive play run spec tool help version -set -l tool_subcommands context expand flags format hierarchy implementations types +set -l tool_subcommands context dependencies expand flags format hierarchy implementations types unreachable complete -c crystal -s h -l help -d "Show help" -x @@ -206,6 +206,21 @@ complete -c crystal -n "__fish_seen_subcommand_from implementations" -s p -l pro complete -c crystal -n "__fish_seen_subcommand_from implementations" -s t -l time -d "Enable execution time output" complete -c crystal -n "__fish_seen_subcommand_from implementations" -l stdin-filename -d "Source file name to be read from STDIN" +complete -c crystal -n "__fish_seen_subcommand_from tool; and not __fish_seen_subcommand_from $tool_subcommands" -a "unreachable" -d "show methods that are never called" -x +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s D -l define -d "Define a compile-time flag" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s f -l format -d "Output format text (default), json, csv, codecov" -a "text json csv codecov" -f +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l tallies -d "Print reachable methods and their call counts as well" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l check -d "Exits with error if there is any unreachable code" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l error-trace -d "Show full error trace" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s i -l include -d "Include path" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s e -l exclude -d "Exclude path (default: lib)" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l no-color -d "Disable colored output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l prelude -d "Use given file as prelude" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s s -l stats -d "Enable statistics output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s p -l progress -d "Enable progress output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -s t -l time -d "Enable execution time output" +complete -c crystal -n "__fish_seen_subcommand_from unreachable" -l stdin-filename -d "Source file name to be read from STDIN" + complete -c crystal -n "__fish_seen_subcommand_from tool; and not __fish_seen_subcommand_from $tool_subcommands" -a "types" -d "show type of main variables" -x complete -c crystal -n "__fish_seen_subcommand_from types" -s D -l define -d "Define a compile-time flag" complete -c crystal -n "__fish_seen_subcommand_from types" -s f -l format -d "Output format text (default) or json" -a "text json" -f diff --git a/etc/completion.zsh b/etc/completion.zsh index ffa12798ca18..0d9ff58a67c2 100644 --- a/etc/completion.zsh +++ b/etc/completion.zsh @@ -41,7 +41,8 @@ local -a exec_args; exec_args=( '(\*)'{-D+,--define=}'[define a compile-time flag]:' \ '(--error-trace)--error-trace[show full error trace]' \ '(-s --stats)'{-s,--stats}'[enable statistics output]' \ - '(-t --time)'{-t,--time}'[enable execution time output]' + '(-t --time)'{-t,--time}'[enable execution time output]' \ + '(-p --progress)'{-p,--progress}'[enable progress output]' ) local -a format_args; format_args=( @@ -61,11 +62,15 @@ local -a cursor_args; cursor_args=( '(-c --cursor)'{-c,--cursor}'[cursor location with LOC as path/to/file.cr:line:column]:LOC' ) -local -a include_exclude_args; cursor_args=( +local -a include_exclude_args; include_exclude_args=( '(-i --include)'{-i,--include}'[Include path in output]' \ '(-i --exclude)'{-i,--exclude}'[Exclude path in output]' ) +local -a stdin_filename_args; stdin_filename_args=( + '(--stdin-filename)--stdin-filename[source file name to be read from STDIN]' +) + local -a programfile; programfile='*:Crystal File:_files -g "*.cr(.)"' # TODO make 'emit' allow completion with more than one @@ -170,6 +175,7 @@ _crystal-tool() { "hierarchy:show type hierarchy" "implementations:show implementations for given call in location" "types:show type of main variables" + "unreachable:show methods that are never called" ) _describe -t commands 'Crystal tool command' commands @@ -187,6 +193,7 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ + $stdin_filename_args \ $cursor_args ;; @@ -198,6 +205,7 @@ _crystal-tool() { $exec_args \ '(-f --format)'{-f,--format}'[output format 'tree' (default), 'flat', 'dot', or 'mermaid']:' \ $prelude_args \ + $stdin_filename_args \ $include_exclude_args ;; @@ -209,12 +217,14 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ + $stdin_filename_args \ $cursor_args ;; (flags) _arguments \ $programfile \ + $no_color_args \ $help_args ;; @@ -223,8 +233,9 @@ _crystal-tool() { $programfile \ $help_args \ $no_color_args \ - $format_args \ + $include_exclude_args \ '(--check)--check[checks that formatting code produces no changes]' \ + '(--show-backtrace)--show-backtrace[show backtrace on a bug (used only for debugging)]' ;; (hierarchy) @@ -235,6 +246,7 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ + $stdin_filename_args \ '(-e)-e[filter types by NAME regex]:' ;; @@ -246,7 +258,22 @@ _crystal-tool() { $exec_args \ $format_args \ $prelude_args \ - $cursor_args + $cursor_args \ + $stdin_filename_args + ;; + + (unreachable) + _arguments \ + $programfile \ + $help_args \ + $no_color_args \ + $exec_args \ + $include_exclude_args \ + '(-f --format)'{-f,--format}'[output format: text (default), json, csv, codecov]:' \ + $prelude_args \ + '(--check)--check[exits with error if there is any unreachable code]' \ + '(--tallies)--tallies[print reachable methods and their call counts as well]' \ + $stdin_filename_args ;; (types) @@ -256,7 +283,8 @@ _crystal-tool() { $no_color_args \ $exec_args \ $format_args \ - $prelude_args + $prelude_args \ + $stdin_filename_args ;; esac ;; diff --git a/man/crystal.1 b/man/crystal.1 index 04f183dd11e3..9134b8fcc8ef 100644 --- a/man/crystal.1 +++ b/man/crystal.1 @@ -369,7 +369,7 @@ Disable colored output. .Op -- .Op arguments .Pp -Run a tool. The available tools are: context, dependencies, flags, format, hierarchy, implementations, and types. +Run a tool. The available tools are: context, dependencies, expand, flags, format, hierarchy, implementations, types, and unreachable. .Pp Tools: .Bl -tag -offset indent @@ -442,7 +442,7 @@ Options: .It Fl D Ar FLAG, Fl -define= Ar FLAG Define a compile-time flag. This is useful to conditionally define types, methods, or commands based on flags available at compile time. The default flags are from the target triple given with --target-triple or the hosts default, if none is given. .It Fl f Ar FORMAT, Fl -format= Ar FORMAT -Output format 'text' (default), 'json', or 'csv'. +Output format 'text' (default), 'json', 'codecov', or 'csv'. .It Fl -tallies Print reachable methods and their call counts as well. .It Fl -check diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 1354594706fb..571c965352e0 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -40,14 +40,14 @@ class Crystal::Command Tool: context show context for given location + dependencies show file dependency tree expand show macro expansion for given location flags print all macro `flag?` values format format project, directories and/or files hierarchy show type hierarchy - dependencies show file dependency tree implementations show implementations for given call in location - unreachable show methods that are never called types show type of main variables + unreachable show methods that are never called --help, -h show this help USAGE From a088858197d04414b27dce8e48b6ba9a0f94d9c9 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 18 Oct 2024 14:49:00 +0800 Subject: [PATCH 156/193] Fix `Complex#/` edge cases (#15086) --- spec/std/complex_spec.cr | 6 ++++++ src/complex.cr | 26 ++++++++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/spec/std/complex_spec.cr b/spec/std/complex_spec.cr index 65add18f8533..2b90239d0796 100644 --- a/spec/std/complex_spec.cr +++ b/spec/std/complex_spec.cr @@ -265,6 +265,12 @@ describe "Complex" do it "complex / complex" do ((Complex.new(4, 6.2))/(Complex.new(0.5, 2.7))).should eq(Complex.new(2.485411140583554, -1.0212201591511936)) ((Complex.new(4.1, 6.0))/(Complex.new(10, 2.2))).should eq(Complex.new(0.5169782525753529, 0.48626478443342236)) + + (1.to_c / -1.to_c).should eq(-1.to_c) + assert_complex_nan 1.to_c / Float64::NAN + + (1.to_c / 0.to_c).real.abs.should eq(Float64::INFINITY) + (1.to_c / 0.to_c).imag.nan?.should be_true end it "complex / number" do diff --git a/src/complex.cr b/src/complex.cr index 65fbc9204b59..e2a5830b395a 100644 --- a/src/complex.cr +++ b/src/complex.cr @@ -237,14 +237,28 @@ struct Complex # Divides `self` by *other*. def /(other : Complex) : Complex - if other.real <= other.imag - r = other.real / other.imag - d = other.imag + r * other.real - Complex.new((@real * r + @imag) / d, (@imag * r - @real) / d) - else + if other.real.nan? || other.imag.nan? + Complex.new(Float64::NAN, Float64::NAN) + elsif other.imag.abs < other.real.abs r = other.imag / other.real d = other.real + r * other.imag - Complex.new((@real + @imag * r) / d, (@imag - @real * r) / d) + + if d.nan? || d == 0 + Complex.new(Float64::NAN, Float64::NAN) + else + Complex.new((@real + @imag * r) / d, (@imag - @real * r) / d) + end + elsif other.imag == 0 # other.real == 0 + Complex.new(@real / other.real, @imag / other.real) + else # 0 < other.real.abs <= other.imag.abs + r = other.real / other.imag + d = other.imag + r * other.real + + if d.nan? || d == 0 + Complex.new(Float64::NAN, Float64::NAN) + else + Complex.new((@real * r + @imag) / d, (@imag * r - @real) / d) + end end end From 126f037f9278f705666737807b736d67ed42e38b Mon Sep 17 00:00:00 2001 From: Lachlan Dowding Date: Fri, 18 Oct 2024 16:49:15 +1000 Subject: [PATCH 157/193] Fix `Number#humanize` printing of `(-)Infinity` and `NaN` (#15090) --- spec/std/humanize_spec.cr | 8 ++++++++ src/humanize.cr | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/spec/std/humanize_spec.cr b/spec/std/humanize_spec.cr index c909417aca36..d24d2017cb28 100644 --- a/spec/std/humanize_spec.cr +++ b/spec/std/humanize_spec.cr @@ -207,6 +207,14 @@ describe Number do it { assert_prints 1.0e+34.humanize, "10,000Q" } it { assert_prints 1.0e+35.humanize, "100,000Q" } + it { assert_prints Float32::INFINITY.humanize, "Infinity" } + it { assert_prints (-Float32::INFINITY).humanize, "-Infinity" } + it { assert_prints Float32::NAN.humanize, "NaN" } + + it { assert_prints Float64::INFINITY.humanize, "Infinity" } + it { assert_prints (-Float64::INFINITY).humanize, "-Infinity" } + it { assert_prints Float64::NAN.humanize, "NaN" } + it { assert_prints 1_234.567_890_123.humanize(precision: 2, significant: false), "1.23k" } it { assert_prints 123.456_789_012_3.humanize(precision: 2, significant: false), "123.46" } it { assert_prints 12.345_678_901_23.humanize(precision: 2, significant: false), "12.35" } diff --git a/src/humanize.cr b/src/humanize.cr index bb285fe3a07d..db9d84c64889 100644 --- a/src/humanize.cr +++ b/src/humanize.cr @@ -216,7 +216,7 @@ struct Number # # See `Int#humanize_bytes` to format a file size. def humanize(io : IO, precision = 3, separator = '.', delimiter = ',', *, base = 10 ** 3, significant = true, &prefixes : (Int32, Float64) -> {Int32, _} | {Int32, _, Bool}) : Nil - if zero? + if zero? || (responds_to?(:infinite?) && self.infinite?) || (responds_to?(:nan?) && self.nan?) digits = 0 else log = Math.log10(abs) From fa2583815e2b60be34778c127d2eae33cf7aa246 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 19 Oct 2024 11:34:25 +0200 Subject: [PATCH 158/193] Fix failing specs on FreeBSD (#15093) --- spec/std/signal_spec.cr | 24 ++++++++++++++---------- spec/std/socket/socket_spec.cr | 23 +++++++++++++---------- spec/std/socket/udp_socket_spec.cr | 3 +++ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/spec/std/signal_spec.cr b/spec/std/signal_spec.cr index 969e4dc3d742..e27373e3be21 100644 --- a/spec/std/signal_spec.cr +++ b/spec/std/signal_spec.cr @@ -19,44 +19,48 @@ pending_interpreted describe: "Signal" do end {% unless flag?(:win32) %} + # can't use SIGUSR1/SIGUSR2 on FreeBSD because Boehm uses them to suspend/resume threads + signal1 = {% if flag?(:freebsd) %} Signal.new(LibC::SIGRTMAX - 1) {% else %} Signal::USR1 {% end %} + signal2 = {% if flag?(:freebsd) %} Signal.new(LibC::SIGRTMAX - 2) {% else %} Signal::USR2 {% end %} + it "runs a signal handler" do ran = false - Signal::USR1.trap do + signal1.trap do ran = true end - Process.signal Signal::USR1, Process.pid + Process.signal signal1, Process.pid 10.times do |i| break if ran sleep 0.1.seconds end ran.should be_true ensure - Signal::USR1.reset + signal1.reset end it "ignores a signal" do - Signal::USR2.ignore - Process.signal Signal::USR2, Process.pid + signal2.ignore + Process.signal signal2, Process.pid end it "allows chaining of signals" do ran_first = false ran_second = false - Signal::USR1.trap { ran_first = true } - existing = Signal::USR1.trap_handler? + signal1.trap { ran_first = true } + existing = signal1.trap_handler? - Signal::USR1.trap do |signal| + signal1.trap do |signal| existing.try &.call(signal) ran_second = true end - Process.signal Signal::USR1, Process.pid + Process.signal signal1, Process.pid sleep 0.1.seconds ran_first.should be_true ran_second.should be_true ensure - Signal::USR1.reset + signal1.reset end it "CHLD.reset sets default Crystal child handler" do diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index f4ff7c90972b..65f7ed72a453 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -24,16 +24,19 @@ describe Socket, tags: "network" do sock.type.should eq(Socket::Type::DGRAM) {% end %} - error = expect_raises(Socket::Error) do - TCPSocket.new(family: :unix) - end - error.os_error.should eq({% if flag?(:win32) %} - WinError::WSAEPROTONOSUPPORT - {% elsif flag?(:wasi) %} - WasiError::PROTONOSUPPORT - {% else %} - Errno.new(LibC::EPROTONOSUPPORT) - {% end %}) + {% unless flag?(:freebsd) %} + # for some reason this doesn't fail on freebsd + error = expect_raises(Socket::Error) do + TCPSocket.new(family: :unix) + end + error.os_error.should eq({% if flag?(:win32) %} + WinError::WSAEPROTONOSUPPORT + {% elsif flag?(:wasi) %} + WasiError::PROTONOSUPPORT + {% else %} + Errno.new(LibC::EPROTONOSUPPORT) + {% end %}) + {% end %} end end diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 113a4ea3cf61..6e4b607b80ea 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -82,6 +82,9 @@ describe UDPSocket, tags: "network" do # TODO: figure out why updating `multicast_loopback` produces a # `setsockopt 18: Invalid argument` error pending "joins and transmits to multicast groups" + elsif {{ flag?(:freebsd) }} && family == Socket::Family::INET6 + # FIXME: fails with "Error sending datagram to [ipv6]:port: Network is unreachable" + pending "joins and transmits to multicast groups" else it "joins and transmits to multicast groups" do udp = UDPSocket.new(family) From 3a78e8a34e984d0d821728c830e9513a72270364 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sat, 19 Oct 2024 17:36:35 +0800 Subject: [PATCH 159/193] Support building from a MinGW-w64-based compiler (#15077) This is a continuation of #15070 that allows a compiler built with MinGW-w64 to itself build programs correctly. Resolves part of #6170. * Because linker flags for GCC may now be executed on a Windows environment, we use the correct form of argument quoting. We also drop `-rdynamic` since that only makes sense for ELF executables. * Targetting `x86_64-windows-gnu`, including normal compilations from such a Crystal compiler, will not copy dependent DLLs to the output directory. Crystal itself and programs built under MSYS2 will just work as long as the proper environment is used. You are on your own here, although `ldd` exists on MSYS2 so that you don't need the MSVC build tools for this. * The correct GCC compiler flag to select `wmain` over `main` as the C entry point is `-municode`. (The system entry point is presumably `_start` now.) * `legacy_stdio_definitions.obj` doesn't exist on MinGW-w64, so we disable it outside MSVC. * For build command lines that are too long on Windows, we use GCC's response file support. To build a MinGW-w64 compiler: ```cmd @REM on the MSVC developer prompt make -fMakefile.win crystal bin\crystal build --cross-compile --target=x86_64-windows-gnu src\compiler\crystal.cr -Dwithout_interpreter ``` ```sh # on MSYS2's UCRT64 environment pacman -Sy \ mingw-w64-ucrt-x86_64-gc mingw-w64-ucrt-x86_64-pcre2 mingw-w64-ucrt-x86_64-libiconv \ mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-llvm cc crystal.obj -o crystal \ $(pkg-config bdw-gc libpcre2-8 iconv zlib openssl --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lDbgHelp -lole32 -lWS2_32 export CRYSTAL_PATH='lib;$ORIGIN\src' export CRYSTAL_LIBRARY_PATH='' ``` Now you can run or build a considerable number of files from here, such as `./crystal.exe samples/2048.cr` and `./crystal.exe spec spec/std/regex_spec.cr`. Notable omissions are OpenSSL and LLVM, as fixing their version detection macros is a bit complicated. The interpreter is not supported. Most likely, `Crystal::Loader` would have a GCC-style `.parse`, but the rest of the functionality would be identical to the MSVC `LoadLibraryExW`-based loader. ~~Also, some invocations like `./crystal.exe spec spec/std/json` will fail since the whole command line string is too long. Similar to MSVC, [GCC also handles response files starting with `@`](https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html), so this can be implemented later; a workaround is to use `--single-module`.~~ For reference, here are all the useful MSYS2 packages and their corresponding pkg-config names: | MSYS2 package name | pkg-config name | |-|-| | mingw-w64-ucrt-x86_64-gc | bdw-gc | | mingw-w64-ucrt-x86_64-pcre2 | libpcre2-8 | | mingw-w64-ucrt-x86_64-libiconv | iconv | | mingw-w64-ucrt-x86_64-gmp | gmp | | mingw-w64-ucrt-x86_64-zlib | zlib | | mingw-w64-ucrt-x86_64-libxml2 | libxml-2.0 | | mingw-w64-ucrt-x86_64-libyaml | yaml-0.1 | | mingw-w64-ucrt-x86_64-openssl | openssl | | mingw-w64-ucrt-x86_64-libffi | libffi | | mingw-w64-ucrt-x86_64-llvm | _(use llvm-config instead)_ | --- src/compiler/crystal/codegen/link.cr | 26 +++++--- src/compiler/crystal/compiler.cr | 78 ++++++++++++++++-------- src/crystal/system/win32/wmain.cr | 7 ++- src/lib_c/x86_64-windows-msvc/c/stdio.cr | 4 +- 4 files changed, 80 insertions(+), 35 deletions(-) diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 33248e93e19b..b2b827916cbf 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -120,18 +120,18 @@ module Crystal end class Program - def lib_flags - has_flag?("msvc") ? lib_flags_windows : lib_flags_posix + def lib_flags(cross_compiling : Bool = false) + has_flag?("msvc") ? lib_flags_windows(cross_compiling) : lib_flags_posix(cross_compiling) end - private def lib_flags_windows + private def lib_flags_windows(cross_compiling) flags = [] of String # Add CRYSTAL_LIBRARY_PATH locations, so the linker preferentially # searches user-given library paths. if has_flag?("msvc") CrystalLibraryPath.paths.each do |path| - flags << Process.quote_windows("/LIBPATH:#{path}") + flags << quote_flag("/LIBPATH:#{path}", cross_compiling) end end @@ -141,14 +141,14 @@ module Crystal end if libname = ann.lib - flags << Process.quote_windows("#{libname}.lib") + flags << quote_flag("#{libname}.lib", cross_compiling) end end flags.join(" ") end - private def lib_flags_posix + private def lib_flags_posix(cross_compiling) flags = [] of String static_build = has_flag?("static") @@ -158,7 +158,7 @@ module Crystal # Add CRYSTAL_LIBRARY_PATH locations, so the linker preferentially # searches user-given library paths. CrystalLibraryPath.paths.each do |path| - flags << Process.quote_posix("-L#{path}") + flags << quote_flag("-L#{path}", cross_compiling) end link_annotations.reverse_each do |ann| @@ -173,17 +173,25 @@ module Crystal elsif (lib_name = ann.lib) && (flag = pkg_config(lib_name, static_build)) flags << flag elsif (lib_name = ann.lib) - flags << Process.quote_posix("-l#{lib_name}") + flags << quote_flag("-l#{lib_name}", cross_compiling) end if framework = ann.framework - flags << "-framework" << Process.quote_posix(framework) + flags << "-framework" << quote_flag(framework, cross_compiling) end end flags.join(" ") end + private def quote_flag(flag, cross_compiling) + if cross_compiling + has_flag?("windows") ? Process.quote_windows(flag) : Process.quote_posix(flag) + else + Process.quote(flag) + end + end + # Searches among CRYSTAL_LIBRARY_PATH, the compiler's directory, and PATH # for every DLL specified in the used `@[Link]` annotations. Yields the # absolute path and `true` if found, the base name and `false` if not found. diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index f620fe2fb312..aa11ef1dc47e 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -354,7 +354,7 @@ module Crystal run_dsymutil(output_filename) unless debug.none? {% end %} - {% if flag?(:windows) %} + {% if flag?(:msvc) %} copy_dlls(program, output_filename) unless static? {% end %} end @@ -424,26 +424,8 @@ module Crystal private def linker_command(program : Program, object_names, output_filename, output_dir, expand = false) if program.has_flag? "msvc" - lib_flags = program.lib_flags - # Execute and expand `subcommands`. - if expand - lib_flags = lib_flags.gsub(/`(.*?)`/) do - command = $1 - begin - error_io = IO::Memory.new - output = Process.run(command, shell: true, output: :pipe, error: error_io) do |process| - process.output.gets_to_end - end - unless $?.success? - error_io.rewind - error "Error executing subcommand for linker flags: #{command.inspect}: #{error_io}" - end - output - rescue exc - error "Error executing subcommand for linker flags: #{command.inspect}: #{exc}" - end - end - end + lib_flags = program.lib_flags(@cross_compile) + lib_flags = expand_lib_flags(lib_flags) if expand object_arg = Process.quote_windows(object_names) output_arg = Process.quote_windows("/Fe#{output_filename}") @@ -487,15 +469,63 @@ module Crystal {linker, cmd, nil} elsif program.has_flag? "wasm32" link_flags = @link_flags || "" - {"wasm-ld", %(wasm-ld "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} -lc #{program.lib_flags}), object_names} + {"wasm-ld", %(wasm-ld "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} -lc #{program.lib_flags(@cross_compile)}), object_names} elsif program.has_flag? "avr" link_flags = @link_flags || "" link_flags += " --target=avr-unknown-unknown -mmcu=#{@mcpu} -Wl,--gc-sections" - {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags}), object_names} + {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} + elsif program.has_flag?("win32") && program.has_flag?("gnu") + link_flags = @link_flags || "" + lib_flags = program.lib_flags(@cross_compile) + lib_flags = expand_lib_flags(lib_flags) if expand + cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}) + + if cmd.size > 32000 + # The command line would be too big, pass the args through a file instead. + # GCC response file does not interpret those args as shell-escaped + # arguments, we must rebuild the whole command line + args_filename = "#{output_dir}/linker_args.txt" + File.open(args_filename, "w") do |f| + object_names.each do |object_name| + f << object_name.gsub(GCC_RESPONSE_FILE_TR) << ' ' + end + f << "-o " << output_filename.gsub(GCC_RESPONSE_FILE_TR) << ' ' + f << link_flags << ' ' << lib_flags + end + cmd = "#{DEFAULT_LINKER} #{Process.quote_windows("@" + args_filename)}" + end + + {DEFAULT_LINKER, cmd, nil} else link_flags = @link_flags || "" link_flags += " -rdynamic" - {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags}), object_names} + {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} + end + end + + private GCC_RESPONSE_FILE_TR = { + " ": %q(\ ), + "'": %q(\'), + "\"": %q(\"), + "\\": "\\\\", + } + + private def expand_lib_flags(lib_flags) + lib_flags.gsub(/`(.*?)`/) do + command = $1 + begin + error_io = IO::Memory.new + output = Process.run(command, shell: true, output: :pipe, error: error_io) do |process| + process.output.gets_to_end + end + unless $?.success? + error_io.rewind + error "Error executing subcommand for linker flags: #{command.inspect}: #{error_io}" + end + output.chomp + rescue exc + error "Error executing subcommand for linker flags: #{command.inspect}: #{exc}" + end end end diff --git a/src/crystal/system/win32/wmain.cr b/src/crystal/system/win32/wmain.cr index 71383c66a88a..3dd64f9c1b92 100644 --- a/src/crystal/system/win32/wmain.cr +++ b/src/crystal/system/win32/wmain.cr @@ -4,7 +4,12 @@ require "c/stdlib" {% begin %} # we have both `main` and `wmain`, so we must choose an unambiguous entry point - @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }}, ldflags: "/ENTRY:wmainCRTStartup")] + @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }})] + {% if flag?(:msvc) %} + @[Link(ldflags: "/ENTRY:wmainCRTStartup")] + {% elsif flag?(:gnu) %} + @[Link(ldflags: "-municode")] + {% end %} {% end %} lib LibCrystalMain end diff --git a/src/lib_c/x86_64-windows-msvc/c/stdio.cr b/src/lib_c/x86_64-windows-msvc/c/stdio.cr index f23bba8503f6..ddfa97235d87 100644 --- a/src/lib_c/x86_64-windows-msvc/c/stdio.cr +++ b/src/lib_c/x86_64-windows-msvc/c/stdio.cr @@ -1,6 +1,8 @@ require "./stddef" -@[Link("legacy_stdio_definitions")] +{% if flag?(:msvc) %} + @[Link("legacy_stdio_definitions")] +{% end %} lib LibC # unused fun printf(format : Char*, ...) : Int From 95044dcb109b964917dc9594f20cd0af75c4d343 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 20 Oct 2024 03:08:54 +0800 Subject: [PATCH 160/193] Make `bin/crystal` work on MSYS2 (#15094) Normally, [MSYS2 automatically translates path lists](https://www.msys2.org/docs/filesystem-paths/) between Windows and Unix formats in the environment variables, as long as the variable looks like a Unix path list. You could try this even with an MSVC-built compiler: ```sh $ /c/crystal/crystal.exe env CRYSTAL_PATH lib;C:\crystal\src $ CRYSTAL_PATH='/c/crystal/src' /c/crystal/crystal.exe env CRYSTAL_PATH C:/crystal/src $ CRYSTAL_PATH='/c/crystal/src:/d/e:/foo/bar' /c/crystal/crystal.exe env CRYSTAL_PATH C:\crystal\src;D:\e;C:\msys64\foo\bar ``` `bin/crystal` defaults `CRYSTAL_PATH` to `lib:.../src` if it is not already set in the current environment. The problem is that `lib:.../src` doesn't look like a Unix path list [according to MSYS2](https://github.com/msys2/msys2-runtime/blob/2bfb7739dadf6a27f9b4c006adfd69944f3df2f1/winsup/cygwin/msys2_path_conv.cc#L339), so no conversion is done: ```sh $ CRYSTAL_PATH='lib:/c/Users/nicet/crystal/crystal/src' /c/crystal/crystal.exe env CRYSTAL_PATH lib:/c/Users/nicet/crystal/crystal/src ``` Turning the `lib` into `./lib` seems to trick MSYS2 into performing the conversion: ```sh $ CRYSTAL_PATH='./lib:/c/Users/nicet/crystal/crystal/src' /c/crystal/crystal.exe env CRYSTAL_PATH .\lib;C:\Users\nicet\crystal\crystal\src ``` Cygwin does not appear to do this automatically and one probably has to use `cygpath -p` directly. --- bin/crystal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/crystal b/bin/crystal index 3f7ceb1b88f4..2282cbefec80 100755 --- a/bin/crystal +++ b/bin/crystal @@ -137,7 +137,7 @@ SCRIPT_ROOT="$(dirname "$SCRIPT_PATH")" CRYSTAL_ROOT="$(dirname "$SCRIPT_ROOT")" CRYSTAL_DIR="$CRYSTAL_ROOT/.build" -export CRYSTAL_PATH="${CRYSTAL_PATH:-lib:$CRYSTAL_ROOT/src}" +export CRYSTAL_PATH="${CRYSTAL_PATH:-./lib:$CRYSTAL_ROOT/src}" if [ -n "${CRYSTAL_PATH##*"$CRYSTAL_ROOT"/src*}" ]; then __warning_msg "CRYSTAL_PATH env variable does not contain $CRYSTAL_ROOT/src" fi From 796a0d2cfdeac49b372ae53550010ede226ee4f8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 20 Oct 2024 03:09:07 +0800 Subject: [PATCH 161/193] Fix libiconv build on Windows (#15095) --- etc/win-ci/cygwin-build-iconv.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/etc/win-ci/cygwin-build-iconv.sh b/etc/win-ci/cygwin-build-iconv.sh index a8507542e646..204427be66fa 100644 --- a/etc/win-ci/cygwin-build-iconv.sh +++ b/etc/win-ci/cygwin-build-iconv.sh @@ -23,8 +23,12 @@ else export CXXFLAGS="-MT" enable_shared=no enable_static=yes + # GNU libiconv appears to define `BUILDING_DLL` unconditionally, so the static + # library contains `/EXPORT` directives that make any executable also export + # the iconv symbols, which we don't want + find . '(' -name '*.h' -or -name '*.h.build.in' ')' -print0 | xargs -0 -i sed -i 's/__declspec(dllexport)//' '{}' fi -export CPPFLAGS="-D_WIN32_WINNT=_WIN32_WINNT_WIN7 -I$(pwd)/iconv/include" +export CPPFLAGS="-O2 -D_WIN32_WINNT=_WIN32_WINNT_WIN7 -I$(pwd)/iconv/include" export LDFLAGS="-L$(pwd)/iconv/lib" ./configure --host=x86_64-w64-mingw32 --prefix="$(pwd)/iconv" --enable-shared="${enable_shared}" --enable-static="${enable_static}" From 57017f6de66d88436bd80862c23a1ed0bff69648 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 19 Oct 2024 21:09:29 +0200 Subject: [PATCH 162/193] Fix Deadlock with parallel stop-world/fork calls in MT (#15096) Trying to stop the world from a thread while threads are forking leads to a deadlock situation in glibc (at least) because we disable the reception of *all* signals while we fork. Since the suspend and resume signals are handled directly and not processed through the event loop (for obvious reasons) we can safely keep these signals enabled. Apparently it's even safer. --- src/crystal/system/unix/process.cr | 7 +++++++ src/crystal/system/unix/pthread.cr | 19 ++++++++++++++++++- src/gc/boehm.cr | 14 ++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 420030f8ba53..0eb58231900e 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -176,7 +176,14 @@ struct Crystal::System::Process newmask = uninitialized LibC::SigsetT oldmask = uninitialized LibC::SigsetT + # block signals while we fork, so the child process won't forward signals it + # may receive to the parent through the signal pipe, but make sure to not + # block stop-the-world signals as it appears to create deadlocks in glibc + # for example; this is safe because these signal handlers musn't be + # registered through `Signal.trap` but directly through `sigaction`. LibC.sigfillset(pointerof(newmask)) + LibC.sigdelset(pointerof(newmask), System::Thread.sig_suspend) + LibC.sigdelset(pointerof(newmask), System::Thread.sig_resume) ret = LibC.pthread_sigmask(LibC::SIG_SETMASK, pointerof(newmask), pointerof(oldmask)) raise RuntimeError.from_errno("Failed to disable signals") unless ret == 0 diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 50a0fc56e818..bbdfcbc3d41c 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -249,7 +249,8 @@ module Crystal::System::Thread end end - # the suspend/resume signals follow BDWGC + # the suspend/resume signals try to follow BDWGC but aren't exact (e.g. it may + # use SIGUSR1 and SIGUSR2 on FreeBSD instead of SIGRT). private SIG_SUSPEND = {% if flag?(:linux) %} @@ -266,6 +267,22 @@ module Crystal::System::Thread {% else %} LibC::SIGXCPU {% end %} + + def self.sig_suspend : ::Signal + if GC.responds_to?(:sig_suspend) + GC.sig_suspend + else + ::Signal.new(SIG_SUSPEND) + end + end + + def self.sig_resume : ::Signal + if GC.responds_to?(:sig_resume) + GC.sig_resume + else + ::Signal.new(SIG_RESUME) + end + end end # In musl (alpine) the calls to unwind API segfaults diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 41c0f43f2a8c..33d6466d792b 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -164,6 +164,8 @@ lib LibGC fun stop_world_external = GC_stop_world_external fun start_world_external = GC_start_world_external + fun get_suspend_signal = GC_get_suspend_signal : Int + fun get_thr_restart_signal = GC_get_thr_restart_signal : Int end module GC @@ -483,4 +485,16 @@ module GC def self.start_world : Nil LibGC.start_world_external end + + {% if flag?(:unix) %} + # :nodoc: + def self.sig_suspend : Signal + Signal.new(LibGC.get_suspend_signal) + end + + # :nodoc: + def self.sig_resume : Signal + Signal.new(LibGC.get_thr_restart_signal) + end + {% end %} end From 7cd5f738ed56aea8e1e9d08ac87f22630c273ee6 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Sun, 20 Oct 2024 07:46:08 -0400 Subject: [PATCH 163/193] Better handle explicit chunked encoding responses (#15092) --- spec/std/http/server/response_spec.cr | 9 +++++++++ src/http/server/response.cr | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/spec/std/http/server/response_spec.cr b/spec/std/http/server/response_spec.cr index 99e462151f6b..c5d775e48b8d 100644 --- a/spec/std/http/server/response_spec.cr +++ b/spec/std/http/server/response_spec.cr @@ -76,6 +76,15 @@ describe HTTP::Server::Response do io.to_s.should eq("HTTP/1.1 304 Not Modified\r\nContent-Length: 5\r\n\r\n") end + it "allow explicitly configuring a `Transfer-Encoding` response" do + io = IO::Memory.new + response = Response.new(io) + response.headers["Transfer-Encoding"] = "chunked" + response.print "Hello" + response.close + io.to_s.should eq("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nHello\r\n0\r\n\r\n") + end + it "prints less then buffer's size" do io = IO::Memory.new response = Response.new(io) diff --git a/src/http/server/response.cr b/src/http/server/response.cr index 5c80b31cce00..4dd6968ac560 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -255,7 +255,9 @@ class HTTP::Server private def unbuffered_write(slice : Bytes) : Nil return if slice.empty? - unless response.wrote_headers? + if response.headers["Transfer-Encoding"]? == "chunked" + @chunked = true + elsif !response.wrote_headers? if response.version != "HTTP/1.0" && !response.headers.has_key?("Content-Length") response.headers["Transfer-Encoding"] = "chunked" @chunked = true @@ -289,7 +291,7 @@ class HTTP::Server status = response.status set_content_length = !(status.not_modified? || status.no_content? || status.informational?) - if !response.wrote_headers? && !response.headers.has_key?("Content-Length") && set_content_length + if !response.wrote_headers? && !response.headers.has_key?("Transfer-Encoding") && !response.headers.has_key?("Content-Length") && set_content_length response.content_length = @out_count end From 8f49ab909dd2c9bfcfe3c2a0745bfcbaa870fc5a Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Sun, 20 Oct 2024 13:11:17 -0400 Subject: [PATCH 164/193] Add location information to more MacroIf related nodes (#15100) The most significant addition is when you have a `MacroIf` node like: ```cr {% if 1 == 2 %} {{2 * 2}} {% end %} ``` The `then` block of it is parsed as an `Expressions` that consists of `[MaccroLiteral, MacroExpression, MacroLiteral]`. I added location information to the nested `MacroExpression` so it knows it's on line 2 instead of line 1, which is what you'd get if you did `node.then.location` since that's the location of the first `MacroLiteral`. This is essentially my workaround for https://github.com/crystal-lang/crystal/issues/14884#issuecomment-2423904262. This PR also handles `Begin` and `MacroIf` nodes when nested in another node. --- spec/compiler/parser/parser_spec.cr | 110 ++++++++++++++++++++++++++ src/compiler/crystal/syntax/parser.cr | 11 +-- 2 files changed, 116 insertions(+), 5 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index db69fa357d59..09569b88f003 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -2696,6 +2696,116 @@ module Crystal location.line_number.should eq 10 end + it "sets the correct location for MacroExpressions in a MacroIf" do + parser = Parser.new(<<-CR) + {% if 1 == 2 %} + {{2 * 2}} + {% else %} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + CR + + node = parser.parse.should be_a MacroIf + location = node.location.should_not be_nil + location.line_number.should eq 1 + location.column_number.should eq 3 + + then_node = node.then.should be_a Expressions + then_node_location = then_node.location.should_not be_nil + then_node_location.line_number.should eq 1 + then_node_location = then_node.end_location.should_not be_nil + then_node_location.line_number.should eq 3 + + then_node_location = then_node.expressions[1].location.should_not be_nil + then_node_location.line_number.should eq 2 + then_node_location = then_node.expressions[1].end_location.should_not be_nil + then_node_location.line_number.should eq 2 + + else_node = node.else.should be_a Expressions + else_node_location = else_node.location.should_not be_nil + else_node_location.line_number.should eq 3 + else_node_location = else_node.end_location.should_not be_nil + else_node_location.line_number.should eq 8 + + else_node = node.else.should be_a Expressions + else_node_location = else_node.expressions[1].location.should_not be_nil + else_node_location.line_number.should eq 4 + else_node_location = else_node.expressions[1].end_location.should_not be_nil + else_node_location.line_number.should eq 7 + end + + it "sets correct location of Begin within another node" do + parser = Parser.new(<<-CR) + macro finished + {% begin %} + {{2 * 2}} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + end + CR + + node = parser.parse.should be_a Macro + node = node.body.should be_a Expressions + node = node.expressions[1].should be_a MacroIf + + location = node.location.should_not be_nil + location.line_number.should eq 2 + location = node.end_location.should_not be_nil + location.line_number.should eq 8 + end + + it "sets correct location of MacroIf within another node" do + parser = Parser.new(<<-CR) + macro finished + {% if false %} + {{2 * 2}} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + end + CR + + node = parser.parse.should be_a Macro + node = node.body.should be_a Expressions + node = node.expressions[1].should be_a MacroIf + + location = node.location.should_not be_nil + location.line_number.should eq 2 + location = node.end_location.should_not be_nil + location.line_number.should eq 8 + end + + it "sets correct location of MacroIf (unless) within another node" do + parser = Parser.new(<<-CR) + macro finished + {% unless false %} + {{2 * 2}} + {% + 1 + 1 + 2 + 2 + %} + {% end %} + end + CR + + node = parser.parse.should be_a Macro + node = node.body.should be_a Expressions + node = node.expressions[1].should be_a MacroIf + + location = node.location.should_not be_nil + location.line_number.should eq 2 + location = node.end_location.should_not be_nil + location.line_number.should eq 8 + end + it "sets correct location of trailing ensure" do parser = Parser.new("foo ensure bar") node = parser.parse.as(ExceptionHandler) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 15bd221fd8b2..1f0b6160a363 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3213,7 +3213,7 @@ module Crystal when .macro_literal? pieces << MacroLiteral.new(@token.value.to_s).at(@token.location).at_end(token_end_location) when .macro_expression_start? - pieces << MacroExpression.new(parse_macro_expression) + pieces << MacroExpression.new(parse_macro_expression).at(@token.location).at_end(token_end_location) check_macro_expression_end skip_whitespace = check_macro_skip_whitespace when .macro_control_start? @@ -3341,6 +3341,7 @@ module Crystal end def parse_macro_control(start_location, macro_state = Token::MacroState.default) + location = @token.location next_token_skip_space_or_newline case @token.value @@ -3385,9 +3386,9 @@ module Crystal return MacroFor.new(vars, exp, body).at_end(token_end_location) when Keyword::IF - return parse_macro_if(start_location, macro_state) + return parse_macro_if(start_location, macro_state).at(location) when Keyword::UNLESS - return parse_macro_if(start_location, macro_state, is_unless: true) + return parse_macro_if(start_location, macro_state, is_unless: true).at(location) when Keyword::BEGIN next_token_skip_space check :OP_PERCENT_RCURLY @@ -3400,7 +3401,7 @@ module Crystal next_token_skip_space check :OP_PERCENT_RCURLY - return MacroIf.new(BoolLiteral.new(true), body).at_end(token_end_location) + return MacroIf.new(BoolLiteral.new(true), body).at(location).at_end(token_end_location) when Keyword::ELSE, Keyword::ELSIF, Keyword::END return nil when Keyword::VERBATIM @@ -3428,7 +3429,7 @@ module Crystal exps = parse_expressions @in_macro_expression = false - MacroExpression.new(exps, output: false).at_end(token_end_location) + MacroExpression.new(exps, output: false).at(location).at_end(token_end_location) end def parse_macro_if(start_location, macro_state, check_end = true, is_unless = false) From b10f1713997facff15a905249ecc78bcfeb8ca43 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 21 Oct 2024 19:58:46 +0800 Subject: [PATCH 165/193] Make `Makefile` work on MSYS2 (#15102) * Uses `llvm-config` instead of `llvm_VERSION` for the `x86_64-windows-gnu` target. Also invokes the `find-llvm-config` script using `sh` explicitly and ensures the script returns a Windows path. (This pretty much means a MinGW-w64-based compiler will require the entire MSYS2 environment to work.) * Increases the stack size from the default 2 MiB to 8 MiB, consistent with builds using the MSVC toolchain. * `Makefile` and the `bin/crystal` script now recognize `.build/crystal.exe` as the local compiler under MSYS2. * If `.build/crystal.exe` already exists and we are on Windows, `Makefile` now builds a temporary executable first, just like in `Makefile.win`. * Ensures `Makefile` doesn't interpret backslashes in `llvm-config.exe`'s path as escape sequences. * Adds a separate `install_dlls` target for installing the compiler's dependent DLLs to the given prefix's `bin` directory. This is only needed if the installation is to be distributed outside MSYS2. * Detects the presence of `lld` which can be installed using the `mingw-w64-ucrt-x86_64-lld` package. With this patch, cross-compiling from MSVC-based Crystal will no longer work until #15091 is merged. Instead it can be done from Linux, including WSL, assuming the host and target LLVM versions are identical (right now MSYS2 has 18.1.8): ```sh # on Linux make clean crystal && make -B target=x86_64-windows-gnu # on MSYS2's UCRT64 environment pacman -Sy \ mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-pkgconf \ mingw-w64-ucrt-x86_64-gc mingw-w64-ucrt-x86_64-pcre2 mingw-w64-ucrt-x86_64-libiconv \ mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-llvm cc .build/crystal.obj -o .build/crystal.exe \ $(pkg-config bdw-gc libpcre2-8 iconv zlib --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 ``` --- Makefile | 44 +++++++++++++++++++++++--------- bin/crystal | 15 ++++++++--- src/compiler/crystal/compiler.cr | 1 + src/llvm/ext/find-llvm-config | 9 ++++++- src/llvm/lib_llvm.cr | 4 +-- 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index e56a14a27c1c..b39c089bef99 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,8 @@ CRYSTAL_CONFIG_BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2> /dev/null) CRYSTAL_CONFIG_PATH := '$$ORIGIN/../share/crystal/src' CRYSTAL_VERSION ?= $(shell cat src/VERSION) SOURCE_DATE_EPOCH ?= $(shell (cat src/SOURCE_DATE_EPOCH || (git show -s --format=%ct HEAD || stat -c "%Y" Makefile || stat -f "%m" Makefile)) 2> /dev/null) -ifeq ($(shell command -v ld.lld >/dev/null && uname -s),Linux) +check_lld := command -v ld.lld >/dev/null && case "$$(uname -s)" in MINGW32*|MINGW64*|Linux) echo 1;; esac +ifeq ($(shell $(check_lld)),1) EXPORT_CC ?= CC="$(CC) -fuse-ld=lld" endif EXPORTS := \ @@ -60,11 +61,20 @@ EXPORTS_BUILD := \ CRYSTAL_CONFIG_LIBRARY_PATH=$(CRYSTAL_CONFIG_LIBRARY_PATH) SHELL = sh LLVM_CONFIG := $(shell src/llvm/ext/find-llvm-config) -LLVM_VERSION := $(if $(LLVM_CONFIG),$(shell $(LLVM_CONFIG) --version 2> /dev/null)) +LLVM_VERSION := $(if $(LLVM_CONFIG),$(shell "$(LLVM_CONFIG)" --version 2> /dev/null)) LLVM_EXT_DIR = src/llvm/ext LLVM_EXT_OBJ = $(LLVM_EXT_DIR)/llvm_ext.o CXXFLAGS += $(if $(debug),-g -O0) +# MSYS2 support (native Windows should use `Makefile.win` instead) +ifeq ($(OS),Windows_NT) + CRYSTAL_BIN := crystal.exe + WINDOWS := 1 +else + CRYSTAL_BIN := crystal + WINDOWS := +endif + DESTDIR ?= PREFIX ?= /usr/local BINDIR ?= $(DESTDIR)$(PREFIX)/bin @@ -74,9 +84,9 @@ DATADIR ?= $(DESTDIR)$(PREFIX)/share/crystal INSTALL ?= /usr/bin/install ifeq ($(or $(TERM),$(TERM),dumb),dumb) - colorize = $(shell printf >&2 "$1") + colorize = $(shell printf "%s" "$1" >&2) else - colorize = $(shell printf >&2 "\033[33m$1\033[0m\n") + colorize = $(shell printf "\033[33m%s\033[0m\n" "$1" >&2) endif DEPS = $(LLVM_EXT_OBJ) @@ -119,7 +129,7 @@ interpreter_spec: $(O)/interpreter_spec ## Run interpreter specs .PHONY: smoke_test smoke_test: ## Build specs as a smoke test -smoke_test: $(O)/std_spec $(O)/compiler_spec $(O)/crystal +smoke_test: $(O)/std_spec $(O)/compiler_spec $(O)/$(CRYSTAL_BIN) .PHONY: all_spec all_spec: $(O)/all_spec ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) @@ -136,7 +146,7 @@ docs: ## Generate standard library documentation cp -av doc/ docs/ .PHONY: crystal -crystal: $(O)/crystal ## Build the compiler +crystal: $(O)/$(CRYSTAL_BIN) ## Build the compiler .PHONY: deps llvm_ext deps: $(DEPS) ## Build dependencies @@ -151,9 +161,9 @@ generate_data: ## Run generator scripts for Unicode, SSL config, ... $(MAKE) -B -f scripts/generate_data.mk .PHONY: install -install: $(O)/crystal man/crystal.1.gz ## Install the compiler at DESTDIR +install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(BINDIR)/" - $(INSTALL) -m 0755 "$(O)/crystal" "$(BINDIR)/crystal" + $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" $(INSTALL) -d -m 0755 $(DATADIR) cp -av src "$(DATADIR)/src" @@ -171,9 +181,16 @@ install: $(O)/crystal man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(DESTDIR)$(PREFIX)/share/fish/vendor_completions.d/" $(INSTALL) -m 644 etc/completion.fish "$(DESTDIR)$(PREFIX)/share/fish/vendor_completions.d/crystal.fish" +ifeq ($(WINDOWS),1) +.PHONY: install_dlls +install_dlls: $(O)/$(CRYSTAL_BIN) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) + $(INSTALL) -d -m 0755 "$(BINDIR)/" + @ldd $(O)/$(CRYSTAL_BIN) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" +endif + .PHONY: uninstall uninstall: ## Uninstall the compiler from DESTDIR - rm -f "$(BINDIR)/crystal" + rm -f "$(BINDIR)/$(CRYSTAL_BIN)" rm -rf "$(DATADIR)/src" @@ -210,7 +227,7 @@ $(O)/compiler_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr --release -$(O)/primitives_spec: $(O)/crystal $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/primitives_spec: $(O)/$(CRYSTAL_BIN) $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/primitives_spec.cr @@ -219,12 +236,15 @@ $(O)/interpreter_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler/interpreter_spec.cr -$(O)/crystal: $(DEPS) $(SOURCES) +$(O)/$(CRYSTAL_BIN): $(DEPS) $(SOURCES) $(call check_llvm_config) @mkdir -p $(O) @# NOTE: USE_PCRE1 is only used for testing compatibility with legacy environments that don't provide libpcre2. @# Newly built compilers should never be distributed with libpcre to ensure syntax consistency. - $(EXPORTS) $(EXPORTS_BUILD) ./bin/crystal build $(FLAGS) -o $@ src/compiler/crystal.cr -D without_openssl -D without_zlib $(if $(USE_PCRE1),-D use_pcre,-D use_pcre2) + $(EXPORTS) $(EXPORTS_BUILD) ./bin/crystal build $(FLAGS) -o $(if $(WINDOWS),$(O)/crystal-next.exe,$@) src/compiler/crystal.cr -D without_openssl -D without_zlib $(if $(USE_PCRE1),-D use_pcre,-D use_pcre2) + @# NOTE: on MSYS2 it is not possible to overwrite a running program, so the compiler must be first built with + @# a different filename and then moved to the final destination. + $(if $(WINDOWS),mv $(O)/crystal-next.exe $@) $(LLVM_EXT_OBJ): $(LLVM_EXT_DIR)/llvm_ext.cc $(call check_llvm_config) diff --git a/bin/crystal b/bin/crystal index 2282cbefec80..e8abdff30ee8 100755 --- a/bin/crystal +++ b/bin/crystal @@ -184,9 +184,18 @@ fi # with symlinks resolved as well (see https://github.com/crystal-lang/crystal/issues/12969). cd "$(realpath "$(pwd)")" -if [ -x "$CRYSTAL_DIR/crystal" ]; then - __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/crystal" - exec "$CRYSTAL_DIR/crystal" "$@" +case "$(uname -s)" in + CYGWIN*|MSYS_NT*|MINGW32_NT*|MINGW64_NT*) + CRYSTAL_BIN="crystal.exe" + ;; + *) + CRYSTAL_BIN="crystal" + ;; +esac + +if [ -x "$CRYSTAL_DIR/${CRYSTAL_BIN}" ]; then + __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/${CRYSTAL_BIN}" + exec "$CRYSTAL_DIR/${CRYSTAL_BIN}" "$@" elif !($PARENT_CRYSTAL_EXISTS); then __error_msg 'You need to have a crystal executable in your path! or set CRYSTAL env variable' exit 1 diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index aa11ef1dc47e..6c7664bacc25 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -476,6 +476,7 @@ module Crystal {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} elsif program.has_flag?("win32") && program.has_flag?("gnu") link_flags = @link_flags || "" + link_flags += " -Wl,--stack,0x800000" lib_flags = program.lib_flags(@cross_compile) lib_flags = expand_lib_flags(lib_flags) if expand cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}) diff --git a/src/llvm/ext/find-llvm-config b/src/llvm/ext/find-llvm-config index 5f40a1f2e6a3..5aa381aaf13b 100755 --- a/src/llvm/ext/find-llvm-config +++ b/src/llvm/ext/find-llvm-config @@ -16,7 +16,14 @@ if ! LLVM_CONFIG=$(command -v "$LLVM_CONFIG"); then fi if [ "$LLVM_CONFIG" ]; then - printf %s "$LLVM_CONFIG" + case "$(uname -s)" in + MINGW32_NT*|MINGW64_NT*) + printf "%s" "$(cygpath -w "$LLVM_CONFIG")" + ;; + *) + printf "%s" "$LLVM_CONFIG" + ;; + esac else printf "Error: Could not find location of llvm-config. Please specify path in environment variable LLVM_CONFIG.\n" >&2 printf "Supported LLVM versions: $(cat "$(dirname $0)/llvm-versions.txt" | sed 's/\.0//g')\n" >&2 diff --git a/src/llvm/lib_llvm.cr b/src/llvm/lib_llvm.cr index 4c7ed49e7900..8b6856631b55 100644 --- a/src/llvm/lib_llvm.cr +++ b/src/llvm/lib_llvm.cr @@ -1,5 +1,5 @@ {% begin %} - {% if flag?(:win32) && !flag?(:static) %} + {% if flag?(:msvc) && !flag?(:static) %} {% config = nil %} {% for dir in Crystal::LIBRARY_PATH.split(Crystal::System::Process::HOST_PATH_DELIMITER) %} {% config ||= read_file?("#{dir.id}/llvm_VERSION") %} @@ -21,7 +21,7 @@ lib LibLLVM end {% else %} - {% llvm_config = env("LLVM_CONFIG") || `#{__DIR__}/ext/find-llvm-config`.stringify %} + {% llvm_config = env("LLVM_CONFIG") || `sh #{__DIR__}/ext/find-llvm-config`.stringify %} {% llvm_version = `#{llvm_config.id} --version`.stringify %} {% llvm_targets = env("LLVM_TARGETS") || `#{llvm_config.id} --targets-built`.stringify %} {% llvm_ldflags = "`#{llvm_config.id} --libs --system-libs --ldflags#{" --link-static".id if flag?(:static)}#{" 2> /dev/null".id unless flag?(:win32)}`" %} From cf801e7cd1c31d5524303c18f7e8057eb684d5c6 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 22 Oct 2024 04:16:59 +0800 Subject: [PATCH 166/193] Restrict GitHub token permissions of CI workflows (#15087) --- .github/workflows/aarch64.yml | 2 ++ .github/workflows/docs.yml | 2 ++ .github/workflows/interpreter.yml | 2 ++ .github/workflows/linux.yml | 2 ++ .github/workflows/llvm.yml | 2 ++ .github/workflows/macos.yml | 2 ++ .github/workflows/openssl.yml | 2 ++ .github/workflows/regex-engine.yml | 2 ++ .github/workflows/smoke.yml | 2 ++ .github/workflows/wasm32.yml | 2 ++ .github/workflows/win.yml | 2 ++ .github/workflows/win_build_portable.yml | 2 ++ 12 files changed, 24 insertions(+) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index aec37c3860e1..14e7c3d9f564 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -2,6 +2,8 @@ name: AArch64 CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 57147238552e..9e576303f479 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,6 +5,8 @@ on: branches: - master +permissions: {} + env: TRAVIS_OS_NAME: linux diff --git a/.github/workflows/interpreter.yml b/.github/workflows/interpreter.yml index 3c74afdd329e..103dc766509b 100644 --- a/.github/workflows/interpreter.yml +++ b/.github/workflows/interpreter.yml @@ -2,6 +2,8 @@ name: Interpreter Test on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ad65fa005259..79c3f143d303 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,6 +2,8 @@ name: Linux CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 65d0744575b9..8caebb3c9c95 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -2,6 +2,8 @@ name: LLVM CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index d4c93a68aabb..8ae3ac28209e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -2,6 +2,8 @@ name: macOS CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/openssl.yml b/.github/workflows/openssl.yml index 7321abddf788..611413e7e678 100644 --- a/.github/workflows/openssl.yml +++ b/.github/workflows/openssl.yml @@ -2,6 +2,8 @@ name: OpenSSL CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index e7ee002103b4..26b406b84d3f 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -2,6 +2,8 @@ name: Regex Engine CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 7ae103e528cf..5a83a26e815a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -31,6 +31,8 @@ name: Smoke tests on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index d0012b67c40f..2bb03f6cc5a3 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -2,6 +2,8 @@ name: WebAssembly CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 9025586fa991..03aac8e2f0b1 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -2,6 +2,8 @@ name: Windows CI on: [push, pull_request] +permissions: {} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/win_build_portable.yml b/.github/workflows/win_build_portable.yml index 889f4d80c629..a81b9e8083ed 100644 --- a/.github/workflows/win_build_portable.yml +++ b/.github/workflows/win_build_portable.yml @@ -10,6 +10,8 @@ on: required: true type: string +permissions: {} + jobs: build: runs-on: windows-2022 From 6b390b0617eb41632c1cb3b5c184dcb1487e3bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 22 Oct 2024 13:18:54 +0200 Subject: [PATCH 167/193] Fix `UNIXSocket#receive` (#15107) --- spec/std/socket/unix_socket_spec.cr | 24 ++++++++++++++++++++++++ src/socket/unix_socket.cr | 6 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index b3bc4931ec78..7e5eda4e2b65 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -63,6 +63,30 @@ describe UNIXSocket do end end + it "#send, #receive" do + with_tempfile("unix_socket-receive.sock") do |path| + UNIXServer.open(path) do |server| + UNIXSocket.open(path) do |client| + server.accept do |sock| + client.send "ping" + message, address = sock.receive + message.should eq("ping") + typeof(address).should eq(Socket::UNIXAddress) + address.path.should eq "" + + sock.send "pong" + message, address = client.receive + message.should eq("pong") + typeof(address).should eq(Socket::UNIXAddress) + # The value of path seems to be system-specific. Some implementations + # return the socket path, others an empty path. + ["", path].should contain address.path + end + end + end + end + end + # `LibC.socketpair` is not supported in Winsock 2.0 yet: # https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/#unsupportedunavailable {% unless flag?(:win32) %} diff --git a/src/socket/unix_socket.cr b/src/socket/unix_socket.cr index 201fd8410bf7..d5ce5857c907 100644 --- a/src/socket/unix_socket.cr +++ b/src/socket/unix_socket.cr @@ -97,8 +97,8 @@ class UNIXSocket < Socket UNIXAddress.new(path.to_s) end - def receive - bytes_read, sockaddr, addrlen = recvfrom - {bytes_read, UNIXAddress.from(sockaddr, addrlen)} + def receive(max_message_size = 512) : {String, UNIXAddress} + message, address = super(max_message_size) + {message, address.as(UNIXAddress)} end end From 5f72133db005325db80312759c56730ee9a09c03 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 22 Oct 2024 19:19:24 +0800 Subject: [PATCH 168/193] Refactor uses of `LibC.dladdr` inside `Exception::CallStack` (#15108) The upcoming MinGW-w64 support for call stacks relies on `Exception::CallStack.unsafe_decode_frame` pointing to a stack-allocated string, so in order to avoid a dangling reference into the unused portion of the stack, this patch converts all the relevant methods into yielding methods. --- src/exception/call_stack/libunwind.cr | 66 ++++++++++++++------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/src/exception/call_stack/libunwind.cr b/src/exception/call_stack/libunwind.cr index 73a851a00339..1542d52cc736 100644 --- a/src/exception/call_stack/libunwind.cr +++ b/src/exception/call_stack/libunwind.cr @@ -122,32 +122,18 @@ struct Exception::CallStack end {% end %} - if frame = unsafe_decode_frame(repeated_frame.ip) - offset, sname, fname = frame + unsafe_decode_frame(repeated_frame.ip) do |offset, sname, fname| Crystal::System.print_error "%s +%lld in %s", sname, offset.to_i64, fname - else - Crystal::System.print_error "???" + return end - end - protected def self.decode_frame(ip, original_ip = ip) - if LibC.dladdr(ip, out info) != 0 - offset = original_ip - info.dli_saddr + Crystal::System.print_error "???" + end - if offset == 0 - return decode_frame(ip - 1, original_ip) - end - return if info.dli_sname.null? && info.dli_fname.null? - if info.dli_sname.null? - symbol = "??" - else - symbol = String.new(info.dli_sname) - end - if info.dli_fname.null? - file = "??" - else - file = String.new(info.dli_fname) - end + protected def self.decode_frame(ip) + decode_frame(ip) do |offset, symbol, file| + symbol = symbol ? String.new(symbol) : "??" + file = file ? String.new(file) : "??" {offset, symbol, file} end end @@ -155,19 +141,35 @@ struct Exception::CallStack # variant of `.decode_frame` that returns the C strings directly instead of # wrapping them in `String.new`, since the SIGSEGV handler cannot allocate # memory via the GC - protected def self.unsafe_decode_frame(ip) + protected def self.unsafe_decode_frame(ip, &) + decode_frame(ip) do |offset, symbol, file| + symbol ||= "??".to_unsafe + file ||= "??".to_unsafe + yield offset, symbol, file + end + end + + private def self.decode_frame(ip, &) original_ip = ip - while LibC.dladdr(ip, out info) != 0 - offset = original_ip - info.dli_saddr - if offset == 0 - ip -= 1 - next + while true + retry = dladdr(ip) do |file, symbol, address| + offset = original_ip - address + if offset == 0 + ip -= 1 + true + elsif symbol.null? && file.null? + false + else + return yield offset, symbol, file + end end + break unless retry + end + end - return if info.dli_sname.null? && info.dli_fname.null? - symbol = info.dli_sname || "??".to_unsafe - file = info.dli_fname || "??".to_unsafe - return {offset, symbol, file} + private def self.dladdr(ip, &) + if LibC.dladdr(ip, out info) != 0 + yield info.dli_fname, info.dli_sname, info.dli_saddr end end end From 89b1a92a531969f2a3d0d02a0359f8b94578c629 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 23 Oct 2024 04:36:17 +0800 Subject: [PATCH 169/193] Use official Apt respositories for LLVM CI (#15103) Co-authored-by: Oleh Prypin --- .github/workflows/llvm.yml | 46 ++++++++++---------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 8caebb3c9c95..a69383319542 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -13,51 +13,29 @@ env: jobs: llvm_test: - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false matrix: include: - - llvm_version: "13.0.0" - llvm_filename: "clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" - - llvm_version: "14.0.0" - llvm_filename: "clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - - llvm_version: "15.0.6" - llvm_filename: "clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - - llvm_version: "16.0.3" - llvm_filename: "clang+llvm-16.0.3-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - - llvm_version: "17.0.6" - llvm_filename: "clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - - llvm_version: "18.1.4" - llvm_filename: "clang+llvm-18.1.4-x86_64-linux-gnu-ubuntu-18.04.tar.xz" - - llvm_version: "19.1.0" - llvm_filename: "LLVM-19.1.0-Linux-X64.tar.xz" + - {llvm_version: 13, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 14, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 15, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 16, runs-on: ubuntu-22.04, codename: jammy} + - {llvm_version: 17, runs-on: ubuntu-24.04, codename: noble} + - {llvm_version: 18, runs-on: ubuntu-24.04, codename: noble} + - {llvm_version: 19, runs-on: ubuntu-24.04, codename: noble} name: "LLVM ${{ matrix.llvm_version }}" steps: - name: Checkout Crystal source uses: actions/checkout@v4 - - name: Cache LLVM - id: cache-llvm - uses: actions/cache@v4 - with: - path: ./llvm - key: llvm-${{ matrix.llvm_version }} - if: "${{ !env.ACT }}" - - name: Install LLVM ${{ matrix.llvm_version }} run: | - mkdir -p llvm - curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.llvm_version }}/${{ matrix.llvm_filename }}" > llvm.tar.xz - tar x --xz -C llvm --strip-components=1 -f llvm.tar.xz - if: steps.cache-llvm.outputs.cache-hit != 'true' - - - name: Set up LLVM - run: | - sudo apt-get install -y libtinfo5 - echo "PATH=$(pwd)/llvm/bin:$PATH" >> $GITHUB_ENV - echo "LLVM_CONFIG=$(pwd)/llvm/bin/llvm-config" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(pwd)/llvm/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + sudo apt remove 'llvm-*' 'libllvm*' + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + sudo apt-add-repository -y deb http://apt.llvm.org/${{ matrix.codename }}/ llvm-toolchain-${{ matrix.codename }}-${{ matrix.llvm_version }} main + sudo apt install -y llvm-${{ matrix.llvm_version }}-dev lld - name: Install Crystal uses: crystal-lang/install-crystal@v1 From 823739796219bb1994b02b1b3db99061ef38f409 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 23 Oct 2024 04:36:46 +0800 Subject: [PATCH 170/193] Drop LLVM Apt installer script on WebAssembly CI (#15109) --- .github/workflows/wasm32.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wasm32.yml b/.github/workflows/wasm32.yml index 2bb03f6cc5a3..9a6472ca2d6e 100644 --- a/.github/workflows/wasm32.yml +++ b/.github/workflows/wasm32.yml @@ -13,7 +13,7 @@ env: jobs: wasm32-test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: crystallang/crystal:1.14.0-build steps: - name: Download Crystal source @@ -27,10 +27,11 @@ jobs: - name: Install LLVM run: | apt-get update - apt-get install -y curl lsb-release wget software-properties-common gnupg - curl -O https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - ./llvm.sh 18 + apt-get remove -y 'llvm-*' 'libllvm*' + apt-get install -y curl software-properties-common + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + apt-add-repository -y deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main + apt-get install -y llvm-18-dev lld-18 ln -s $(which wasm-ld-18) /usr/bin/wasm-ld - name: Download wasm32 libs From 9d8c2e4a1a7521949324a30aa6774495620fa5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustavo=20Gir=C3=A1ldez?= Date: Wed, 23 Oct 2024 04:38:09 -0400 Subject: [PATCH 171/193] Inline `ASTNode` bindings dependencies and observers (#15098) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every `ASTNode` contains two arrays used for type inference and checking: dependencies and observers. By default, these are created lazily, but most active (ie. effectively typed) `ASTNode`s end up creating them. Furthermore, on average both these lists contain less than 2 elements each. This PR replaces both `Array(ASTNode)?` references in `ASTNode` by inlined structs that can hold two elements and a tail array for the cases where more links are needed. This reduces the number of allocations, bytes allocated, number of instructions executed and running time. Some numbers from compiling the Crystal compiler itself without running codegen (since type binding occurs in the semantic phase anyway). * Running time (measured with hyperfine running with `GC_DONT_GC=1`): ~6% reduction Before: ``` Benchmark 1: ./self-semantic-only.sh Time (mean ± σ): 3.398 s ± 0.152 s [User: 2.264 s, System: 0.470 s] Range (min … max): 3.029 s … 3.575 s 10 runs ``` After: ``` Benchmark 1: ./self-semantic-only.sh Time (mean ± σ): 3.180 s ± 0.095 s [User: 2.153 s, System: 0.445 s] Range (min … max): 3.046 s … 3.345 s 10 runs ``` * Memory (as reported by the compiler itself, with GC): ~9.6% reduction Before: ``` Parse: 00:00:00.000038590 ( 1.05MB) Semantic (top level): 00:00:00.483357706 ( 174.13MB) Semantic (new): 00:00:00.002156811 ( 174.13MB) Semantic (type declarations): 00:00:00.038313066 ( 174.13MB) Semantic (abstract def check): 00:00:00.014283169 ( 190.13MB) Semantic (restrictions augmenter): 00:00:00.010672651 ( 206.13MB) Semantic (ivars initializers): 00:00:04.660611385 (1250.07MB) Semantic (cvars initializers): 00:00:00.008343907 (1250.07MB) Semantic (main): 00:00:00.780627942 (1346.07MB) Semantic (cleanup): 00:00:00.000961914 (1346.07MB) Semantic (recursive struct check): 00:00:00.001121766 (1346.07MB) ``` After: ``` Parse: 00:00:00.000044417 ( 1.05MB) Semantic (top level): 00:00:00.546445955 ( 190.03MB) Semantic (new): 00:00:00.002488975 ( 190.03MB) Semantic (type declarations): 00:00:00.040234541 ( 206.03MB) Semantic (abstract def check): 00:00:00.015473723 ( 222.03MB) Semantic (restrictions augmenter): 00:00:00.010828366 ( 222.03MB) Semantic (ivars initializers): 00:00:03.324639987 (1135.96MB) Semantic (cvars initializers): 00:00:00.007359853 (1135.96MB) Semantic (main): 00:00:01.806822202 (1217.96MB) Semantic (cleanup): 00:00:00.000626975 (1217.96MB) Semantic (recursive struct check): 00:00:00.001435494 (1217.96MB) ``` * Callgrind stats: - Instruction refs: 17,477,865,704 -> 16,712,610,033 (~4.4% reduction) - Estimated cycles: 26,835,733,874 -> 26,154,926,143 (~2.5% reduction) - `GC_malloc_kind` call count: 35,161,616 -> 25,684,997 (~27% reduction) Co-authored-by: Oleh Prypin Co-authored-by: Johannes Müller --- src/compiler/crystal/semantic/bindings.cr | 111 ++++++++++++++---- src/compiler/crystal/semantic/call_error.cr | 3 +- .../crystal/semantic/cleanup_transformer.cr | 5 +- src/compiler/crystal/semantic/filters.cr | 2 +- src/compiler/crystal/semantic/main_visitor.cr | 4 +- src/compiler/crystal/semantic/type_merge.cr | 10 +- 6 files changed, 97 insertions(+), 38 deletions(-) diff --git a/src/compiler/crystal/semantic/bindings.cr b/src/compiler/crystal/semantic/bindings.cr index c5fe9f164742..a7dacb8668c9 100644 --- a/src/compiler/crystal/semantic/bindings.cr +++ b/src/compiler/crystal/semantic/bindings.cr @@ -1,7 +1,77 @@ module Crystal + # Specialized container for ASTNodes to use for bindings tracking. + # + # The average number of elements in both dependencies and observers is below 2 + # for ASTNodes. This struct inlines the first two elements saving up 4 + # allocations per node (two arrays, with a header and buffer for each) but we + # need to pay a slight extra cost in memory upfront: a total of 6 pointers (48 + # bytes) vs 2 pointers (16 bytes). The other downside is that since this is a + # struct, we need to be careful with mutation. + struct SmallNodeList + include Enumerable(ASTNode) + + @first : ASTNode? + @second : ASTNode? + @tail : Array(ASTNode)? + + def each(& : ASTNode ->) + yield @first || return + yield @second || return + @tail.try(&.each { |node| yield node }) + end + + def size + if @first.nil? + 0 + elsif @second.nil? + 1 + elsif (tail = @tail).nil? + 2 + else + 2 + tail.size + end + end + + def push(node : ASTNode) : self + if @first.nil? + @first = node + elsif @second.nil? + @second = node + elsif (tail = @tail).nil? + @tail = [node] of ASTNode + else + tail.push(node) + end + self + end + + def reject!(& : ASTNode ->) : self + if first = @first + if second = @second + if tail = @tail + tail.reject! { |node| yield node } + end + if yield second + @second = tail.try &.shift? + end + end + if yield first + @first = @second + @second = tail.try &.shift? + end + end + self + end + + def concat(nodes : Enumerable(ASTNode)) : self + nodes.each { |node| self.push(node) } + self + end + end + class ASTNode - property! dependencies : Array(ASTNode) - property observers : Array(ASTNode)? + getter dependencies : SmallNodeList = SmallNodeList.new + @observers : SmallNodeList = SmallNodeList.new property enclosing_call : Call? @dirty = false @@ -107,8 +177,8 @@ module Crystal end def bind_to(node : ASTNode) : Nil - bind(node) do |dependencies| - dependencies.push node + bind(node) do + @dependencies.push node node.add_observer self end end @@ -116,8 +186,8 @@ module Crystal def bind_to(nodes : Indexable) : Nil return if nodes.empty? - bind do |dependencies| - dependencies.concat nodes + bind do + @dependencies.concat nodes nodes.each &.add_observer self end end @@ -130,9 +200,7 @@ module Crystal raise_frozen_type freeze_type, from_type, from end - dependencies = @dependencies ||= [] of ASTNode - - yield dependencies + yield new_type = type_from_dependencies new_type = map_type(new_type) if new_type @@ -150,7 +218,7 @@ module Crystal end def type_from_dependencies : Type? - Type.merge dependencies + Type.merge @dependencies end def unbind_from(nodes : Nil) @@ -158,18 +226,17 @@ module Crystal end def unbind_from(node : ASTNode) - @dependencies.try &.reject! &.same?(node) + @dependencies.reject! &.same?(node) node.remove_observer self end - def unbind_from(nodes : Array(ASTNode)) - @dependencies.try &.reject! { |dep| nodes.any? &.same?(dep) } + def unbind_from(nodes : Enumerable(ASTNode)) + @dependencies.reject! { |dep| nodes.any? &.same?(dep) } nodes.each &.remove_observer self end def add_observer(observer) - observers = @observers ||= [] of ASTNode - observers.push observer + @observers.push observer end def remove_observer(observer) @@ -269,16 +336,10 @@ module Crystal visited = Set(ASTNode).new.compare_by_identity owner_trace << node if node.type?.try &.includes_type?(owner) visited.add node - while deps = node.dependencies? - dependencies = deps.select { |dep| dep.type? && dep.type.includes_type?(owner) && !visited.includes?(dep) } - if dependencies.size > 0 - node = dependencies.first - nil_reason = node.nil_reason if node.is_a?(MetaTypeVar) - owner_trace << node if node - visited.add node - else - break - end + while node = node.dependencies.find { |dep| dep.type? && dep.type.includes_type?(owner) && !visited.includes?(dep) } + nil_reason = node.nil_reason if node.is_a?(MetaTypeVar) + owner_trace << node if node + visited.add node end MethodTraceException.new(owner, owner_trace, nil_reason, program.show_error_trace?) diff --git a/src/compiler/crystal/semantic/call_error.cr b/src/compiler/crystal/semantic/call_error.cr index aee5b9e2019b..d19be20afbad 100644 --- a/src/compiler/crystal/semantic/call_error.cr +++ b/src/compiler/crystal/semantic/call_error.cr @@ -643,8 +643,7 @@ class Crystal::Call if obj.is_a?(InstanceVar) scope = self.scope ivar = scope.lookup_instance_var(obj.name) - deps = ivar.dependencies? - if deps && deps.size == 1 && deps.first.same?(program.nil_var) + if ivar.dependencies.size == 1 && ivar.dependencies.first.same?(program.nil_var) similar_name = scope.lookup_similar_instance_var_name(ivar.name) if similar_name msg << colorize(" (#{ivar.name} was never assigned a value, did you mean #{similar_name}?)").yellow.bold diff --git a/src/compiler/crystal/semantic/cleanup_transformer.cr b/src/compiler/crystal/semantic/cleanup_transformer.cr index 541e0f51d662..054c7871bd8e 100644 --- a/src/compiler/crystal/semantic/cleanup_transformer.cr +++ b/src/compiler/crystal/semantic/cleanup_transformer.cr @@ -1090,10 +1090,7 @@ module Crystal node = super unless node.type? - if dependencies = node.dependencies? - node.unbind_from node.dependencies - end - + node.unbind_from node.dependencies node.bind_to node.expressions end diff --git a/src/compiler/crystal/semantic/filters.cr b/src/compiler/crystal/semantic/filters.cr index 66d1a728804b..7dd253fc2292 100644 --- a/src/compiler/crystal/semantic/filters.cr +++ b/src/compiler/crystal/semantic/filters.cr @@ -1,7 +1,7 @@ module Crystal class TypeFilteredNode < ASTNode def initialize(@filter : TypeFilter, @node : ASTNode) - @dependencies = [@node] of ASTNode + @dependencies.push @node node.add_observer self update(@node) end diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index 905d5bac8cb1..efd76f76f056 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -373,7 +373,7 @@ module Crystal var.bind_to(@program.nil_var) var.nil_if_read = false - meta_var.bind_to(@program.nil_var) unless meta_var.dependencies.try &.any? &.same?(@program.nil_var) + meta_var.bind_to(@program.nil_var) unless meta_var.dependencies.any? &.same?(@program.nil_var) node.bind_to(@program.nil_var) end @@ -1283,7 +1283,7 @@ module Crystal # It can happen that this call is inside an ArrayLiteral or HashLiteral, # was expanded but isn't bound to the expansion because the call (together # with its expansion) was cloned. - if (expanded = node.expanded) && (!node.dependencies? || !node.type?) + if (expanded = node.expanded) && (node.dependencies.empty? || !node.type?) node.bind_to(expanded) end diff --git a/src/compiler/crystal/semantic/type_merge.cr b/src/compiler/crystal/semantic/type_merge.cr index 874949dd516d..67e9f1b61911 100644 --- a/src/compiler/crystal/semantic/type_merge.cr +++ b/src/compiler/crystal/semantic/type_merge.cr @@ -17,7 +17,7 @@ module Crystal end end - def type_merge(nodes : Array(ASTNode)) : Type? + def type_merge(nodes : Enumerable(ASTNode)) : Type? case nodes.size when 0 nil @@ -25,8 +25,10 @@ module Crystal nodes.first.type? when 2 # Merging two types is the most common case, so we optimize it - first, second = nodes - type_merge(first.type?, second.type?) + # We use `#each_cons_pair` to avoid any intermediate allocation + nodes.each_cons_pair do |first, second| + return type_merge(first.type?, second.type?) + end else combined_union_of compact_types(nodes, &.type?) end @@ -161,7 +163,7 @@ module Crystal end class Type - def self.merge(nodes : Array(ASTNode)) : Type? + def self.merge(nodes : Enumerable(ASTNode)) : Type? nodes.find(&.type?).try &.type.program.type_merge(nodes) end From f2a6628672557ff95b645f67fee8dd6b3092f667 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 23 Oct 2024 16:38:35 +0800 Subject: [PATCH 172/193] Add CI workflow for cross-compiling Crystal on MSYS2 (#15110) Cross-compiles a MinGW-w64-based Crystal compiler from Ubuntu, then links it on MSYS2's UCRT64 environment. Resolves part of #6170. The artifact includes the compiler, all dependent DLLs, and the source code only. It is not a complete installation since it is missing e.g. the documentation and the licenses, but it is sufficient for bootstrapping further native compiler builds within MSYS2. The resulting binary is portable within MSYS2 and can be executed from anywhere inside an MSYS2 shell, although compilation requires `mingw-w64-ucrt-x86_64-cc`, probably `mingw-w64-ucrt-x86_64-pkgconf`, plus the respective development libraries listed in #15077. The DLLs bundled under `bin/` are needed to even start Crystal since they are dynamically linked at load time; they are not strictly needed if Crystal is always run only within MSYS2, but that is the job of an actual `PKGBUILD` file. --- .github/workflows/mingw-w64.yml | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/mingw-w64.yml diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml new file mode 100644 index 000000000000..2370ae133cdd --- /dev/null +++ b/.github/workflows/mingw-w64.yml @@ -0,0 +1,91 @@ +name: MinGW-w64 CI + +on: [push, pull_request] + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +jobs: + x86_64-mingw-w64-cross-compile: + runs-on: ubuntu-24.04 + steps: + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Install LLVM 18 + run: | + sudo apt remove 'llvm-*' 'libllvm*' + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + sudo apt-add-repository -y deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main + sudo apt install -y llvm-18-dev + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1 + with: + crystal: "1.14.0" + + - name: Cross-compile Crystal + run: make && make -B target=x86_64-windows-gnu release=1 + + - name: Upload crystal.obj + uses: actions/upload-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-obj + path: .build/crystal.obj + + - name: Upload standard library + uses: actions/upload-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-stdlib + path: src + + x86_64-mingw-w64-link: + runs-on: windows-2022 + needs: [x86_64-mingw-w64-cross-compile] + steps: + - name: Setup MSYS2 + id: msys2 + uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1 + with: + msystem: UCRT64 + update: true + install: >- + mingw-w64-ucrt-x86_64-pkgconf + mingw-w64-ucrt-x86_64-cc + mingw-w64-ucrt-x86_64-gc + mingw-w64-ucrt-x86_64-pcre2 + mingw-w64-ucrt-x86_64-libiconv + mingw-w64-ucrt-x86_64-zlib + mingw-w64-ucrt-x86_64-llvm + + - name: Download crystal.obj + uses: actions/download-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-obj + + - name: Download standard library + uses: actions/download-artifact@v4 + with: + name: x86_64-mingw-w64-crystal-stdlib + path: share/crystal/src + + - name: Link Crystal executable + shell: msys2 {0} + run: | + mkdir bin + cc crystal.obj -o bin/crystal.exe \ + $(pkg-config bdw-gc libpcre2-8 iconv zlib --libs) \ + $(llvm-config --libs --system-libs --ldflags) \ + -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 + ldd bin/crystal.exe | grep -iv /c/windows/system32 | sed 's/.* => //; s/ (.*//' | xargs -t -i cp '{}' bin/ + + - name: Upload Crystal + uses: actions/upload-artifact@v4 + with: + name: x86_64-mingw-w64-crystal + path: | + bin/ + share/ From b8475639fd087b00db5759f0a574ccf5fd67a18d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 23 Oct 2024 23:26:26 +0200 Subject: [PATCH 173/193] Fix: LibC bindings and std specs on NetBSD 10 (#15115) The LibC bindings for NetBSD were a bit wrong and some std specs also didn't work as expected. The segfault handler is also broken on NetBSD (the process crashes with SIGILL after receiving SIGSEGV). With these fixes + pending specs I can run the std and compiler test suites in a NetBSD 10.0 VM. **Caveat**: the pkgsrc for LLVM enforces partial RELRO and linking the std specs from crystal fails. You must pass `--single-module` for the executable to be able to start without missing runtime symbols from libxml2. See #11046. --- spec/std/io/io_spec.cr | 2 +- spec/std/kernel_spec.cr | 62 +++++++++++++++------------ spec/std/socket/tcp_server_spec.cr | 4 +- spec/std/socket/tcp_socket_spec.cr | 6 +-- spec/std/socket/udp_socket_spec.cr | 3 ++ spec/std/string_spec.cr | 8 ++-- src/crystal/system/unix/dir.cr | 7 ++- src/lib_c/x86_64-netbsd/c/dirent.cr | 1 - src/lib_c/x86_64-netbsd/c/netdb.cr | 1 + src/lib_c/x86_64-netbsd/c/sys/time.cr | 2 +- 10 files changed, 55 insertions(+), 41 deletions(-) diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 3be5c07e1479..620a1d034d9f 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -736,7 +736,7 @@ describe IO do it "says invalid byte sequence" do io = SimpleIOMemory.new(Slice.new(1, 255_u8)) io.set_encoding("EUC-JP") - expect_raises ArgumentError, {% if flag?(:musl) || flag?(:freebsd) %}"Incomplete multibyte sequence"{% else %}"Invalid multibyte sequence"{% end %} do + expect_raises ArgumentError, {% if flag?(:musl) || flag?(:freebsd) || flag?(:netbsd) %}"Incomplete multibyte sequence"{% else %}"Invalid multibyte sequence"{% end %} do io.read_char end end diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index f41529af901a..7f3c39d9e9ec 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -254,38 +254,44 @@ describe "hardware exception" do error.should_not contain("Stack overflow") end - it "detects stack overflow on the main stack", tags: %w[slow] do - # This spec can take some time under FreeBSD where - # the default stack size is 0.5G. Setting a - # smaller stack size with `ulimit -s 8192` - # will address this. - status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) + {% if flag?(:netbsd) %} + # FIXME: on netbsd the process crashes with SIGILL after receiving SIGSEGV + pending "detects stack overflow on the main stack" + pending "detects stack overflow on a fiber stack" + {% else %} + it "detects stack overflow on the main stack", tags: %w[slow] do + # This spec can take some time under FreeBSD where + # the default stack size is 0.5G. Setting a + # smaller stack size with `ulimit -s 8192` + # will address this. + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end foo - end - foo - CRYSTAL + CRYSTAL - status.success?.should be_false - error.should contain("Stack overflow") - end + status.success?.should be_false + error.should contain("Stack overflow") + end - it "detects stack overflow on a fiber stack", tags: %w[slow] do - status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) - foo - end + it "detects stack overflow on a fiber stack", tags: %w[slow] do + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end - spawn do - foo - end + spawn do + foo + end - sleep 60.seconds - CRYSTAL + sleep 60.seconds + CRYSTAL - status.success?.should be_false - error.should contain("Stack overflow") - end + status.success?.should be_false + error.should contain("Stack overflow") + end + {% end %} end diff --git a/spec/std/socket/tcp_server_spec.cr b/spec/std/socket/tcp_server_spec.cr index 0c6113a4a7ff..ee3c861956b8 100644 --- a/spec/std/socket/tcp_server_spec.cr +++ b/spec/std/socket/tcp_server_spec.cr @@ -96,7 +96,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -110,7 +110,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index f3d460f92401..5ec3467362e0 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -79,7 +79,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -93,7 +93,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) %} + {% elsif flag?(:android) || flag?(:netbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -142,7 +142,7 @@ describe TCPSocket, tags: "network" do (client.tcp_nodelay = false).should be_false client.tcp_nodelay?.should be_false - {% unless flag?(:openbsd) %} + {% unless flag?(:openbsd) || flag?(:netbsd) %} (client.tcp_keepalive_idle = 42).should eq 42 client.tcp_keepalive_idle.should eq 42 (client.tcp_keepalive_interval = 42).should eq 42 diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 6e4b607b80ea..9b624110fad9 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -85,6 +85,9 @@ describe UDPSocket, tags: "network" do elsif {{ flag?(:freebsd) }} && family == Socket::Family::INET6 # FIXME: fails with "Error sending datagram to [ipv6]:port: Network is unreachable" pending "joins and transmits to multicast groups" + elsif {{ flag?(:netbsd) }} && family == Socket::Family::INET6 + # FIXME: fails with "setsockopt: EADDRNOTAVAIL" + pending "joins and transmits to multicast groups" else it "joins and transmits to multicast groups" do udp = UDPSocket.new(family) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 2ffe5bf3d1fa..6d7487ded0e2 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -2830,7 +2830,7 @@ describe "String" do bytes.to_a.should eq([72, 0, 101, 0, 108, 0, 108, 0, 111, 0]) end - {% unless flag?(:musl) || flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) %} + {% unless flag?(:musl) || flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "flushes the shift state (#11992)" do "\u{00CA}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x66]) "\u{00CA}\u{0304}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x62]) @@ -2839,7 +2839,7 @@ describe "String" do # FreeBSD iconv encoder expects ISO/IEC 10646 compatibility code points, # see https://www.ccli.gov.hk/doc/e_hkscs_2008.pdf for details. - {% if flag?(:freebsd) || flag?(:dragonfly) %} + {% if flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "flushes the shift state (#11992)" do "\u{F329}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x66]) "\u{F325}".encode("BIG5-HKSCS").should eq(Bytes[0x88, 0x62]) @@ -2883,7 +2883,7 @@ describe "String" do String.new(bytes, "UTF-16LE").should eq("Hello") end - {% unless flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) %} + {% unless flag?(:solaris) || flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "decodes with shift state" do String.new(Bytes[0x88, 0x66], "BIG5-HKSCS").should eq("\u{00CA}") String.new(Bytes[0x88, 0x62], "BIG5-HKSCS").should eq("\u{00CA}\u{0304}") @@ -2892,7 +2892,7 @@ describe "String" do # FreeBSD iconv decoder returns ISO/IEC 10646-1:2000 code points, # see https://www.ccli.gov.hk/doc/e_hkscs_2008.pdf for details. - {% if flag?(:freebsd) || flag?(:dragonfly) %} + {% if flag?(:freebsd) || flag?(:dragonfly) || flag?(:netbsd) %} it "decodes with shift state" do String.new(Bytes[0x88, 0x66], "BIG5-HKSCS").should eq("\u{00CA}") String.new(Bytes[0x88, 0x62], "BIG5-HKSCS").should eq("\u{F325}") diff --git a/src/crystal/system/unix/dir.cr b/src/crystal/system/unix/dir.cr index 5e66b33b65e7..72d1183dcc72 100644 --- a/src/crystal/system/unix/dir.cr +++ b/src/crystal/system/unix/dir.cr @@ -42,7 +42,12 @@ module Crystal::System::Dir end def self.info(dir, path) : ::File::Info - Crystal::System::FileDescriptor.system_info LibC.dirfd(dir) + fd = {% if flag?(:netbsd) %} + dir.value.dd_fd + {% else %} + LibC.dirfd(dir) + {% end %} + Crystal::System::FileDescriptor.system_info(fd) end def self.close(dir, path) : Nil diff --git a/src/lib_c/x86_64-netbsd/c/dirent.cr b/src/lib_c/x86_64-netbsd/c/dirent.cr index 71dabe7b08ce..e3b8492083f7 100644 --- a/src/lib_c/x86_64-netbsd/c/dirent.cr +++ b/src/lib_c/x86_64-netbsd/c/dirent.cr @@ -29,5 +29,4 @@ lib LibC fun opendir = __opendir30(x0 : Char*) : DIR* fun readdir = __readdir30(x0 : DIR*) : Dirent* fun rewinddir(x0 : DIR*) : Void - fun dirfd(dirp : DIR*) : Int end diff --git a/src/lib_c/x86_64-netbsd/c/netdb.cr b/src/lib_c/x86_64-netbsd/c/netdb.cr index 4443325cd487..c098ab2f5fc6 100644 --- a/src/lib_c/x86_64-netbsd/c/netdb.cr +++ b/src/lib_c/x86_64-netbsd/c/netdb.cr @@ -13,6 +13,7 @@ lib LibC EAI_FAIL = 4 EAI_FAMILY = 5 EAI_MEMORY = 6 + EAI_NODATA = 7 EAI_NONAME = 8 EAI_SERVICE = 9 EAI_SOCKTYPE = 10 diff --git a/src/lib_c/x86_64-netbsd/c/sys/time.cr b/src/lib_c/x86_64-netbsd/c/sys/time.cr index f276784708c0..3bb54d42c5cd 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/time.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/time.cr @@ -13,5 +13,5 @@ lib LibC fun gettimeofday = __gettimeofday50(x0 : Timeval*, x1 : Timezone*) : Int fun utimes = __utimes50(path : Char*, times : Timeval[2]) : Int - fun futimens = __futimens50(fd : Int, times : Timespec[2]) : Int + fun futimens(fd : Int, times : Timespec[2]) : Int end From 2e8715d67a568ad4d4ea3d703fd8a0bbf2fe5434 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 05:27:02 +0800 Subject: [PATCH 174/193] Disable specs that break on MinGW-w64 (#15116) These specs do not have straightforward workarounds when run under MSYS2. --- spec/compiler/codegen/pointer_spec.cr | 53 ++++++++++++---------- spec/compiler/codegen/thread_local_spec.cr | 2 +- spec/std/kernel_spec.cr | 8 ++++ 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/spec/compiler/codegen/pointer_spec.cr b/spec/compiler/codegen/pointer_spec.cr index 1230d80cb5f6..da132cdee406 100644 --- a/spec/compiler/codegen/pointer_spec.cr +++ b/spec/compiler/codegen/pointer_spec.cr @@ -492,28 +492,33 @@ describe "Code gen: pointer" do )).to_b.should be_true end - it "takes pointerof lib external var" do - test_c( - %( - int external_var = 0; - ), - %( - lib LibFoo - $external_var : Int32 - end - - LibFoo.external_var = 1 - - ptr = pointerof(LibFoo.external_var) - x = ptr.value - - ptr.value = 10 - y = ptr.value - - ptr.value = 100 - z = LibFoo.external_var - - x + y + z - ), &.to_i.should eq(111)) - end + # FIXME: `$external_var` implies __declspec(dllimport), but we only have an + # object file, so MinGW-w64 fails linking (actually MSVC also emits an + # LNK4217 linker warning) + {% unless flag?(:win32) && flag?(:gnu) %} + it "takes pointerof lib external var" do + test_c( + %( + int external_var = 0; + ), + %( + lib LibFoo + $external_var : Int32 + end + + LibFoo.external_var = 1 + + ptr = pointerof(LibFoo.external_var) + x = ptr.value + + ptr.value = 10 + y = ptr.value + + ptr.value = 100 + z = LibFoo.external_var + + x + y + z + ), &.to_i.should eq(111)) + end + {% end %} end diff --git a/spec/compiler/codegen/thread_local_spec.cr b/spec/compiler/codegen/thread_local_spec.cr index 694cb430b8c1..386043f2c5fd 100644 --- a/spec/compiler/codegen/thread_local_spec.cr +++ b/spec/compiler/codegen/thread_local_spec.cr @@ -1,4 +1,4 @@ -{% skip_file if flag?(:openbsd) %} +{% skip_file if flag?(:openbsd) || (flag?(:win32) && flag?(:gnu)) %} require "../../spec_helper" diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index 7f3c39d9e9ec..f8e4ff1e8ae2 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -8,6 +8,14 @@ describe "PROGRAM_NAME" do pending! "Example is broken in Nix shell (#12332)" end + # MSYS2: gcc/ld doesn't support unicode paths + # https://github.com/msys2/MINGW-packages/issues/17812 + {% if flag?(:windows) %} + if ENV["MSYSTEM"]? + pending! "Example is broken in MSYS2 shell" + end + {% end %} + File.write(source_file, "File.basename(PROGRAM_NAME).inspect(STDOUT)") compile_file(source_file, bin_name: "×‽😂") do |executable_file| From 94386b640c4d70305bf12f7da9e2184694277587 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 05:27:25 +0800 Subject: [PATCH 175/193] Treat `WinError::ERROR_DIRECTORY` as an error for non-existent files (#15114) --- src/crystal/system/win32/file.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 7b7b443ce310..b6f9cf2b7ccd 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -116,6 +116,7 @@ module Crystal::System::File WinError::ERROR_FILE_NOT_FOUND, WinError::ERROR_PATH_NOT_FOUND, WinError::ERROR_INVALID_NAME, + WinError::ERROR_DIRECTORY, } def self.check_not_found_error(message, path) From 454744a35e34fdd995c22200a32f67a2b4320f1c Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 18:30:42 +0800 Subject: [PATCH 176/193] Support call stacks for MinGW-w64 builds (#15117) Introduces new methods for extracting COFF debug information from programs in the PE format, integrating them into Crystal's existing DWARF parsing functionality. Resolves part of #6170. It is questionable whether reusing `src/exception/call_stack/elf.cr` for MinGW-w64 is appropriate, since nothing here is in the ELF format, but this PR tries to avoid moving existing code around, save for the old `Exception::CallStack.setup_crash_handler` as it remains the only common portion between MSVC and MinGW-w64. --- spec/std/exception/call_stack_spec.cr | 6 +- src/crystal/pe.cr | 110 +++++++++++++++++ src/crystal/system/win32/signal.cr | 44 +++++++ src/exception/call_stack.cr | 5 +- src/exception/call_stack/dwarf.cr | 4 + src/exception/call_stack/elf.cr | 86 +++++++------ src/exception/call_stack/libunwind.cr | 113 ++++++++++++++++-- src/exception/call_stack/stackwalk.cr | 61 +--------- .../x86_64-windows-msvc/c/libloaderapi.cr | 3 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 54 +++++++++ src/raise.cr | 2 +- 11 files changed, 379 insertions(+), 109 deletions(-) create mode 100644 src/crystal/pe.cr diff --git a/spec/std/exception/call_stack_spec.cr b/spec/std/exception/call_stack_spec.cr index c01fb0ff6b8a..7c6f5d746bdc 100644 --- a/spec/std/exception/call_stack_spec.cr +++ b/spec/std/exception/call_stack_spec.cr @@ -12,9 +12,9 @@ describe "Backtrace" do _, output, _ = compile_and_run_file(source_file) - # resolved file:line:column (no column for windows PDB because of poor - # support in general) - {% if flag?(:win32) %} + # resolved file:line:column (no column for MSVC PDB because of poor support + # by external tooling in general) + {% if flag?(:msvc) %} output.should match(/^#{Regex.escape(source_file)}:3 in 'callee1'/m) output.should match(/^#{Regex.escape(source_file)}:13 in 'callee3'/m) {% else %} diff --git a/src/crystal/pe.cr b/src/crystal/pe.cr new file mode 100644 index 000000000000..d1b19401ad19 --- /dev/null +++ b/src/crystal/pe.cr @@ -0,0 +1,110 @@ +module Crystal + # :nodoc: + # + # Portable Executable reader. + # + # Documentation: + # - + struct PE + class Error < Exception + end + + record SectionHeader, name : String, virtual_offset : UInt32, offset : UInt32, size : UInt32 + + record COFFSymbol, offset : UInt32, name : String + + # addresses in COFF debug info are relative to this image base; used by + # `Exception::CallStack.read_dwarf_sections` to calculate the real relocated + # addresses + getter original_image_base : UInt64 + + @section_headers : Slice(SectionHeader) + @string_table_base : UInt32 + + # mapping from zero-based section index to list of symbols sorted by + # offsets within that section + getter coff_symbols = Hash(Int32, Array(COFFSymbol)).new + + def self.open(path : String | ::Path, &) + File.open(path, "r") do |file| + yield new(file) + end + end + + def initialize(@io : IO::FileDescriptor) + dos_header = uninitialized LibC::IMAGE_DOS_HEADER + io.read_fully(pointerof(dos_header).to_slice(1).to_unsafe_bytes) + raise Error.new("Invalid DOS header") unless dos_header.e_magic == 0x5A4D # MZ + + io.seek(dos_header.e_lfanew) + nt_header = uninitialized LibC::IMAGE_NT_HEADERS + io.read_fully(pointerof(nt_header).to_slice(1).to_unsafe_bytes) + raise Error.new("Invalid PE header") unless nt_header.signature == 0x00004550 # PE\0\0 + + @original_image_base = nt_header.optionalHeader.imageBase + @string_table_base = nt_header.fileHeader.pointerToSymbolTable + nt_header.fileHeader.numberOfSymbols * sizeof(LibC::IMAGE_SYMBOL) + + section_count = nt_header.fileHeader.numberOfSections + nt_section_headers = Pointer(LibC::IMAGE_SECTION_HEADER).malloc(section_count).to_slice(section_count) + io.read_fully(nt_section_headers.to_unsafe_bytes) + + @section_headers = nt_section_headers.map do |nt_header| + if nt_header.name[0] === '/' + # section name is longer than 8 bytes; look up the COFF string table + name_buf = nt_header.name.to_slice + 1 + string_offset = String.new(name_buf.to_unsafe, name_buf.index(0) || name_buf.size).to_i + io.seek(@string_table_base + string_offset) + name = io.gets('\0', chomp: true).not_nil! + else + name = String.new(nt_header.name.to_unsafe, nt_header.name.index(0) || nt_header.name.size) + end + + SectionHeader.new(name: name, virtual_offset: nt_header.virtualAddress, offset: nt_header.pointerToRawData, size: nt_header.virtualSize) + end + + io.seek(nt_header.fileHeader.pointerToSymbolTable) + image_symbol_count = nt_header.fileHeader.numberOfSymbols + image_symbols = Pointer(LibC::IMAGE_SYMBOL).malloc(image_symbol_count).to_slice(image_symbol_count) + io.read_fully(image_symbols.to_unsafe_bytes) + + aux_count = 0 + image_symbols.each_with_index do |sym, i| + if aux_count == 0 + aux_count = sym.numberOfAuxSymbols.to_i + else + aux_count &-= 1 + end + + next unless aux_count == 0 + next unless sym.type.bits_set?(0x20) # COFF function + next unless sym.sectionNumber > 0 # one-based section index + next unless sym.storageClass.in?(LibC::IMAGE_SYM_CLASS_EXTERNAL, LibC::IMAGE_SYM_CLASS_STATIC) + + if sym.n.name.short == 0 + io.seek(@string_table_base + sym.n.name.long) + name = io.gets('\0', chomp: true).not_nil! + else + name = String.new(sym.n.shortName.to_slice).rstrip('\0') + end + + # `@coff_symbols` uses zero-based indices + section_coff_symbols = @coff_symbols.put_if_absent(sym.sectionNumber.to_i &- 1) { [] of COFFSymbol } + section_coff_symbols << COFFSymbol.new(sym.value, name) + end + + # add one sentinel symbol to ensure binary search on the offsets works + @coff_symbols.each_with_index do |(_, symbols), i| + symbols.sort_by!(&.offset) + symbols << COFFSymbol.new(@section_headers[i].size, "??") + end + end + + def read_section?(name : String, &) + if sh = @section_headers.find(&.name.== name) + @io.seek(sh.offset) do + yield sh, @io + end + end + end + end +end diff --git a/src/crystal/system/win32/signal.cr b/src/crystal/system/win32/signal.cr index d805ea4fd1ab..4cebe7cf9c6a 100644 --- a/src/crystal/system/win32/signal.cr +++ b/src/crystal/system/win32/signal.cr @@ -1,4 +1,5 @@ require "c/signal" +require "c/malloc" module Crystal::System::Signal def self.trap(signal, handler) : Nil @@ -16,4 +17,47 @@ module Crystal::System::Signal def self.ignore(signal) : Nil raise NotImplementedError.new("Crystal::System::Signal.ignore") end + + def self.setup_seh_handler + LibC.AddVectoredExceptionHandler(1, ->(exception_info) do + case exception_info.value.exceptionRecord.value.exceptionCode + when LibC::EXCEPTION_ACCESS_VIOLATION + addr = exception_info.value.exceptionRecord.value.exceptionInformation[1] + Crystal::System.print_error "Invalid memory access (C0000005) at address %p\n", Pointer(Void).new(addr) + {% if flag?(:gnu) %} + Exception::CallStack.print_backtrace + {% else %} + Exception::CallStack.print_backtrace(exception_info) + {% end %} + LibC._exit(1) + when LibC::EXCEPTION_STACK_OVERFLOW + LibC._resetstkoflw + Crystal::System.print_error "Stack overflow (e.g., infinite or very deep recursion)\n" + {% if flag?(:gnu) %} + Exception::CallStack.print_backtrace + {% else %} + Exception::CallStack.print_backtrace(exception_info) + {% end %} + LibC._exit(1) + else + LibC::EXCEPTION_CONTINUE_SEARCH + end + end) + + # ensure that even in the case of stack overflow there is enough reserved + # stack space for recovery (for other threads this is done in + # `Crystal::System::Thread.thread_proc`) + stack_size = Crystal::System::Fiber::RESERVED_STACK_SIZE + LibC.SetThreadStackGuarantee(pointerof(stack_size)) + + # this catches invalid argument checks inside the C runtime library + LibC._set_invalid_parameter_handler(->(expression, _function, _file, _line, _pReserved) do + message = expression ? String.from_utf16(expression)[0] : "(no message)" + Crystal::System.print_error "CRT invalid parameter handler invoked: %s\n", message + caller.each do |frame| + Crystal::System.print_error " from %s\n", frame + end + LibC._exit(1) + end) + end end diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index 44a281570c1c..506317d2580e 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -1,10 +1,7 @@ {% if flag?(:interpreted) %} require "./call_stack/interpreter" -{% elsif flag?(:win32) %} +{% elsif flag?(:win32) && !flag?(:gnu) %} require "./call_stack/stackwalk" - {% if flag?(:gnu) %} - require "./lib_unwind" - {% end %} {% elsif flag?(:wasm32) %} require "./call_stack/null" {% else %} diff --git a/src/exception/call_stack/dwarf.cr b/src/exception/call_stack/dwarf.cr index 96d99f03205a..253a72a38ebc 100644 --- a/src/exception/call_stack/dwarf.cr +++ b/src/exception/call_stack/dwarf.cr @@ -10,6 +10,10 @@ struct Exception::CallStack @@dwarf_line_numbers : Crystal::DWARF::LineNumbers? @@dwarf_function_names : Array(Tuple(LibC::SizeT, LibC::SizeT, String))? + {% if flag?(:win32) %} + @@coff_symbols : Hash(Int32, Array(Crystal::PE::COFFSymbol))? + {% end %} + # :nodoc: def self.load_debug_info : Nil return if ENV["CRYSTAL_LOAD_DEBUG_INFO"]? == "0" diff --git a/src/exception/call_stack/elf.cr b/src/exception/call_stack/elf.cr index efa54f41329c..51d565528577 100644 --- a/src/exception/call_stack/elf.cr +++ b/src/exception/call_stack/elf.cr @@ -1,65 +1,83 @@ -require "crystal/elf" -{% unless flag?(:wasm32) %} - require "c/link" +{% if flag?(:win32) %} + require "crystal/pe" +{% else %} + require "crystal/elf" + {% unless flag?(:wasm32) %} + require "c/link" + {% end %} {% end %} struct Exception::CallStack - private struct DlPhdrData - getter program : String - property base_address : LibC::Elf_Addr = 0 + {% unless flag?(:win32) %} + private struct DlPhdrData + getter program : String + property base_address : LibC::Elf_Addr = 0 - def initialize(@program : String) + def initialize(@program : String) + end end - end + {% end %} protected def self.load_debug_info_impl : Nil program = Process.executable_path return unless program && File::Info.readable? program - data = DlPhdrData.new(program) - - phdr_callback = LibC::DlPhdrCallback.new do |info, size, data| - # `dl_iterate_phdr` does not always visit the current program first; on - # Android the first object is `/system/bin/linker64`, the second is the - # full program path (not the empty string), so we check both here - name_c_str = info.value.name - if name_c_str && (name_c_str.value == 0 || LibC.strcmp(name_c_str, data.as(DlPhdrData*).value.program) == 0) - # The first entry is the header for the current program. - # Note that we avoid allocating here and just store the base address - # to be passed to self.read_dwarf_sections when dl_iterate_phdr returns. - # Calling self.read_dwarf_sections from this callback may lead to reallocations - # and deadlocks due to the internal lock held by dl_iterate_phdr (#10084). - data.as(DlPhdrData*).value.base_address = info.value.addr - 1 - else - 0 + + {% if flag?(:win32) %} + if LibC.GetModuleHandleExW(LibC::GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, nil, out hmodule) != 0 + self.read_dwarf_sections(program, hmodule.address) end - end + {% else %} + data = DlPhdrData.new(program) - LibC.dl_iterate_phdr(phdr_callback, pointerof(data)) - self.read_dwarf_sections(data.program, data.base_address) + phdr_callback = LibC::DlPhdrCallback.new do |info, size, data| + # `dl_iterate_phdr` does not always visit the current program first; on + # Android the first object is `/system/bin/linker64`, the second is the + # full program path (not the empty string), so we check both here + name_c_str = info.value.name + if name_c_str && (name_c_str.value == 0 || LibC.strcmp(name_c_str, data.as(DlPhdrData*).value.program) == 0) + # The first entry is the header for the current program. + # Note that we avoid allocating here and just store the base address + # to be passed to self.read_dwarf_sections when dl_iterate_phdr returns. + # Calling self.read_dwarf_sections from this callback may lead to reallocations + # and deadlocks due to the internal lock held by dl_iterate_phdr (#10084). + data.as(DlPhdrData*).value.base_address = info.value.addr + 1 + else + 0 + end + end + + LibC.dl_iterate_phdr(phdr_callback, pointerof(data)) + self.read_dwarf_sections(data.program, data.base_address) + {% end %} end protected def self.read_dwarf_sections(program, base_address = 0) - Crystal::ELF.open(program) do |elf| - line_strings = elf.read_section?(".debug_line_str") do |sh, io| + {{ flag?(:win32) ? Crystal::PE : Crystal::ELF }}.open(program) do |image| + {% if flag?(:win32) %} + base_address -= image.original_image_base + @@coff_symbols = image.coff_symbols + {% end %} + + line_strings = image.read_section?(".debug_line_str") do |sh, io| Crystal::DWARF::Strings.new(io, sh.offset, sh.size) end - strings = elf.read_section?(".debug_str") do |sh, io| + strings = image.read_section?(".debug_str") do |sh, io| Crystal::DWARF::Strings.new(io, sh.offset, sh.size) end - elf.read_section?(".debug_line") do |sh, io| + image.read_section?(".debug_line") do |sh, io| @@dwarf_line_numbers = Crystal::DWARF::LineNumbers.new(io, sh.size, base_address, strings, line_strings) end - elf.read_section?(".debug_info") do |sh, io| + image.read_section?(".debug_info") do |sh, io| names = [] of {LibC::SizeT, LibC::SizeT, String} while (offset = io.pos - sh.offset) < sh.size info = Crystal::DWARF::Info.new(io, offset) - elf.read_section?(".debug_abbrev") do |sh, io| + image.read_section?(".debug_abbrev") do |sh, io| info.read_abbreviations(io) end diff --git a/src/exception/call_stack/libunwind.cr b/src/exception/call_stack/libunwind.cr index 1542d52cc736..c0f75867aeba 100644 --- a/src/exception/call_stack/libunwind.cr +++ b/src/exception/call_stack/libunwind.cr @@ -1,9 +1,11 @@ -require "c/dlfcn" +{% unless flag?(:win32) %} + require "c/dlfcn" +{% end %} require "c/stdio" require "c/string" require "../lib_unwind" -{% if flag?(:darwin) || flag?(:bsd) || flag?(:linux) || flag?(:solaris) %} +{% if flag?(:darwin) || flag?(:bsd) || flag?(:linux) || flag?(:solaris) || flag?(:win32) %} require "./dwarf" {% else %} require "./null" @@ -33,7 +35,11 @@ struct Exception::CallStack {% end %} def self.setup_crash_handler - Crystal::System::Signal.setup_segfault_handler + {% if flag?(:win32) %} + Crystal::System::Signal.setup_seh_handler + {% else %} + Crystal::System::Signal.setup_segfault_handler + {% end %} end {% if flag?(:interpreted) %} @[Primitive(:interpreter_call_stack_unwind)] {% end %} @@ -167,9 +173,102 @@ struct Exception::CallStack end end - private def self.dladdr(ip, &) - if LibC.dladdr(ip, out info) != 0 - yield info.dli_fname, info.dli_sname, info.dli_saddr + {% if flag?(:win32) %} + def self.dladdr(ip, &) + if LibC.GetModuleHandleExW(LibC::GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT | LibC::GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, ip.as(LibC::LPWSTR), out hmodule) != 0 + symbol, address = internal_symbol(hmodule, ip) || external_symbol(hmodule, ip) || return + + utf16_file = uninitialized LibC::WCHAR[LibC::MAX_PATH] + len = LibC.GetModuleFileNameW(hmodule, utf16_file, utf16_file.size) + if 0 < len < utf16_file.size + utf8_file = uninitialized UInt8[sizeof(UInt8[LibC::MAX_PATH][3])] + file = utf8_file.to_unsafe + appender = file.appender + String.each_utf16_char(utf16_file.to_slice[0, len + 1]) do |ch| + ch.each_byte { |b| appender << b } + end + else + file = Pointer(UInt8).null + end + + yield file, symbol, address + end end - end + + private def self.internal_symbol(hmodule, ip) + if coff_symbols = @@coff_symbols + if LibC.GetModuleHandleExW(LibC::GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, nil, out this_hmodule) != 0 && this_hmodule == hmodule + section_base, section_index = lookup_section(hmodule, ip) || return + offset = ip - section_base + section_coff_symbols = coff_symbols[section_index]? || return + next_sym = section_coff_symbols.bsearch_index { |sym| offset < sym.offset } || return + sym = section_coff_symbols[next_sym - 1]? || return + + {sym.name.to_unsafe, section_base + sym.offset} + end + end + end + + private def self.external_symbol(hmodule, ip) + if dir = data_directory(hmodule, LibC::IMAGE_DIRECTORY_ENTRY_EXPORT) + exports = dir.to_unsafe.as(LibC::IMAGE_EXPORT_DIRECTORY*).value + + found_address = Pointer(Void).null + found_index = -1 + + func_address_offsets = (hmodule + exports.addressOfFunctions).as(LibC::DWORD*).to_slice(exports.numberOfFunctions) + func_address_offsets.each_with_index do |offset, i| + address = hmodule + offset + if found_address < address <= ip + found_address, found_index = address, i + end + end + + return unless found_address + + func_name_ordinals = (hmodule + exports.addressOfNameOrdinals).as(LibC::WORD*).to_slice(exports.numberOfNames) + if ordinal_index = func_name_ordinals.index(&.== found_index) + symbol = (hmodule + (hmodule + exports.addressOfNames).as(LibC::DWORD*)[ordinal_index]).as(UInt8*) + {symbol, found_address} + end + end + end + + private def self.lookup_section(hmodule, ip) + dos_header = hmodule.as(LibC::IMAGE_DOS_HEADER*) + return unless dos_header.value.e_magic == 0x5A4D # MZ + + nt_header = (hmodule + dos_header.value.e_lfanew).as(LibC::IMAGE_NT_HEADERS*) + return unless nt_header.value.signature == 0x00004550 # PE\0\0 + + section_headers = (nt_header + 1).as(LibC::IMAGE_SECTION_HEADER*).to_slice(nt_header.value.fileHeader.numberOfSections) + section_headers.each_with_index do |header, i| + base = hmodule + header.virtualAddress + if base <= ip < base + header.virtualSize + return base, i + end + end + end + + private def self.data_directory(hmodule, index) + dos_header = hmodule.as(LibC::IMAGE_DOS_HEADER*) + return unless dos_header.value.e_magic == 0x5A4D # MZ + + nt_header = (hmodule + dos_header.value.e_lfanew).as(LibC::IMAGE_NT_HEADERS*) + return unless nt_header.value.signature == 0x00004550 # PE\0\0 + return unless nt_header.value.optionalHeader.magic == {{ flag?(:bits64) ? 0x20b : 0x10b }} + return unless index.in?(0...{16, nt_header.value.optionalHeader.numberOfRvaAndSizes}.min) + + directory = nt_header.value.optionalHeader.dataDirectory.to_unsafe[index] + if directory.virtualAddress != 0 + Bytes.new(hmodule.as(UInt8*) + directory.virtualAddress, directory.size, read_only: true) + end + end + {% else %} + private def self.dladdr(ip, &) + if LibC.dladdr(ip, out info) != 0 + yield info.dli_fname, info.dli_sname, info.dli_saddr + end + end + {% end %} end diff --git a/src/exception/call_stack/stackwalk.cr b/src/exception/call_stack/stackwalk.cr index 6ac59fa6db48..d7e3da8e35f1 100644 --- a/src/exception/call_stack/stackwalk.cr +++ b/src/exception/call_stack/stackwalk.cr @@ -1,5 +1,4 @@ require "c/dbghelp" -require "c/malloc" # :nodoc: struct Exception::CallStack @@ -33,38 +32,7 @@ struct Exception::CallStack end def self.setup_crash_handler - LibC.AddVectoredExceptionHandler(1, ->(exception_info) do - case exception_info.value.exceptionRecord.value.exceptionCode - when LibC::EXCEPTION_ACCESS_VIOLATION - addr = exception_info.value.exceptionRecord.value.exceptionInformation[1] - Crystal::System.print_error "Invalid memory access (C0000005) at address %p\n", Pointer(Void).new(addr) - print_backtrace(exception_info) - LibC._exit(1) - when LibC::EXCEPTION_STACK_OVERFLOW - LibC._resetstkoflw - Crystal::System.print_error "Stack overflow (e.g., infinite or very deep recursion)\n" - print_backtrace(exception_info) - LibC._exit(1) - else - LibC::EXCEPTION_CONTINUE_SEARCH - end - end) - - # ensure that even in the case of stack overflow there is enough reserved - # stack space for recovery (for other threads this is done in - # `Crystal::System::Thread.thread_proc`) - stack_size = Crystal::System::Fiber::RESERVED_STACK_SIZE - LibC.SetThreadStackGuarantee(pointerof(stack_size)) - - # this catches invalid argument checks inside the C runtime library - LibC._set_invalid_parameter_handler(->(expression, _function, _file, _line, _pReserved) do - message = expression ? String.from_utf16(expression)[0] : "(no message)" - Crystal::System.print_error "CRT invalid parameter handler invoked: %s\n", message - caller.each do |frame| - Crystal::System.print_error " from %s\n", frame - end - LibC._exit(1) - end) + Crystal::System::Signal.setup_seh_handler end {% if flag?(:interpreted) %} @[Primitive(:interpreter_call_stack_unwind)] {% end %} @@ -168,33 +136,6 @@ struct Exception::CallStack end end - # TODO: needed only if `__crystal_raise` fails, check if this actually works - {% if flag?(:gnu) %} - def self.print_backtrace : Nil - backtrace_fn = ->(context : LibUnwind::Context, data : Void*) do - last_frame = data.as(RepeatedFrame*) - - ip = {% if flag?(:arm) %} - Pointer(Void).new(__crystal_unwind_get_ip(context)) - {% else %} - Pointer(Void).new(LibUnwind.get_ip(context)) - {% end %} - - if last_frame.value.ip == ip - last_frame.value.incr - else - print_frame(last_frame.value) unless last_frame.value.ip.address == 0 - last_frame.value = RepeatedFrame.new ip - end - LibUnwind::ReasonCode::NO_REASON - end - - rf = RepeatedFrame.new(Pointer(Void).null) - LibUnwind.backtrace(backtrace_fn, pointerof(rf).as(Void*)) - print_frame(rf) - end - {% end %} - private def self.print_frame(repeated_frame) Crystal::System.print_error "[%p] ", repeated_frame.ip print_frame_location(repeated_frame) diff --git a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr index 37a95f3fa089..5612233553d9 100644 --- a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr @@ -9,6 +9,9 @@ lib LibC fun LoadLibraryExW(lpLibFileName : LPWSTR, hFile : HANDLE, dwFlags : DWORD) : HMODULE fun FreeLibrary(hLibModule : HMODULE) : BOOL + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT = 0x00000002 + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS = 0x00000004 + fun GetModuleHandleExW(dwFlags : DWORD, lpModuleName : LPWSTR, phModule : HMODULE*) : BOOL fun GetProcAddress(hModule : HMODULE, lpProcName : LPSTR) : FARPROC diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index 1db4b2def700..e9aecc01e033 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -392,11 +392,65 @@ lib LibC optionalHeader : IMAGE_OPTIONAL_HEADER64 end + IMAGE_DIRECTORY_ENTRY_EXPORT = 0 + IMAGE_DIRECTORY_ENTRY_IMPORT = 1 + IMAGE_DIRECTORY_ENTRY_IAT = 12 + + struct IMAGE_SECTION_HEADER + name : BYTE[8] + virtualSize : DWORD + virtualAddress : DWORD + sizeOfRawData : DWORD + pointerToRawData : DWORD + pointerToRelocations : DWORD + pointerToLinenumbers : DWORD + numberOfRelocations : WORD + numberOfLinenumbers : WORD + characteristics : DWORD + end + + struct IMAGE_EXPORT_DIRECTORY + characteristics : DWORD + timeDateStamp : DWORD + majorVersion : WORD + minorVersion : WORD + name : DWORD + base : DWORD + numberOfFunctions : DWORD + numberOfNames : DWORD + addressOfFunctions : DWORD + addressOfNames : DWORD + addressOfNameOrdinals : DWORD + end + struct IMAGE_IMPORT_BY_NAME hint : WORD name : CHAR[1] end + struct IMAGE_SYMBOL_n_name + short : DWORD + long : DWORD + end + + union IMAGE_SYMBOL_n + shortName : BYTE[8] + name : IMAGE_SYMBOL_n_name + end + + IMAGE_SYM_CLASS_EXTERNAL = 2 + IMAGE_SYM_CLASS_STATIC = 3 + + @[Packed] + struct IMAGE_SYMBOL + n : IMAGE_SYMBOL_n + value : DWORD + sectionNumber : Short + type : WORD + storageClass : BYTE + numberOfAuxSymbols : BYTE + end + union IMAGE_THUNK_DATA64_u1 forwarderString : ULongLong function : ULongLong diff --git a/src/raise.cr b/src/raise.cr index a8e06a3c3930..0c9563495a94 100644 --- a/src/raise.cr +++ b/src/raise.cr @@ -181,7 +181,7 @@ end 0u64 end {% else %} - {% mingw = flag?(:windows) && flag?(:gnu) %} + {% mingw = flag?(:win32) && flag?(:gnu) %} fun {{ mingw ? "__crystal_personality_imp".id : "__crystal_personality".id }}( version : Int32, actions : LibUnwind::Action, exception_class : UInt64, exception_object : LibUnwind::Exception*, context : Void*, ) : LibUnwind::ReasonCode From 24fc1a91ac5c9057a1de24090a40d7badf9f81c8 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 24 Oct 2024 18:30:54 +0800 Subject: [PATCH 177/193] Support OpenSSL on MSYS2 (#15111) --- src/openssl/lib_crypto.cr | 12 +++++++----- src/openssl/lib_ssl.cr | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/openssl/lib_crypto.cr b/src/openssl/lib_crypto.cr index 8d450b28ff17..fecc69ad44fc 100644 --- a/src/openssl/lib_crypto.cr +++ b/src/openssl/lib_crypto.cr @@ -1,6 +1,6 @@ {% begin %} lib LibCrypto - {% if flag?(:win32) %} + {% if flag?(:msvc) %} {% from_libressl = false %} {% ssl_version = nil %} {% for dir in Crystal::LIBRARY_PATH.split(Crystal::System::Process::HOST_PATH_DELIMITER) %} @@ -13,10 +13,12 @@ {% end %} {% ssl_version ||= "0.0.0" %} {% else %} - {% from_libressl = (`hash pkg-config 2> /dev/null || printf %s false` != "false") && - (`test -f $(pkg-config --silence-errors --variable=includedir libcrypto)/openssl/opensslv.h || printf %s false` != "false") && - (`printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libcrypto || true) -E -`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} - {% ssl_version = `hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libcrypto || printf %s 0.0.0`.split.last.gsub(/[^0-9.]/, "") %} + # these have to be wrapped in `sh -c` since for MinGW-w64 the compiler + # passes the command string to `LibC.CreateProcessW` + {% from_libressl = (`sh -c 'hash pkg-config 2> /dev/null || printf %s false'` != "false") && + (`sh -c 'test -f $(pkg-config --silence-errors --variable=includedir libcrypto)/openssl/opensslv.h || printf %s false'` != "false") && + (`sh -c 'printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libcrypto || true) -E -'`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} + {% ssl_version = `sh -c 'hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libcrypto || printf %s 0.0.0'`.split.last.gsub(/[^0-9.]/, "") %} {% end %} {% if from_libressl %} diff --git a/src/openssl/lib_ssl.cr b/src/openssl/lib_ssl.cr index 6adb3f172a3b..4e7e2def549c 100644 --- a/src/openssl/lib_ssl.cr +++ b/src/openssl/lib_ssl.cr @@ -6,7 +6,7 @@ require "./lib_crypto" {% begin %} lib LibSSL - {% if flag?(:win32) %} + {% if flag?(:msvc) %} {% from_libressl = false %} {% ssl_version = nil %} {% for dir in Crystal::LIBRARY_PATH.split(Crystal::System::Process::HOST_PATH_DELIMITER) %} @@ -19,10 +19,12 @@ require "./lib_crypto" {% end %} {% ssl_version ||= "0.0.0" %} {% else %} - {% from_libressl = (`hash pkg-config 2> /dev/null || printf %s false` != "false") && - (`test -f $(pkg-config --silence-errors --variable=includedir libssl)/openssl/opensslv.h || printf %s false` != "false") && - (`printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libssl || true) -E -`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} - {% ssl_version = `hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libssl || printf %s 0.0.0`.split.last.gsub(/[^0-9.]/, "") %} + # these have to be wrapped in `sh -c` since for MinGW-w64 the compiler + # passes the command string to `LibC.CreateProcessW` + {% from_libressl = (`sh -c 'hash pkg-config 2> /dev/null || printf %s false'` != "false") && + (`sh -c 'test -f $(pkg-config --silence-errors --variable=includedir libssl)/openssl/opensslv.h || printf %s false'` != "false") && + (`sh -c 'printf "#include \nLIBRESSL_VERSION_NUMBER" | ${CC:-cc} $(pkg-config --cflags --silence-errors libssl || true) -E -'`.chomp.split('\n').last != "LIBRESSL_VERSION_NUMBER") %} + {% ssl_version = `sh -c 'hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libssl || printf %s 0.0.0'`.split.last.gsub(/[^0-9.]/, "") %} {% end %} {% if from_libressl %} From 4016f39d2ed781031613ad027bb13a4529984bfa Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 15:34:02 +0800 Subject: [PATCH 178/193] Use per-thread libxml2 global state on all platforms (#15121) libxml2's build files enable threads by default, so I'd be surprised if any platform still disables them (embedded systems don't count yet). --- src/xml.cr | 14 ++------------ src/xml/libxml2.cr | 10 ++-------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/xml.cr b/src/xml.cr index e0529be130f3..a9c9eab5d64e 100644 --- a/src/xml.cr +++ b/src/xml.cr @@ -107,12 +107,7 @@ module XML end protected def self.with_indent_tree_output(indent : Bool, &) - ptr = {% if flag?(:win32) %} - LibXML.__xmlIndentTreeOutput - {% else %} - pointerof(LibXML.xmlIndentTreeOutput) - {% end %} - + ptr = LibXML.__xmlIndentTreeOutput old, ptr.value = ptr.value, indent ? 1 : 0 begin yield @@ -122,12 +117,7 @@ module XML end protected def self.with_tree_indent_string(string : String, &) - ptr = {% if flag?(:win32) %} - LibXML.__xmlTreeIndentString - {% else %} - pointerof(LibXML.xmlTreeIndentString) - {% end %} - + ptr = LibXML.__xmlTreeIndentString old, ptr.value = ptr.value, string.to_unsafe begin yield diff --git a/src/xml/libxml2.cr b/src/xml/libxml2.cr index e1c2b8d12372..fbfb0702faef 100644 --- a/src/xml/libxml2.cr +++ b/src/xml/libxml2.cr @@ -13,14 +13,8 @@ lib LibXML fun xmlInitParser - # TODO: check if other platforms also support per-thread globals - {% if flag?(:win32) %} - fun __xmlIndentTreeOutput : Int* - fun __xmlTreeIndentString : UInt8** - {% else %} - $xmlIndentTreeOutput : Int - $xmlTreeIndentString : UInt8* - {% end %} + fun __xmlIndentTreeOutput : Int* + fun __xmlTreeIndentString : UInt8** alias Dtd = Void* alias Dict = Void* From 0236a68b93e9695d7b70ae61813c26ca9ed90f05 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 15:34:20 +0800 Subject: [PATCH 179/193] Support "long format" DLL import libraries (#15119) `Crystal::System::LibraryArchive.imported_dlls` is used by the interpreter to obtain all dependent DLLs of a given import library. Currently, the method only supports libraries using the [short format](https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#import-library-format), emitted by MSVC's linker. This PR implements the long format used by MinGW-w64 when passing the `-Wl,--out-implib` flag to `cc`. Specs will be added when `Crystal::Loader` supports MinGW-w64. In the mean time, if you have MSYS2, you could try this on the UCRT64 import libraries: ```crystal require "crystal/system/win32/library_archive" Crystal::System::LibraryArchive.imported_dlls("C:/msys64/ucrt64/lib/libpcre2-8.dll.a") # => Set{"libpcre2-8-0.dll"} Crystal::System::LibraryArchive.imported_dlls("C:/msys64/ucrt64/lib/libucrt.a") # => Set{"api-ms-win-crt-utility-l1-1-0.dll", "api-ms-win-crt-time-l1-1-0.dll", "api-ms-win-crt-string-l1-1-0.dll", "api-ms-win-crt-stdio-l1-1-0.dll", "api-ms-win-crt-runtime-l1-1-0.dll", "api-ms-win-crt-process-l1-1-0.dll", "api-ms-win-crt-private-l1-1-0.dll", "api-ms-win-crt-multibyte-l1-1-0.dll", "api-ms-win-crt-math-l1-1-0.dll", "api-ms-win-crt-locale-l1-1-0.dll", "api-ms-win-crt-heap-l1-1-0.dll", "api-ms-win-crt-filesystem-l1-1-0.dll", "api-ms-win-crt-environment-l1-1-0.dll", "api-ms-win-crt-convert-l1-1-0.dll", "api-ms-win-crt-conio-l1-1-0.dll"} ``` --- src/crystal/system/win32/library_archive.cr | 74 +++++++++++++++++---- src/lib_c/x86_64-windows-msvc/c/winnt.cr | 2 + 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/crystal/system/win32/library_archive.cr b/src/crystal/system/win32/library_archive.cr index 775677938bac..24c50f3405fa 100644 --- a/src/crystal/system/win32/library_archive.cr +++ b/src/crystal/system/win32/library_archive.cr @@ -17,6 +17,10 @@ module Crystal::System::LibraryArchive private struct COFFReader getter dlls = Set(String).new + # MSVC-style import libraries include the `__NULL_IMPORT_DESCRIPTOR` symbol, + # MinGW-style ones do not + getter? msvc = false + def initialize(@ar : ::File) end @@ -39,6 +43,7 @@ module Crystal::System::LibraryArchive if first first = false return unless filename == "/" + handle_first_member(io) elsif !filename.in?("/", "//") handle_standard_member(io) end @@ -62,26 +67,69 @@ module Crystal::System::LibraryArchive @ar.seek(new_pos) end + private def handle_first_member(io) + symbol_count = io.read_bytes(UInt32, IO::ByteFormat::BigEndian) + + # 4-byte offset per symbol + io.skip(symbol_count * 4) + + symbol_count.times do + symbol = io.gets('\0', chomp: true) + if symbol == "__NULL_IMPORT_DESCRIPTOR" + @msvc = true + break + end + end + end + private def handle_standard_member(io) - sig1 = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - return unless sig1 == 0x0000 # IMAGE_FILE_MACHINE_UNKNOWN + machine = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) + section_count = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - sig2 = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - return unless sig2 == 0xFFFF + if machine == 0x0000 && section_count == 0xFFFF + # short import library + version = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) + return unless version == 0 # 1 and 2 are used by object files (ANON_OBJECT_HEADER) - version = io.read_bytes(UInt16, IO::ByteFormat::LittleEndian) - return unless version == 0 # 1 and 2 are used by object files (ANON_OBJECT_HEADER) + # machine(2) + time(4) + size(4) + ordinal/hint(2) + flags(2) + io.skip(14) - # machine(2) + time(4) + size(4) + ordinal/hint(2) + flags(2) - io.skip(14) + # TODO: is there a way to do this without constructing a temporary string, + # but with the optimizations present in `IO#gets`? + return unless io.gets('\0') # symbol name - # TODO: is there a way to do this without constructing a temporary string, - # but with the optimizations present in `IO#gets`? - return unless io.gets('\0') # symbol name + if dll_name = io.gets('\0', chomp: true) + @dlls << dll_name if valid_dll?(dll_name) + end + else + # long import library, code based on GNU binutils `dlltool -I`: + # https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=binutils/dlltool.c;hb=967dc35c78adb85ee1e2e596047d9dc69107a9db#l3231 + + # timeDateStamp(4) + pointerToSymbolTable(4) + numberOfSymbols(4) + sizeOfOptionalHeader(2) + characteristics(2) + io.skip(16) + + section_count.times do |i| + section_header = uninitialized LibC::IMAGE_SECTION_HEADER + return unless io.read_fully?(pointerof(section_header).to_slice(1).to_unsafe_bytes) + + name = String.new(section_header.name.to_unsafe, section_header.name.index(0) || section_header.name.size) + next unless name == (msvc? ? ".idata$6" : ".idata$7") + + if msvc? ? section_header.characteristics.bits_set?(LibC::IMAGE_SCN_CNT_INITIALIZED_DATA) : section_header.pointerToRelocations == 0 + bytes_read = sizeof(LibC::IMAGE_FILE_HEADER) + sizeof(LibC::IMAGE_SECTION_HEADER) * (i + 1) + io.skip(section_header.pointerToRawData - bytes_read) + if dll_name = io.gets('\0', chomp: true, limit: section_header.sizeOfRawData) + @dlls << dll_name if valid_dll?(dll_name) + end + end - if dll_name = io.gets('\0', chomp: true) - @dlls << dll_name + return + end end end + + private def valid_dll?(name) + name.size >= 5 && name[-4..].compare(".dll", case_insensitive: true) == 0 + end end end diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index e9aecc01e033..99c8f24ac9e1 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -396,6 +396,8 @@ lib LibC IMAGE_DIRECTORY_ENTRY_IMPORT = 1 IMAGE_DIRECTORY_ENTRY_IAT = 12 + IMAGE_SCN_CNT_INITIALIZED_DATA = 0x00000040 + struct IMAGE_SECTION_HEADER name : BYTE[8] virtualSize : DWORD From 8a96e33ae303bcc819cda90fe283e2643165ee64 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 25 Oct 2024 09:35:49 +0200 Subject: [PATCH 180/193] OpenBSD: fix integration and broken specs (#15118) Fixes compatibility with OpenBSD 7.4 that enforced indirect branch tracking by default but we're not compatible yet (see #13665), so we must disable it. With this patch I can run the std specs, except for the SSL specs because of openssl/libressl mess, as well as the compiler specs, except for the interpreter specs that regularly crash with SIGABRT (may help to debug issues in the interpreter). Note: the segfault handler is broken on OpenBSD and processes eventually crash with SIGILL after receiving SIGSEGV. --- bin/crystal | 2 +- spec/compiler/loader/unix_spec.cr | 6 +- spec/std/exception/call_stack_spec.cr | 17 +++-- spec/std/io/io_spec.cr | 8 +-- spec/std/kernel_spec.cr | 89 ++++++++++++++------------- spec/std/socket/addrinfo_spec.cr | 4 +- spec/std/socket/tcp_server_spec.cr | 6 +- spec/std/socket/tcp_socket_spec.cr | 6 +- spec/std/socket/udp_socket_spec.cr | 3 + src/compiler/crystal/compiler.cr | 13 ++++ src/lib_c/x86_64-openbsd/c/netdb.cr | 1 + 11 files changed, 93 insertions(+), 62 deletions(-) diff --git a/bin/crystal b/bin/crystal index e8abdff30ee8..a1fddf1c58b4 100755 --- a/bin/crystal +++ b/bin/crystal @@ -196,7 +196,7 @@ esac if [ -x "$CRYSTAL_DIR/${CRYSTAL_BIN}" ]; then __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/${CRYSTAL_BIN}" exec "$CRYSTAL_DIR/${CRYSTAL_BIN}" "$@" -elif !($PARENT_CRYSTAL_EXISTS); then +elif (! $PARENT_CRYSTAL_EXISTS); then __error_msg 'You need to have a crystal executable in your path! or set CRYSTAL env variable' exit 1 else diff --git a/spec/compiler/loader/unix_spec.cr b/spec/compiler/loader/unix_spec.cr index 42a63b88e860..6adcb040148b 100644 --- a/spec/compiler/loader/unix_spec.cr +++ b/spec/compiler/loader/unix_spec.cr @@ -40,7 +40,11 @@ describe Crystal::Loader do exc = expect_raises(Crystal::Loader::LoadError, /no such file|not found|cannot open/i) do Crystal::Loader.parse(["-l", "foo/bar.o"], search_paths: [] of String) end - exc.message.should contain File.join(Dir.current, "foo", "bar.o") + {% if flag?(:openbsd) %} + exc.message.should contain "foo/bar.o" + {% else %} + exc.message.should contain File.join(Dir.current, "foo", "bar.o") + {% end %} end end diff --git a/spec/std/exception/call_stack_spec.cr b/spec/std/exception/call_stack_spec.cr index 7c6f5d746bdc..6df0741d2a7b 100644 --- a/spec/std/exception/call_stack_spec.cr +++ b/spec/std/exception/call_stack_spec.cr @@ -55,14 +55,19 @@ describe "Backtrace" do error.to_s.should contain("IndexError") end - it "prints crash backtrace to stderr", tags: %w[slow] do - sample = datapath("crash_backtrace_sample") + {% if flag?(:openbsd) %} + # FIXME: the segfault handler doesn't work on OpenBSD + pending "prints crash backtrace to stderr" + {% else %} + it "prints crash backtrace to stderr", tags: %w[slow] do + sample = datapath("crash_backtrace_sample") - _, output, error = compile_and_run_file(sample) + _, output, error = compile_and_run_file(sample) - output.to_s.should be_empty - error.to_s.should contain("Invalid memory access") - end + output.to_s.should be_empty + error.to_s.should contain("Invalid memory access") + end + {% end %} # Do not test this on platforms that cannot remove the current working # directory of the process: diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 620a1d034d9f..1904940f4883 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -425,9 +425,9 @@ describe IO do str.read_fully?(slice).should be_nil end - # pipe(2) returns bidirectional file descriptors on FreeBSD and Solaris, + # pipe(2) returns bidirectional file descriptors on some platforms, # gate this test behind the platform flag. - {% unless flag?(:freebsd) || flag?(:solaris) %} + {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) %} it "raises if trying to read to an IO not opened for reading" do IO.pipe do |r, w| expect_raises(IO::Error, "File not open for reading") do @@ -574,9 +574,9 @@ describe IO do io.read_byte.should be_nil end - # pipe(2) returns bidirectional file descriptors on FreeBSD and Solaris, + # pipe(2) returns bidirectional file descriptors on some platforms, # gate this test behind the platform flag. - {% unless flag?(:freebsd) || flag?(:solaris) %} + {% unless flag?(:freebsd) || flag?(:solaris) || flag?(:openbsd) %} it "raises if trying to write to an IO not opened for writing" do IO.pipe do |r, w| # unless sync is used the flush on close triggers the exception again diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index f8e4ff1e8ae2..0a682af8381b 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -251,55 +251,60 @@ describe "at_exit" do end end -describe "hardware exception" do - it "reports invalid memory access", tags: %w[slow] do - status, _, error = compile_and_run_source <<-'CRYSTAL' - puts Pointer(Int64).null.value - CRYSTAL - - status.success?.should be_false - error.should contain("Invalid memory access") - error.should_not contain("Stack overflow") - end - - {% if flag?(:netbsd) %} - # FIXME: on netbsd the process crashes with SIGILL after receiving SIGSEGV - pending "detects stack overflow on the main stack" - pending "detects stack overflow on a fiber stack" - {% else %} - it "detects stack overflow on the main stack", tags: %w[slow] do - # This spec can take some time under FreeBSD where - # the default stack size is 0.5G. Setting a - # smaller stack size with `ulimit -s 8192` - # will address this. +{% if flag?(:openbsd) %} + # FIXME: the segfault handler doesn't work on OpenBSD + pending "hardware exception" +{% else %} + describe "hardware exception" do + it "reports invalid memory access", tags: %w[slow] do status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) - foo - end - foo + puts Pointer(Int64).null.value CRYSTAL status.success?.should be_false - error.should contain("Stack overflow") + error.should contain("Invalid memory access") + error.should_not contain("Stack overflow") end - it "detects stack overflow on a fiber stack", tags: %w[slow] do - status, _, error = compile_and_run_source <<-'CRYSTAL' - def foo - y = StaticArray(Int8, 512).new(0) + {% if flag?(:netbsd) %} + # FIXME: on netbsd the process crashes with SIGILL after receiving SIGSEGV + pending "detects stack overflow on the main stack" + pending "detects stack overflow on a fiber stack" + {% else %} + it "detects stack overflow on the main stack", tags: %w[slow] do + # This spec can take some time under FreeBSD where + # the default stack size is 0.5G. Setting a + # smaller stack size with `ulimit -s 8192` + # will address this. + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end foo - end + CRYSTAL - spawn do - foo - end + status.success?.should be_false + error.should contain("Stack overflow") + end - sleep 60.seconds - CRYSTAL + it "detects stack overflow on a fiber stack", tags: %w[slow] do + status, _, error = compile_and_run_source <<-'CRYSTAL' + def foo + y = StaticArray(Int8, 512).new(0) + foo + end - status.success?.should be_false - error.should contain("Stack overflow") - end - {% end %} -end + spawn do + foo + end + + sleep 60.seconds + CRYSTAL + + status.success?.should be_false + error.should contain("Stack overflow") + end + {% end %} + end +{% end %} diff --git a/spec/std/socket/addrinfo_spec.cr b/spec/std/socket/addrinfo_spec.cr index 109eb383562b..b1d6b459623d 100644 --- a/spec/std/socket/addrinfo_spec.cr +++ b/spec/std/socket/addrinfo_spec.cr @@ -24,8 +24,8 @@ describe Socket::Addrinfo, tags: "network" do end it "raises helpful message on getaddrinfo failure" do - expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname failed: ") do - Socket::Addrinfo.resolve("badhostname", 80, type: Socket::Type::DGRAM) + expect_raises(Socket::Addrinfo::Error, "Hostname lookup for badhostname.unknown failed: ") do + Socket::Addrinfo.resolve("badhostname.unknown", 80, type: Socket::Type::DGRAM) end end diff --git a/spec/std/socket/tcp_server_spec.cr b/spec/std/socket/tcp_server_spec.cr index ee3c861956b8..a7d85b8edeff 100644 --- a/spec/std/socket/tcp_server_spec.cr +++ b/spec/std/socket/tcp_server_spec.cr @@ -43,7 +43,7 @@ describe TCPServer, tags: "network" do end error.os_error.should eq({% if flag?(:win32) %} WinError::WSATYPE_NOT_FOUND - {% elsif flag?(:linux) && !flag?(:android) %} + {% elsif (flag?(:linux) && !flag?(:android)) || flag?(:openbsd) %} Errno.new(LibC::EAI_SERVICE) {% else %} Errno.new(LibC::EAI_NONAME) @@ -96,7 +96,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -110,7 +110,7 @@ describe TCPServer, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error diff --git a/spec/std/socket/tcp_socket_spec.cr b/spec/std/socket/tcp_socket_spec.cr index 5ec3467362e0..0b3a381372bf 100644 --- a/spec/std/socket/tcp_socket_spec.cr +++ b/spec/std/socket/tcp_socket_spec.cr @@ -47,7 +47,7 @@ describe TCPSocket, tags: "network" do end error.os_error.should eq({% if flag?(:win32) %} WinError::WSATYPE_NOT_FOUND - {% elsif flag?(:linux) && !flag?(:android) %} + {% elsif (flag?(:linux) && !flag?(:android)) || flag?(:openbsd) %} Errno.new(LibC::EAI_SERVICE) {% else %} Errno.new(LibC::EAI_NONAME) @@ -79,7 +79,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error @@ -93,7 +93,7 @@ describe TCPSocket, tags: "network" do # FIXME: Resolve special handling for win32. The error code handling should be identical. {% if flag?(:win32) %} [WinError::WSAHOST_NOT_FOUND, WinError::WSATRY_AGAIN].should contain err.os_error - {% elsif flag?(:android) || flag?(:netbsd) %} + {% elsif flag?(:android) || flag?(:netbsd) || flag?(:openbsd) %} err.os_error.should eq(Errno.new(LibC::EAI_NODATA)) {% else %} [Errno.new(LibC::EAI_NONAME), Errno.new(LibC::EAI_AGAIN)].should contain err.os_error diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 9b624110fad9..dc66d8038036 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -88,6 +88,9 @@ describe UDPSocket, tags: "network" do elsif {{ flag?(:netbsd) }} && family == Socket::Family::INET6 # FIXME: fails with "setsockopt: EADDRNOTAVAIL" pending "joins and transmits to multicast groups" + elsif {{ flag?(:openbsd) }} + # FIXME: fails with "setsockopt: EINVAL (ipv4) or EADDRNOTAVAIL (ipv6)" + pending "joins and transmits to multicast groups" else it "joins and transmits to multicast groups" do udp = UDPSocket.new(family) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 6c7664bacc25..83509bf88392 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -500,6 +500,19 @@ module Crystal else link_flags = @link_flags || "" link_flags += " -rdynamic" + + if program.has_flag?("freebsd") || program.has_flag?("openbsd") + # pkgs are installed to usr/local/lib but it's not in LIBRARY_PATH by + # default; we declare it to ease linking on these platforms: + link_flags += " -L/usr/local/lib" + end + + if program.has_flag?("openbsd") + # OpenBSD requires Indirect Branch Tracking by default, but we're not + # compatible (yet), so we disable it for now: + link_flags += " -Wl,-znobtcfi" + end + {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} end end diff --git a/src/lib_c/x86_64-openbsd/c/netdb.cr b/src/lib_c/x86_64-openbsd/c/netdb.cr index be3c5f06ab2d..6dd1e6c8513f 100644 --- a/src/lib_c/x86_64-openbsd/c/netdb.cr +++ b/src/lib_c/x86_64-openbsd/c/netdb.cr @@ -13,6 +13,7 @@ lib LibC EAI_FAIL = -4 EAI_FAMILY = -6 EAI_MEMORY = -10 + EAI_NODATA = -5 EAI_NONAME = -2 EAI_SERVICE = -8 EAI_SOCKTYPE = -7 From 0987812fad23b4c28dc6101a3667c3285a238bf1 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 23:50:13 +0800 Subject: [PATCH 181/193] Support `.exe` file extension in `Makefile` on MSYS2 (#15123) `.build/crystal` already adds the `.exe` file extension in `Makefile` when run on MSYS2. This PR does the same to the spec executables so that running `make std_spec` twice will only build the executable once. --- Makefile | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index b39c089bef99..37c42890c9f4 100644 --- a/Makefile +++ b/Makefile @@ -68,10 +68,10 @@ CXXFLAGS += $(if $(debug),-g -O0) # MSYS2 support (native Windows should use `Makefile.win` instead) ifeq ($(OS),Windows_NT) - CRYSTAL_BIN := crystal.exe + EXE := .exe WINDOWS := 1 else - CRYSTAL_BIN := crystal + EXE := WINDOWS := endif @@ -112,28 +112,28 @@ test: spec ## Run tests spec: std_spec primitives_spec compiler_spec .PHONY: std_spec -std_spec: $(O)/std_spec ## Run standard library specs - $(O)/std_spec $(SPEC_FLAGS) +std_spec: $(O)/std_spec$(EXE) ## Run standard library specs + $(O)/std_spec$(EXE) $(SPEC_FLAGS) .PHONY: compiler_spec -compiler_spec: $(O)/compiler_spec ## Run compiler specs - $(O)/compiler_spec $(SPEC_FLAGS) +compiler_spec: $(O)/compiler_spec$(EXE) ## Run compiler specs + $(O)/compiler_spec$(EXE) $(SPEC_FLAGS) .PHONY: primitives_spec -primitives_spec: $(O)/primitives_spec ## Run primitives specs - $(O)/primitives_spec $(SPEC_FLAGS) +primitives_spec: $(O)/primitives_spec$(EXE) ## Run primitives specs + $(O)/primitives_spec$(EXE) $(SPEC_FLAGS) .PHONY: interpreter_spec -interpreter_spec: $(O)/interpreter_spec ## Run interpreter specs - $(O)/interpreter_spec $(SPEC_FLAGS) +interpreter_spec: $(O)/interpreter_spec$(EXE) ## Run interpreter specs + $(O)/interpreter_spec$(EXE) $(SPEC_FLAGS) .PHONY: smoke_test smoke_test: ## Build specs as a smoke test -smoke_test: $(O)/std_spec $(O)/compiler_spec $(O)/$(CRYSTAL_BIN) +smoke_test: $(O)/std_spec$(EXE) $(O)/compiler_spec$(EXE) $(O)/$(CRYSTAL)$(EXE) .PHONY: all_spec -all_spec: $(O)/all_spec ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) - $(O)/all_spec $(SPEC_FLAGS) +all_spec: $(O)/all_spec$(EXE) ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) + $(O)/all_spec$(EXE) $(SPEC_FLAGS) .PHONY: samples samples: ## Build example programs @@ -146,7 +146,7 @@ docs: ## Generate standard library documentation cp -av doc/ docs/ .PHONY: crystal -crystal: $(O)/$(CRYSTAL_BIN) ## Build the compiler +crystal: $(O)/$(CRYSTAL)$(EXE) ## Build the compiler .PHONY: deps llvm_ext deps: $(DEPS) ## Build dependencies @@ -161,9 +161,9 @@ generate_data: ## Run generator scripts for Unicode, SSL config, ... $(MAKE) -B -f scripts/generate_data.mk .PHONY: install -install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR +install: $(O)/$(CRYSTAL)$(EXE) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(BINDIR)/" - $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" + $(INSTALL) -m 0755 "$(O)/$(CRYSTAL)$(EXE)" "$(BINDIR)/$(CRYSTAL)$(EXE)" $(INSTALL) -d -m 0755 $(DATADIR) cp -av src "$(DATADIR)/src" @@ -183,14 +183,14 @@ install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR ifeq ($(WINDOWS),1) .PHONY: install_dlls -install_dlls: $(O)/$(CRYSTAL_BIN) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) +install_dlls: $(O)/$(CRYSTAL)$(EXE) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) $(INSTALL) -d -m 0755 "$(BINDIR)/" - @ldd $(O)/$(CRYSTAL_BIN) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" + @ldd $(O)/$(CRYSTAL)$(EXE) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" endif .PHONY: uninstall uninstall: ## Uninstall the compiler from DESTDIR - rm -f "$(BINDIR)/$(CRYSTAL_BIN)" + rm -f "$(BINDIR)/$(CRYSTAL)$(EXE)" rm -rf "$(DATADIR)/src" @@ -212,31 +212,31 @@ uninstall_docs: ## Uninstall docs from DESTDIR rm -rf "$(DATADIR)/docs" rm -rf "$(DATADIR)/examples" -$(O)/all_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/all_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(call check_llvm_config) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/all_spec.cr -$(O)/std_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/std_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(call check_llvm_config) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/std_spec.cr -$(O)/compiler_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/compiler_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(call check_llvm_config) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr --release -$(O)/primitives_spec: $(O)/$(CRYSTAL_BIN) $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/primitives_spec$(EXE): $(O)/$(CRYSTAL)$(EXE) $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/primitives_spec.cr -$(O)/interpreter_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/interpreter_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) $(eval interpreter=1) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler/interpreter_spec.cr -$(O)/$(CRYSTAL_BIN): $(DEPS) $(SOURCES) +$(O)/$(CRYSTAL)$(EXE): $(DEPS) $(SOURCES) $(call check_llvm_config) @mkdir -p $(O) @# NOTE: USE_PCRE1 is only used for testing compatibility with legacy environments that don't provide libpcre2. From adeda557febce9abdce0b2ae48dcac9e3a6953f0 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 23:50:28 +0800 Subject: [PATCH 182/193] Allow skipping compiler tool specs that require Git (#15125) Under rare circumstances like a minimal MSYS2 setup (e.g. the CI in #15124), Git might not be available at all when running the compiler specs, but this is not a hard failure for `crystal init` and `crystal docs`. This PR marks the relevant specs as pending if Git cannot be found. --- .../crystal/tools/doc/project_info_spec.cr | 2 ++ spec/compiler/crystal/tools/init_spec.cr | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/spec/compiler/crystal/tools/doc/project_info_spec.cr b/spec/compiler/crystal/tools/doc/project_info_spec.cr index 61bf20c2da67..c92ee9d12f9d 100644 --- a/spec/compiler/crystal/tools/doc/project_info_spec.cr +++ b/spec/compiler/crystal/tools/doc/project_info_spec.cr @@ -5,6 +5,8 @@ private alias ProjectInfo = Crystal::Doc::ProjectInfo private def run_git(command) Process.run(%(git -c user.email="" -c user.name="spec" #{command}), shell: true) +rescue IO::Error + pending! "Git is not available" end private def assert_with_defaults(initial, expected, *, file = __FILE__, line = __LINE__) diff --git a/spec/compiler/crystal/tools/init_spec.cr b/spec/compiler/crystal/tools/init_spec.cr index 71bbd8de9d35..9149986a673c 100644 --- a/spec/compiler/crystal/tools/init_spec.cr +++ b/spec/compiler/crystal/tools/init_spec.cr @@ -41,9 +41,17 @@ private def run_init_project(skeleton_type, name, author, email, github_name, di ).run end +private def git_available? + Process.run(Crystal::Git.executable).success? +rescue IO::Error + false +end + module Crystal describe Init::InitProject do it "correctly uses git config" do + pending! "Git is not available" unless git_available? + within_temporary_directory do File.write(".gitconfig", <<-INI) [user] @@ -212,9 +220,11 @@ module Crystal ) end - with_file "example/.git/config" { } + if git_available? + with_file "example/.git/config" { } - with_file "other-example-directory/.git/config" { } + with_file "other-example-directory/.git/config" { } + end end end end From 0e1018fb2369afef0bf4c4aa9a3944cff10c39ec Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 25 Oct 2024 23:50:56 +0800 Subject: [PATCH 183/193] Add MinGW-w64 CI workflow for stdlib and compiler specs (#15124) The use of the build artifact is temporary; once Crystal is available from MSYS2's Pacman, we should be able to use that directly for the test job. (The build job needs to stay because we need that artifact to bootstrap the Pacman build first.) The MSYS Git is only needed for `spec/compiler/crystal/tools/init_spec.cr` and `spec/compiler/crystal/tools/doc/project_info_spec.cr`. Apparently some of the specs in those files fail if Git cannot be located at all. As a final touch, this PR also ensures build commands have their embedded newlines replaced with whitespace, just like for MSVC, otherwise tools like `ld.exe` might consider `\n-lLLVM-18\n` on the command line a single argument and fail. --- .github/workflows/mingw-w64.yml | 63 ++++++++++++++++++++++++++++++++ src/compiler/crystal/compiler.cr | 2 +- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index 2370ae133cdd..b32aed414f77 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -8,6 +8,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} +env: + SPEC_SPLIT_DOTS: 160 + jobs: x86_64-mingw-w64-cross-compile: runs-on: ubuntu-24.04 @@ -89,3 +92,63 @@ jobs: path: | bin/ share/ + + x86_64-mingw-w64-test: + runs-on: windows-2022 + needs: [x86_64-mingw-w64-link] + steps: + - name: Setup MSYS2 + id: msys2 + uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1 + with: + msystem: UCRT64 + update: true + install: >- + git + make + mingw-w64-ucrt-x86_64-pkgconf + mingw-w64-ucrt-x86_64-cc + mingw-w64-ucrt-x86_64-gc + mingw-w64-ucrt-x86_64-pcre2 + mingw-w64-ucrt-x86_64-libiconv + mingw-w64-ucrt-x86_64-zlib + mingw-w64-ucrt-x86_64-llvm + mingw-w64-ucrt-x86_64-gmp + mingw-w64-ucrt-x86_64-libxml2 + mingw-w64-ucrt-x86_64-libyaml + mingw-w64-ucrt-x86_64-openssl + mingw-w64-ucrt-x86_64-libffi + + - name: Disable CRLF line ending substitution + run: | + git config --global core.autocrlf false + + - name: Download Crystal source + uses: actions/checkout@v4 + + - name: Download Crystal executable + uses: actions/download-artifact@v4 + with: + name: x86_64-mingw-w64-crystal + path: crystal + + - name: Run stdlib specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make std_spec + + - name: Run compiler specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make compiler_spec FLAGS=-Dwithout_ffi + + - name: Run primitives specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make -o .build/crystal.exe primitives_spec # we know the compiler is fresh; do not rebuild it here diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 83509bf88392..ffd2b3e4a7d2 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -479,7 +479,7 @@ module Crystal link_flags += " -Wl,--stack,0x800000" lib_flags = program.lib_flags(@cross_compile) lib_flags = expand_lib_flags(lib_flags) if expand - cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}) + cmd = %(#{DEFAULT_LINKER} #{Process.quote_windows(object_names)} -o #{Process.quote_windows(output_filename)} #{link_flags} #{lib_flags}).gsub('\n', ' ') if cmd.size > 32000 # The command line would be too big, pass the args through a file instead. From be1e1e54308c3986f59af993d290f70ab5516daf Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 28 Oct 2024 17:50:48 +0800 Subject: [PATCH 184/193] Add `cc`'s search paths to Unix dynamic library loader (#15127) --- spec/std/llvm/aarch64_spec.cr | 7 ------- spec/std/llvm/arm_abi_spec.cr | 7 ------- spec/std/llvm/avr_spec.cr | 7 ------- spec/std/llvm/llvm_spec.cr | 7 ------- spec/std/llvm/type_spec.cr | 7 ------- spec/std/llvm/x86_64_abi_spec.cr | 7 ------- spec/std/llvm/x86_abi_spec.cr | 7 ------- src/compiler/crystal/loader/unix.cr | 31 ++++++++++++++++++++++++++++- 8 files changed, 30 insertions(+), 50 deletions(-) diff --git a/spec/std/llvm/aarch64_spec.cr b/spec/std/llvm/aarch64_spec.cr index 6e2bac04dc47..41a308b480ec 100644 --- a/spec/std/llvm/aarch64_spec.cr +++ b/spec/std/llvm/aarch64_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::AArch64 - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:aarch64) %} diff --git a/spec/std/llvm/arm_abi_spec.cr b/spec/std/llvm/arm_abi_spec.cr index 8132ca0a38ce..98ae9b588a41 100644 --- a/spec/std/llvm/arm_abi_spec.cr +++ b/spec/std/llvm/arm_abi_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::ARM - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:arm) %} diff --git a/spec/std/llvm/avr_spec.cr b/spec/std/llvm/avr_spec.cr index 3c23c9bbed6e..a6e95d8937be 100644 --- a/spec/std/llvm/avr_spec.cr +++ b/spec/std/llvm/avr_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::AVR - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:avr) %} diff --git a/spec/std/llvm/llvm_spec.cr b/spec/std/llvm/llvm_spec.cr index 17ea96d5e261..e39398879e5d 100644 --- a/spec/std/llvm/llvm_spec.cr +++ b/spec/std/llvm/llvm_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM - {% skip_file %} -{% end %} - require "llvm" describe LLVM do diff --git a/spec/std/llvm/type_spec.cr b/spec/std/llvm/type_spec.cr index 8c6b99662ca2..94e34f226250 100644 --- a/spec/std/llvm/type_spec.cr +++ b/spec/std/llvm/type_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::Type - {% skip_file %} -{% end %} - require "llvm" describe LLVM::Type do diff --git a/spec/std/llvm/x86_64_abi_spec.cr b/spec/std/llvm/x86_64_abi_spec.cr index 8b971a679c2a..0ba644cefa01 100644 --- a/spec/std/llvm/x86_64_abi_spec.cr +++ b/spec/std/llvm/x86_64_abi_spec.cr @@ -1,11 +1,4 @@ require "spec" - -{% if flag?(:interpreted) && !flag?(:win32) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::X86_64 - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:x86) %} diff --git a/spec/std/llvm/x86_abi_spec.cr b/spec/std/llvm/x86_abi_spec.cr index b79ebc4d4d5c..27d387820298 100644 --- a/spec/std/llvm/x86_abi_spec.cr +++ b/spec/std/llvm/x86_abi_spec.cr @@ -1,13 +1,6 @@ {% skip_file if flag?(:win32) %} # 32-bit windows is not supported require "spec" - -{% if flag?(:interpreted) %} - # TODO: figure out how to link against libstdc++ in interpreted code (#14398) - pending LLVM::ABI::X86 - {% skip_file %} -{% end %} - require "llvm" {% if LibLLVM::BUILT_TARGETS.includes?(:x86) %} diff --git a/src/compiler/crystal/loader/unix.cr b/src/compiler/crystal/loader/unix.cr index dfab9736b038..962a3a47f22a 100644 --- a/src/compiler/crystal/loader/unix.cr +++ b/src/compiler/crystal/loader/unix.cr @@ -76,6 +76,15 @@ class Crystal::Loader parser.unknown_args do |args, after_dash| file_paths.concat args end + + # although flags starting with `-Wl,` appear in `args` above, this is + # still called by `OptionParser`, so we assume it is fine to ignore these + # flags + parser.invalid_option do |arg| + unless arg.starts_with?("-Wl,") + raise LoadError.new "Not a recognized linker flag: #{arg}" + end + end end search_paths = extra_search_paths + search_paths @@ -162,6 +171,10 @@ class Crystal::Loader read_ld_conf(default_search_paths) {% end %} + cc_each_library_path do |path| + default_search_paths << path + end + {% if flag?(:darwin) %} default_search_paths << "/usr/lib" default_search_paths << "/usr/local/lib" @@ -179,7 +192,7 @@ class Crystal::Loader default_search_paths << "/usr/lib" {% end %} - default_search_paths + default_search_paths.uniq! end def self.read_ld_conf(array = [] of String, path = "/etc/ld.so.conf") : Nil @@ -201,4 +214,20 @@ class Crystal::Loader end end end + + def self.cc_each_library_path(& : String ->) : Nil + search_dirs = begin + `#{Crystal::Compiler::DEFAULT_LINKER} -print-search-dirs` + rescue IO::Error + return + end + + search_dirs.each_line do |line| + if libraries = line.lchop?("libraries: =") + libraries.split(Process::PATH_DELIMITER) do |path| + yield File.expand_path(path) + end + end + end + end end From e60cb731f4db08c8502428511a5b9d68c8f64260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 28 Oct 2024 16:16:43 +0100 Subject: [PATCH 185/193] Fix remove trailing whitespace from CRYSTAL definition (#15131) #15123 broke the Makefile (and in turn CI: https://app.circleci.com/pipelines/github/crystal-lang/crystal/16310/workflows/f2194e36-a31e-4df1-87ab-64fa2ced45e2/jobs/86740) Since this commit, `"$(O)/$(CRYSTAL)$(EXE)"` in the `install` recpie resolves to the path `.build/crystal ` which doesn't exist. It looks like `$(EXE)` resolves to a single whitespace, but the error is actually in the definition of `CRYSTAL` which contains a trailing whitespace. This is only an issue in the `install` recipe because it's the only place where we put the path in quotes. So it would be simple to fix this by removing the quotes. The introduction of `$(EXE)` replaced `$(CRYSTAL_BIN)` with `$(CRYSTAL)$(EXE)`. But this is wrong. `CRYSTAL` describes the base compiler, not the output path. This patch partially reverts #15123 and reintroduces `$(CRYSTAL_BIN)`, but it's now based on `$(EXE)`. --- Makefile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 37c42890c9f4..51ab60bb40ec 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ all: ## Run generators (Unicode, SSL config, ...) ## $ make -B generate_data -CRYSTAL ?= crystal ## which previous crystal compiler use +CRYSTAL ?= crystal## which previous crystal compiler use LLVM_CONFIG ?= ## llvm-config command path to use release ?= ## Compile in release mode @@ -74,6 +74,7 @@ else EXE := WINDOWS := endif +CRYSTAL_BIN := crystal$(EXE) DESTDIR ?= PREFIX ?= /usr/local @@ -129,7 +130,7 @@ interpreter_spec: $(O)/interpreter_spec$(EXE) ## Run interpreter specs .PHONY: smoke_test smoke_test: ## Build specs as a smoke test -smoke_test: $(O)/std_spec$(EXE) $(O)/compiler_spec$(EXE) $(O)/$(CRYSTAL)$(EXE) +smoke_test: $(O)/std_spec$(EXE) $(O)/compiler_spec$(EXE) $(O)/$(CRYSTAL_BIN) .PHONY: all_spec all_spec: $(O)/all_spec$(EXE) ## Run all specs (note: this builds a huge program; `test` recipe builds individual binaries and is recommended for reduced resource usage) @@ -146,7 +147,7 @@ docs: ## Generate standard library documentation cp -av doc/ docs/ .PHONY: crystal -crystal: $(O)/$(CRYSTAL)$(EXE) ## Build the compiler +crystal: $(O)/$(CRYSTAL_BIN) ## Build the compiler .PHONY: deps llvm_ext deps: $(DEPS) ## Build dependencies @@ -161,9 +162,9 @@ generate_data: ## Run generator scripts for Unicode, SSL config, ... $(MAKE) -B -f scripts/generate_data.mk .PHONY: install -install: $(O)/$(CRYSTAL)$(EXE) man/crystal.1.gz ## Install the compiler at DESTDIR +install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -d -m 0755 "$(BINDIR)/" - $(INSTALL) -m 0755 "$(O)/$(CRYSTAL)$(EXE)" "$(BINDIR)/$(CRYSTAL)$(EXE)" + $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" $(INSTALL) -d -m 0755 $(DATADIR) cp -av src "$(DATADIR)/src" @@ -183,14 +184,14 @@ install: $(O)/$(CRYSTAL)$(EXE) man/crystal.1.gz ## Install the compiler at DESTD ifeq ($(WINDOWS),1) .PHONY: install_dlls -install_dlls: $(O)/$(CRYSTAL)$(EXE) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) +install_dlls: $(O)/$(CRYSTAL_BIN) ## Install the compiler's dependent DLLs at DESTDIR (Windows only) $(INSTALL) -d -m 0755 "$(BINDIR)/" - @ldd $(O)/$(CRYSTAL)$(EXE) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" + @ldd $(O)/$(CRYSTAL_BIN) | grep -iv ' => /c/windows/system32' | sed 's/.* => //; s/ (.*//' | xargs -t -i $(INSTALL) -m 0755 '{}' "$(BINDIR)/" endif .PHONY: uninstall uninstall: ## Uninstall the compiler from DESTDIR - rm -f "$(BINDIR)/$(CRYSTAL)$(EXE)" + rm -f "$(BINDIR)/$(CRYSTAL_BIN)" rm -rf "$(DATADIR)/src" @@ -227,7 +228,7 @@ $(O)/compiler_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr --release -$(O)/primitives_spec$(EXE): $(O)/$(CRYSTAL)$(EXE) $(DEPS) $(SOURCES) $(SPEC_SOURCES) +$(O)/primitives_spec$(EXE): $(O)/$(CRYSTAL_BIN) $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/primitives_spec.cr @@ -236,7 +237,7 @@ $(O)/interpreter_spec$(EXE): $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler/interpreter_spec.cr -$(O)/$(CRYSTAL)$(EXE): $(DEPS) $(SOURCES) +$(O)/$(CRYSTAL_BIN): $(DEPS) $(SOURCES) $(call check_llvm_config) @mkdir -p $(O) @# NOTE: USE_PCRE1 is only used for testing compatibility with legacy environments that don't provide libpcre2. From 6118fa2393120808c67f01831206bbbd28087ab7 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Tue, 29 Oct 2024 19:40:10 +0800 Subject: [PATCH 186/193] Fix `Crystal::Loader.default_search_paths` spec for macOS (#15135) --- spec/compiler/loader/unix_spec.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/compiler/loader/unix_spec.cr b/spec/compiler/loader/unix_spec.cr index 6adcb040148b..e3309346803c 100644 --- a/spec/compiler/loader/unix_spec.cr +++ b/spec/compiler/loader/unix_spec.cr @@ -53,7 +53,7 @@ describe Crystal::Loader do with_env "LD_LIBRARY_PATH": "ld1::ld2", "DYLD_LIBRARY_PATH": nil do search_paths = Crystal::Loader.default_search_paths {% if flag?(:darwin) %} - search_paths.should eq ["/usr/lib", "/usr/local/lib"] + search_paths[-2..].should eq ["/usr/lib", "/usr/local/lib"] {% else %} search_paths[0, 2].should eq ["ld1", "ld2"] {% if flag?(:android) %} From 04ace0405c19785a45112b50e0753b445a599e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 30 Oct 2024 11:14:00 +0100 Subject: [PATCH 187/193] Make `Box` constructor and `object` getter nodoc (#15136) --- src/box.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/box.cr b/src/box.cr index 78799838e688..a5a6900b2ea1 100644 --- a/src/box.cr +++ b/src/box.cr @@ -5,9 +5,13 @@ # # For an example usage, see `Proc`'s explanation about sending Procs to C. class Box(T) + # :nodoc: + # # Returns the original object getter object : T + # :nodoc: + # # Creates a `Box` with the given object. # # This method isn't usually used directly. Instead, `Box.box` is used. From fd44c0816f85ea0e4e75419a0784984bad796603 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 30 Oct 2024 23:36:07 +0100 Subject: [PATCH 188/193] Add indirect branch tracking (#15122) Adds support for indirect branch tracking for X86[_64] (CET) and AArch64 targets through the following compile time flags (taken from gcc/clang/rust): - `-Dcf-protection=branch` (or `=return` or `=full`) for X86 - `-Dbranch-protection=bti` for AArch64 These flags are automatically set for OpenBSD, that enforces IBT or BTI on all user land applications. The patch also removes the `-Wl-znobtcfi` linker option since we don't need to disable it anymore. OpenBSD is the only OS I know to support _and_ enforce IBT or BTI in user land. Linux for example only supports it for kernel code (for the time being). I manually tested IBT in an OpenBSD VM on x86_64 with a supported CPU (Intel Raptor Lake). I can compile & recompile crystal as well as run `gmake std_spec` without running into IBT issues :+1: Notes: - I expected to have to add the ASM instructions to the fiber context switch ASM... but messing with the stack pointer isn't considered as a conditional jump apparently :shrug: - I'm using the genius idea from @straight-shoota that we can pass `-Dkey=value` then test for `flag?("key=value")` and it just worked :astonished: - I can't test BTI on AArch64: I have no hardware and there are no bindings for the `aarch64-unknown-openbsd` target; there are little reasons it wouldn't work though; - I added support for shadow stack (SHSTK) on X86 (`-Dcf-protection=return`). I'm not sure we really support it though, since fibers are messing with the stacks? --- src/compiler/crystal/codegen/codegen.cr | 32 +++++++++++++++++++------ src/compiler/crystal/codegen/debug.cr | 8 ++----- src/compiler/crystal/codegen/fun.cr | 3 +-- src/compiler/crystal/compiler.cr | 6 ----- src/compiler/crystal/semantic/flags.cr | 13 +++++++++- src/llvm/lib_llvm/core.cr | 7 +++++- src/llvm/module.cr | 4 ++++ 7 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index c4844df9a5e8..7e15b1bdc385 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -274,7 +274,7 @@ module Crystal @llvm_context : LLVM::Context = LLVM::Context.new) @abi = @program.target_machine.abi # LLVM::Context.register(@llvm_context, "main") - @llvm_mod = @llvm_context.new_module("main_module") + @llvm_mod = configure_module(@llvm_context.new_module("main_module")) @main_mod = @llvm_mod @main_llvm_context = @main_mod.context @llvm_typer = LLVMTyper.new(@program, @llvm_context) @@ -345,8 +345,6 @@ module Crystal @unused_fun_defs = [] of FunDef @proc_counts = Hash(String, Int32).new(0) - @llvm_mod.data_layout = self.data_layout - # We need to define __crystal_malloc and __crystal_realloc as soon as possible, # to avoid some memory being allocated with plain malloc. codegen_well_known_functions @node @@ -367,6 +365,30 @@ module Crystal getter llvm_context + def configure_module(llvm_mod) + llvm_mod.data_layout = @program.target_machine.data_layout + + # enable branch authentication instructions (BTI) + if @program.has_flag?("aarch64") + if @program.has_flag?("branch-protection=bti") + llvm_mod.add_flag(:override, "branch-target-enforcement", 1) + end + end + + # enable control flow enforcement protection (CET): IBT and/or SHSTK + if @program.has_flag?("x86_64") || @program.has_flag?("i386") + if @program.has_flag?("cf-protection=branch") || @program.has_flag?("cf-protection=full") + llvm_mod.add_flag(:override, "cf-protection-branch", 1) + end + + if @program.has_flag?("cf-protection=return") || @program.has_flag?("cf-protection=full") + llvm_mod.add_flag(:override, "cf-protection-return", 1) + end + end + + llvm_mod + end + def new_builder(llvm_context) wrap_builder(llvm_context.new_builder) end @@ -419,10 +441,6 @@ module Crystal global.initializer = llvm_element_type.const_array(llvm_elements) end - def data_layout - @program.target_machine.data_layout - end - class CodegenWellKnownFunctions < Visitor @codegen : CodeGenVisitor diff --git a/src/compiler/crystal/codegen/debug.cr b/src/compiler/crystal/codegen/debug.cr index dd4b6c361905..870506377f7a 100644 --- a/src/compiler/crystal/codegen/debug.cr +++ b/src/compiler/crystal/codegen/debug.cr @@ -42,17 +42,13 @@ module Crystal if @program.has_flag?("msvc") # Windows uses CodeView instead of DWARF - mod.add_flag( - LibLLVM::ModuleFlagBehavior::Warning, - "CodeView", - mod.context.int32.const_int(1) - ) + mod.add_flag(LibLLVM::ModuleFlagBehavior::Warning, "CodeView", 1) end mod.add_flag( LibLLVM::ModuleFlagBehavior::Warning, "Debug Info Version", - mod.context.int32.const_int(LLVM::DEBUG_METADATA_VERSION) + LLVM::DEBUG_METADATA_VERSION ) end diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index 616b21b79d24..c56bde6e5c2a 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -626,8 +626,7 @@ class Crystal::CodeGenVisitor # LLVM::Context.register(llvm_context, type_name) llvm_typer = LLVMTyper.new(@program, llvm_context) - llvm_mod = llvm_context.new_module(type_name) - llvm_mod.data_layout = self.data_layout + llvm_mod = configure_module(llvm_context.new_module(type_name)) llvm_builder = new_builder(llvm_context) define_symbol_table llvm_mod, llvm_typer diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index ffd2b3e4a7d2..878a1ae4896a 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -507,12 +507,6 @@ module Crystal link_flags += " -L/usr/local/lib" end - if program.has_flag?("openbsd") - # OpenBSD requires Indirect Branch Tracking by default, but we're not - # compatible (yet), so we disable it for now: - link_flags += " -Wl,-znobtcfi" - end - {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names} end end diff --git a/src/compiler/crystal/semantic/flags.cr b/src/compiler/crystal/semantic/flags.cr index d455f1fdb0c7..d4b0f265a3d1 100644 --- a/src/compiler/crystal/semantic/flags.cr +++ b/src/compiler/crystal/semantic/flags.cr @@ -49,7 +49,18 @@ class Crystal::Program flags.add "freebsd#{target.freebsd_version}" end flags.add "netbsd" if target.netbsd? - flags.add "openbsd" if target.openbsd? + + if target.openbsd? + flags.add "openbsd" + + case target.architecture + when "aarch64" + flags.add "branch-protection=bti" unless flags.any?(&.starts_with?("branch-protection=")) + when "x86_64", "i386" + flags.add "cf-protection=branch" unless flags.any?(&.starts_with?("cf-protection=")) + end + end + flags.add "dragonfly" if target.dragonfly? flags.add "solaris" if target.solaris? flags.add "android" if target.android? diff --git a/src/llvm/lib_llvm/core.cr b/src/llvm/lib_llvm/core.cr index 1796bd00a0ee..7137501fdb31 100644 --- a/src/llvm/lib_llvm/core.cr +++ b/src/llvm/lib_llvm/core.cr @@ -5,7 +5,12 @@ lib LibLLVM # counterparts (e.g. `LLVMModuleFlagBehavior` v.s. `LLVM::Module::ModFlagBehavior`) enum ModuleFlagBehavior - Warning = 1 + Error = 0 + Warning = 1 + Require = 2 + Override = 3 + Append = 4 + AppendUnique = 5 end alias AttributeIndex = UInt diff --git a/src/llvm/module.cr b/src/llvm/module.cr index 32b025bffee7..0e73e983358a 100644 --- a/src/llvm/module.cr +++ b/src/llvm/module.cr @@ -45,6 +45,10 @@ class LLVM::Module GlobalCollection.new(self) end + def add_flag(module_flag : LibLLVM::ModuleFlagBehavior, key : String, val : Int32) + add_flag(module_flag, key, @context.int32.const_int(val)) + end + def add_flag(module_flag : LibLLVM::ModuleFlagBehavior, key : String, val : Value) LibLLVM.add_module_flag( self, From d353bb42b2a3c7dcb5d03d169634313a856d1445 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 31 Oct 2024 18:16:14 +0800 Subject: [PATCH 189/193] Support deferencing symlinks in `make install` (#15138) On platforms without complete symbolic link support (e.g. native MSYS2 environments), `make install deref_symlinks=1` will dereference the individual directories under `src/lib_c` and copy the contents, instead of copying the directories as symbolic links. --- Makefile | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 51ab60bb40ec..d30db53464f7 100644 --- a/Makefile +++ b/Makefile @@ -24,18 +24,19 @@ all: CRYSTAL ?= crystal## which previous crystal compiler use LLVM_CONFIG ?= ## llvm-config command path to use -release ?= ## Compile in release mode -stats ?= ## Enable statistics output -progress ?= ## Enable progress output -threads ?= ## Maximum number of threads to use -debug ?= ## Add symbolic debug info -verbose ?= ## Run specs in verbose mode -junit_output ?= ## Path to output junit results -static ?= ## Enable static linking -target ?= ## Cross-compilation target -interpreter ?= ## Enable interpreter feature -check ?= ## Enable only check when running format -order ?=random ## Enable order for spec execution (values: "default" | "random" | seed number) +release ?= ## Compile in release mode +stats ?= ## Enable statistics output +progress ?= ## Enable progress output +threads ?= ## Maximum number of threads to use +debug ?= ## Add symbolic debug info +verbose ?= ## Run specs in verbose mode +junit_output ?= ## Path to output junit results +static ?= ## Enable static linking +target ?= ## Cross-compilation target +interpreter ?= ## Enable interpreter feature +check ?= ## Enable only check when running format +order ?=random ## Enable order for spec execution (values: "default" | "random" | seed number) +deref_symlinks ?= ## Deference symbolic links for `make install` O := .build SOURCES := $(shell find src -name '*.cr') @@ -167,7 +168,7 @@ install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR $(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)" $(INSTALL) -d -m 0755 $(DATADIR) - cp -av src "$(DATADIR)/src" + cp $(if $(deref_symlinks),-rvL --preserve=all,-av) src "$(DATADIR)/src" rm -rf "$(DATADIR)/$(LLVM_EXT_OBJ)" # Don't install llvm_ext.o $(INSTALL) -d -m 0755 "$(MANDIR)/man1/" From 8635bce731a58b55a7a95a1a21ed892e81590b5d Mon Sep 17 00:00:00 2001 From: Barney <86712892+BigBoyBarney@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:17:43 +0100 Subject: [PATCH 190/193] Improve docs for `String#rindex!` (#15132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller --- src/string.cr | 70 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/src/string.cr b/src/string.cr index f0dbd1a1eae3..7507e3b7249e 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3473,8 +3473,8 @@ class String # ``` # "Hello, World".rindex('o') # => 8 # "Hello, World".rindex('Z') # => nil - # "Hello, World".rindex("o", 5) # => 4 - # "Hello, World".rindex("W", 2) # => nil + # "Hello, World".rindex('o', 5) # => 4 + # "Hello, World".rindex('W', 2) # => nil # ``` def rindex(search : Char, offset = size - 1) # If it's ASCII we can delegate to slice @@ -3519,7 +3519,16 @@ class String end end - # :ditto: + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # + # ``` + # "Hello, World".rindex("orld") # => 8 + # "Hello, World".rindex("snorlax") # => nil + # "Hello, World".rindex("o", 5) # => 4 + # "Hello, World".rindex("W", 2) # => nil + # ``` def rindex(search : String, offset = size - search.size) : Int32? offset += size if offset < 0 return if offset < 0 @@ -3572,7 +3581,16 @@ class String end end - # :ditto: + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # + # ``` + # "Hello, World".rindex(/world/i) # => 7 + # "Hello, World".rindex(/world/) # => nil + # "Hello, World".rindex(/o/, 5) # => 4 + # "Hello, World".rindex(/W/, 2) # => nil + # ``` def rindex(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32? offset += size if offset < 0 return nil unless 0 <= offset <= size @@ -3586,21 +3604,49 @@ class String match_result.try &.begin end - # :ditto: - # + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`. - def rindex!(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32 - rindex(search, offset, options: options) || raise Enumerable::NotFoundError.new + # + # ``` + # "Hello, World".rindex!('o') # => 8 + # "Hello, World".rindex!('Z') # raises Enumerable::NotFoundError + # "Hello, World".rindex!('o', 5) # => 4 + # "Hello, World".rindex!('W', 2) # raises Enumerable::NotFoundError + # ``` + def rindex!(search : Char, offset = size - 1) : Int32 + rindex(search, offset) || raise Enumerable::NotFoundError.new end - # :ditto: + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`. + # + # ``` + # "Hello, World".rindex!("orld") # => 8 + # "Hello, World".rindex!("snorlax") # raises Enumerable::NotFoundError + # "Hello, World".rindex!("o", 5) # => 4 + # "Hello, World".rindex!("W", 2) # raises Enumerable::NotFoundError + # ``` def rindex!(search : String, offset = size - search.size) : Int32 rindex(search, offset) || raise Enumerable::NotFoundError.new end - # :ditto: - def rindex!(search : Char, offset = size - 1) : Int32 - rindex(search, offset) || raise Enumerable::NotFoundError.new + # Returns the index of the _last_ appearance of *search* in the string, + # If *offset* is present, it defines the position to _end_ the search + # (characters beyond this point are ignored). + # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`. + # + # ``` + # "Hello, World".rindex!(/world/i) # => 7 + # "Hello, World".rindex!(/world/) # raises Enumerable::NotFoundError + # "Hello, World".rindex!(/o/, 5) # => 4 + # "Hello, World".rindex!(/W/, 2) # raises Enumerable::NotFoundError + # ``` + def rindex!(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32 + rindex(search, offset, options: options) || raise Enumerable::NotFoundError.new end # Searches separator or pattern (`Regex`) in the string, and returns From 4aac6f2ee3494a593d79eeea389c77037e025adc Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 31 Oct 2024 18:20:50 +0800 Subject: [PATCH 191/193] Protect constant initializers with mutex on Windows (#15134) `Crystal::System::FileDescriptor#@@reader_thread` is initialized before `Crystal::System::Fiber::RESERVED_STACK_SIZE` which creates a race condition. Regression from #14947 --- src/crystal/once.cr | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/crystal/once.cr b/src/crystal/once.cr index 1e6243669809..56eea2be693a 100644 --- a/src/crystal/once.cr +++ b/src/crystal/once.cr @@ -11,9 +11,6 @@ # :nodoc: class Crystal::OnceState @rec = [] of Bool* - {% if flag?(:preview_mt) %} - @mutex = Mutex.new(:reentrant) - {% end %} def once(flag : Bool*, initializer : Void*) unless flag.value @@ -29,7 +26,13 @@ class Crystal::OnceState end end - {% if flag?(:preview_mt) %} + # on Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new + # thread even without the `preview_mt` flag, and the thread can also reference + # Crystal constants, leading to race conditions, so we always enable the mutex + # TODO: can this be improved? + {% if flag?(:preview_mt) || flag?(:win32) %} + @mutex = Mutex.new(:reentrant) + def once(flag : Bool*, initializer : Void*) unless flag.value @mutex.synchronize do From 2eb1b5fb25b3240dda9f9710f8df2c69462f6e8f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 4 Nov 2024 06:22:09 +0800 Subject: [PATCH 192/193] Basic MinGW-w64-based interpreter support (#15140) Implements a MinGW-based loader for `x86_64-windows-gnu` and enables the interpreter. --- .github/workflows/mingw-w64.yml | 14 +- spec/compiler/ffi/ffi_spec.cr | 10 +- spec/compiler/interpreter/lib_spec.cr | 39 ++-- spec/compiler/loader/spec_helper.cr | 3 + src/compiler/crystal/interpreter/context.cr | 8 +- src/compiler/crystal/loader.cr | 4 +- src/compiler/crystal/loader/mingw.cr | 195 ++++++++++++++++++++ src/crystal/system/win32/wmain.cr | 2 +- 8 files changed, 246 insertions(+), 29 deletions(-) create mode 100644 src/compiler/crystal/loader/mingw.cr diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index b32aed414f77..10841a325bf5 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -31,7 +31,7 @@ jobs: crystal: "1.14.0" - name: Cross-compile Crystal - run: make && make -B target=x86_64-windows-gnu release=1 + run: make && make -B target=x86_64-windows-gnu release=1 interpreter=1 - name: Upload crystal.obj uses: actions/upload-artifact@v4 @@ -63,6 +63,7 @@ jobs: mingw-w64-ucrt-x86_64-libiconv mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-llvm + mingw-w64-ucrt-x86_64-libffi - name: Download crystal.obj uses: actions/download-artifact@v4 @@ -80,7 +81,7 @@ jobs: run: | mkdir bin cc crystal.obj -o bin/crystal.exe \ - $(pkg-config bdw-gc libpcre2-8 iconv zlib --libs) \ + $(pkg-config bdw-gc libpcre2-8 iconv zlib libffi --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 ldd bin/crystal.exe | grep -iv /c/windows/system32 | sed 's/.* => //; s/ (.*//' | xargs -t -i cp '{}' bin/ @@ -144,7 +145,14 @@ jobs: run: | export PATH="$(pwd)/crystal/bin:$PATH" export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" - make compiler_spec FLAGS=-Dwithout_ffi + make compiler_spec + + - name: Run interpreter specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make interpreter_spec - name: Run primitives specs shell: msys2 {0} diff --git a/spec/compiler/ffi/ffi_spec.cr b/spec/compiler/ffi/ffi_spec.cr index ec644e45870d..a4718edb3501 100644 --- a/spec/compiler/ffi/ffi_spec.cr +++ b/spec/compiler/ffi/ffi_spec.cr @@ -27,7 +27,7 @@ private def dll_search_paths {% end %} end -{% if flag?(:unix) %} +{% if flag?(:unix) || (flag?(:win32) && flag?(:gnu)) %} class Crystal::Loader def self.new(search_paths : Array(String), *, dll_search_paths : Nil) new(search_paths) @@ -39,9 +39,17 @@ describe Crystal::FFI::CallInterface do before_all do FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) build_c_dynlib(compiler_datapath("ffi", "sum.c")) + + {% if flag?(:win32) && flag?(:gnu) %} + ENV["PATH"] = "#{SPEC_CRYSTAL_LOADER_LIB_PATH}#{Process::PATH_DELIMITER}#{ENV["PATH"]}" + {% end %} end after_all do + {% if flag?(:win32) && flag?(:gnu) %} + ENV["PATH"] = ENV["PATH"].delete_at(0, ENV["PATH"].index!(Process::PATH_DELIMITER) + 1) + {% end %} + FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) end diff --git a/spec/compiler/interpreter/lib_spec.cr b/spec/compiler/interpreter/lib_spec.cr index 2c1798645645..bbf6367ee6df 100644 --- a/spec/compiler/interpreter/lib_spec.cr +++ b/spec/compiler/interpreter/lib_spec.cr @@ -3,7 +3,7 @@ require "./spec_helper" require "../loader/spec_helper" private def ldflags - {% if flag?(:win32) %} + {% if flag?(:msvc) %} "/LIBPATH:#{SPEC_CRYSTAL_LOADER_LIB_PATH} sum.lib" {% else %} "-L#{SPEC_CRYSTAL_LOADER_LIB_PATH} -lsum" @@ -11,7 +11,7 @@ private def ldflags end private def ldflags_with_backtick - {% if flag?(:win32) %} + {% if flag?(:msvc) %} "/LIBPATH:#{SPEC_CRYSTAL_LOADER_LIB_PATH} `powershell.exe -C Write-Host -NoNewline sum.lib`" {% else %} "-L#{SPEC_CRYSTAL_LOADER_LIB_PATH} -l`echo sum`" @@ -19,12 +19,24 @@ private def ldflags_with_backtick end describe Crystal::Repl::Interpreter do - context "variadic calls" do - before_all do - FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) - build_c_dynlib(compiler_datapath("interpreter", "sum.c")) - end + before_all do + FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) + build_c_dynlib(compiler_datapath("interpreter", "sum.c")) + + {% if flag?(:win32) %} + ENV["PATH"] = "#{SPEC_CRYSTAL_LOADER_LIB_PATH}#{Process::PATH_DELIMITER}#{ENV["PATH"]}" + {% end %} + end + + after_all do + {% if flag?(:win32) %} + ENV["PATH"] = ENV["PATH"].delete_at(0, ENV["PATH"].index!(Process::PATH_DELIMITER) + 1) + {% end %} + + FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) + end + context "variadic calls" do it "promotes float" do interpret(<<-CRYSTAL).should eq 3.5 @[Link(ldflags: #{ldflags.inspect})] @@ -65,18 +77,9 @@ describe Crystal::Repl::Interpreter do LibSum.sum_int(2, E::ONE, F::FOUR) CRYSTAL end - - after_all do - FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) - end end context "command expansion" do - before_all do - FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) - build_c_dynlib(compiler_datapath("interpreter", "sum.c")) - end - it "expands ldflags" do interpret(<<-CRYSTAL).should eq 4 @[Link(ldflags: #{ldflags_with_backtick.inspect})] @@ -87,9 +90,5 @@ describe Crystal::Repl::Interpreter do LibSum.simple_sum_int(2, 2) CRYSTAL end - - after_all do - FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) - end end end diff --git a/spec/compiler/loader/spec_helper.cr b/spec/compiler/loader/spec_helper.cr index 0db69dc19752..5b2a6454bfa1 100644 --- a/spec/compiler/loader/spec_helper.cr +++ b/spec/compiler/loader/spec_helper.cr @@ -8,6 +8,9 @@ def build_c_dynlib(c_filename, *, lib_name = nil, target_dir = SPEC_CRYSTAL_LOAD {% if flag?(:msvc) %} o_basename = o_filename.rchop(".lib") `#{ENV["CC"]? || "cl.exe"} /nologo /LD #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_basename}")} #{Process.quote("/Fe#{o_basename}")}` + {% elsif flag?(:win32) && flag?(:gnu) %} + o_basename = o_filename.rchop(".a") + `#{ENV["CC"]? || "cc"} -shared -fvisibility=hidden #{Process.quote(c_filename)} -o #{Process.quote(o_basename + ".dll")} #{Process.quote("-Wl,--out-implib,#{o_basename}.a")}` {% else %} `#{ENV["CC"]? || "cc"} -shared -fvisibility=hidden #{Process.quote(c_filename)} -o #{Process.quote(o_filename)}` {% end %} diff --git a/src/compiler/crystal/interpreter/context.cr b/src/compiler/crystal/interpreter/context.cr index 50e36a3ff8b7..c2c1537e002d 100644 --- a/src/compiler/crystal/interpreter/context.cr +++ b/src/compiler/crystal/interpreter/context.cr @@ -393,14 +393,16 @@ class Crystal::Repl::Context getter(loader : Loader) { lib_flags = program.lib_flags # Execute and expand `subcommands`. - lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}` } + lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}`.chomp } args = Process.parse_arguments(lib_flags) # FIXME: Part 1: This is a workaround for initial integration of the interpreter: # The loader can't handle the static libgc.a usually shipped with crystal and loading as a shared library conflicts # with the compiler's own GC. - # (MSVC doesn't seem to have this issue) - args.delete("-lgc") + # (Windows doesn't seem to have this issue) + unless program.has_flag?("win32") && program.has_flag?("gnu") + args.delete("-lgc") + end # recreate the MSVC developer prompt environment, similar to how compiled # code does it in `Compiler#linker_command` diff --git a/src/compiler/crystal/loader.cr b/src/compiler/crystal/loader.cr index 5a147dad590f..84ff43d03d8e 100644 --- a/src/compiler/crystal/loader.cr +++ b/src/compiler/crystal/loader.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) || flag?(:msvc) %} +{% skip_file unless flag?(:unix) || flag?(:win32) %} require "option_parser" # This loader component imitates the behaviour of `ld.so` for linking and loading @@ -105,4 +105,6 @@ end require "./loader/unix" {% elsif flag?(:msvc) %} require "./loader/msvc" +{% elsif flag?(:win32) && flag?(:gnu) %} + require "./loader/mingw" {% end %} diff --git a/src/compiler/crystal/loader/mingw.cr b/src/compiler/crystal/loader/mingw.cr new file mode 100644 index 000000000000..677f564cec16 --- /dev/null +++ b/src/compiler/crystal/loader/mingw.cr @@ -0,0 +1,195 @@ +{% skip_file unless flag?(:win32) && flag?(:gnu) %} + +require "crystal/system/win32/library_archive" + +# MinGW-based loader used on Windows. Assumes an MSYS2 shell. +# +# The core implementation is derived from the MSVC loader. Main deviations are: +# +# - `.parse` follows GNU `ld`'s style, rather than MSVC `link`'s; +# - `#library_filename` follows the usual naming of the MinGW linker: `.dll.a` +# for DLL import libraries, `.a` for other libraries; +# - `.default_search_paths` relies solely on `.cc_each_library_path`. +# +# TODO: The actual MinGW linker supports linking to DLLs directly, figure out +# how this is done. + +class Crystal::Loader + alias Handle = Void* + + def initialize(@search_paths : Array(String)) + end + + # Parses linker arguments in the style of `ld`. + # + # This is identical to the Unix loader. *dll_search_paths* has no effect. + def self.parse(args : Array(String), *, search_paths : Array(String) = default_search_paths, dll_search_paths : Array(String)? = nil) : self + libnames = [] of String + file_paths = [] of String + extra_search_paths = [] of String + + OptionParser.parse(args.dup) do |parser| + parser.on("-L DIRECTORY", "--library-path DIRECTORY", "Add DIRECTORY to library search path") do |directory| + extra_search_paths << directory + end + parser.on("-l LIBNAME", "--library LIBNAME", "Search for library LIBNAME") do |libname| + libnames << libname + end + parser.on("-static", "Do not link against shared libraries") do + raise LoadError.new "static libraries are not supported by Crystal's runtime loader" + end + parser.unknown_args do |args, after_dash| + file_paths.concat args + end + + parser.invalid_option do |arg| + unless arg.starts_with?("-Wl,") + raise LoadError.new "Not a recognized linker flag: #{arg}" + end + end + end + + search_paths = extra_search_paths + search_paths + + begin + loader = new(search_paths) + loader.load_all(libnames, file_paths) + loader + rescue exc : LoadError + exc.args = args + exc.search_paths = search_paths + raise exc + end + end + + def self.library_filename(libname : String) : String + "lib#{libname}.a" + end + + def find_symbol?(name : String) : Handle? + @handles.each do |handle| + address = LibC.GetProcAddress(handle, name.check_no_null_byte) + return address if address + end + end + + def load_file(path : String | ::Path) : Nil + load_file?(path) || raise LoadError.new "cannot load #{path}" + end + + def load_file?(path : String | ::Path) : Bool + if api_set?(path) + return load_dll?(path.to_s) + end + + return false unless File.file?(path) + + System::LibraryArchive.imported_dlls(path).all? do |dll| + load_dll?(dll) + end + end + + private def load_dll?(dll) + handle = open_library(dll) + return false unless handle + + @handles << handle + @loaded_libraries << (module_filename(handle) || dll) + true + end + + def load_library(libname : String) : Nil + load_library?(libname) || raise LoadError.new "cannot find #{Loader.library_filename(libname)}" + end + + def load_library?(libname : String) : Bool + if ::Path::SEPARATORS.any? { |separator| libname.includes?(separator) } + return load_file?(::Path[libname].expand) + end + + # attempt .dll.a before .a + # TODO: verify search order + @search_paths.each do |directory| + library_path = File.join(directory, Loader.library_filename(libname + ".dll")) + return true if load_file?(library_path) + + library_path = File.join(directory, Loader.library_filename(libname)) + return true if load_file?(library_path) + end + + false + end + + private def open_library(path : String) + LibC.LoadLibraryExW(System.to_wstr(path), nil, 0) + end + + def load_current_program_handle + if LibC.GetModuleHandleExW(0, nil, out hmodule) != 0 + @handles << hmodule + @loaded_libraries << (Process.executable_path || "current program handle") + end + end + + def close_all : Nil + @handles.each do |handle| + LibC.FreeLibrary(handle) + end + @handles.clear + end + + private def api_set?(dll) + dll.to_s.matches?(/^(?:api-|ext-)[a-zA-Z0-9-]*l\d+-\d+-\d+\.dll$/) + end + + private def module_filename(handle) + Crystal::System.retry_wstr_buffer do |buffer, small_buf| + len = LibC.GetModuleFileNameW(handle, buffer, buffer.size) + if 0 < len < buffer.size + break String.from_utf16(buffer[0, len]) + elsif small_buf && len == buffer.size + next 32767 # big enough. 32767 is the maximum total path length of UNC path. + else + break nil + end + end + end + + # Returns a list of directories used as the default search paths. + # + # Right now this depends on `cc` exclusively. + def self.default_search_paths : Array(String) + default_search_paths = [] of String + + cc_each_library_path do |path| + default_search_paths << path + end + + default_search_paths.uniq! + end + + # identical to the Unix loader + def self.cc_each_library_path(& : String ->) : Nil + search_dirs = begin + cc = + {% if Crystal.has_constant?("Compiler") %} + Crystal::Compiler::DEFAULT_LINKER + {% else %} + # this allows the loader to be required alone without the compiler + ENV["CC"]? || "cc" + {% end %} + + `#{cc} -print-search-dirs` + rescue IO::Error + return + end + + search_dirs.each_line do |line| + if libraries = line.lchop?("libraries: =") + libraries.split(Process::PATH_DELIMITER) do |path| + yield File.expand_path(path) + end + end + end + end +end diff --git a/src/crystal/system/win32/wmain.cr b/src/crystal/system/win32/wmain.cr index 3dd64f9c1b92..caad6748229f 100644 --- a/src/crystal/system/win32/wmain.cr +++ b/src/crystal/system/win32/wmain.cr @@ -7,7 +7,7 @@ require "c/stdlib" @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }})] {% if flag?(:msvc) %} @[Link(ldflags: "/ENTRY:wmainCRTStartup")] - {% elsif flag?(:gnu) %} + {% elsif flag?(:gnu) && !flag?(:interpreted) %} @[Link(ldflags: "-municode")] {% end %} {% end %} From 563d6d2de572bfec3b631f433d9b8bf99e3992c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 4 Nov 2024 12:21:15 +0100 Subject: [PATCH 193/193] Fix: use `uninitialized LibC::SigsetT` (#15144) `LibC::SigsetT` is a `struct` on some platforms and an alias to `UInt32` on others. `.new` is only valid for the struct variant. `uninitialized` should work in either case. This code is only used with `-Dgc_none`, so a reproduction needs to include that flag and build for a target where `LibC::SigsetT = alias UInt32`, such as `aarch64-apple-darwin`. --- src/crystal/system/unix/pthread.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index bbdfcbc3d41c..73aa2a652ca1 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -208,7 +208,7 @@ module Crystal::System::Thread Thread.current_thread.@suspended.set(true) # block all signals but SIG_RESUME - mask = LibC::SigsetT.new + mask = uninitialized LibC::SigsetT LibC.sigfillset(pointerof(mask)) LibC.sigdelset(pointerof(mask), SIG_RESUME)