From 2eca3988ed58ab851cf9f170a4cec4c85513c7f0 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 15:44:01 +0100 Subject: [PATCH 01/14] fix: lock various methods --- src/Convert/pyconvert.jl | 10 +++++--- src/Core/Py.jl | 6 +++-- src/JlWrap/C.jl | 53 ++++++++++++++++++++++++++++------------ src/JlWrap/base.jl | 1 + 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index d61268de..ac2750d7 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -73,7 +73,7 @@ function pyconvert_add_rule( get!(Vector{PyConvertRule}, PYCONVERT_RULES, pytypename), PyConvertRule(type, func, priority), ) - empty!.(values(PYCONVERT_RULES_CACHE)) + Base.@lock PYCONVERT_RULES_CACHE_LOCK empty!.(values(PYCONVERT_RULES_CACHE)) return end @@ -262,9 +262,10 @@ function _pyconvert_get_rules(pytype::Py) end const PYCONVERT_PREFERRED_TYPE = Dict{Py,Type}() +const PYCONVERT_PREFERRED_TYPE_LOCK = Threads.SpinLock() pyconvert_preferred_type(pytype::Py) = - get!(PYCONVERT_PREFERRED_TYPE, pytype) do + Base.@lock PYCONVERT_PREFERRED_TYPE_LOCK get!(PYCONVERT_PREFERRED_TYPE, pytype) do if pyissubclass(pytype, pybuiltins.int) Union{Int,BigInt} else @@ -308,9 +309,10 @@ end pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) const PYCONVERT_RULES_CACHE = Dict{Type,Dict{C.PyPtr,Vector{Function}}}() +const PYCONVERT_RULES_CACHE_LOCK = Threads.SpinLock() @generated pyconvert_rules_cache(::Type{T}) where {T} = - get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE, T) + Base.@lock PYCONVERT_RULES_CACHE_LOCK get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE, T) function pyconvert_rule_fast(::Type{T}, x::Py) where {T} if T isa Union @@ -352,7 +354,7 @@ function pytryconvert(::Type{T}, x_) where {T} # TODO: we should hold weak references and clear the cache if types get deleted tptr = C.Py_Type(x) trules = pyconvert_rules_cache(T) - rules = get!(trules, tptr) do + rules = Base.@lock PYCONVERT_RULES_CACHE_LOCK get!(trules, tptr) do t = pynew(incref(tptr)) ans = pyconvert_get_rules(T, t)::Vector{Function} pydel!(t) diff --git a/src/Core/Py.jl b/src/Core/Py.jl index a457ce90..dff462be 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -57,6 +57,7 @@ decref(x::Py) = Base.GC.@preserve x (decref(getptr(x)); x) Base.unsafe_convert(::Type{C.PyPtr}, x::Py) = getptr(x) const PYNULL_CACHE = Py[] +const PYNULL_CACHE_LOCK = Threads.SpinLock() """ pynew([ptr]) @@ -69,12 +70,13 @@ points at, i.e. the new `Py` object owns a reference. Note that NULL Python objects are not safe in the sense that most API functions will probably crash your Julia session if you pass a NULL argument. """ -pynew() = +pynew() = Base.@lock PYNULL_CACHE_LOCK begin if isempty(PYNULL_CACHE) Py(Val(:new), C.PyNULL) else pop!(PYNULL_CACHE) end +end const PyNULL = pynew() @@ -119,7 +121,7 @@ function pydel!(x::Py) C.Py_DecRef(ptr) setptr!(x, C.PyNULL) end - push!(PYNULL_CACHE, x) + Base.@lock PYNULL_CACHE_LOCK push!(PYNULL_CACHE, x) return end diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index fa96dd36..2cde6494 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -19,6 +19,8 @@ const PyJuliaBase_Type = Ref(C.PyNULL) const PYJLVALUES = [] # unused indices in PYJLVALUES const PYJLFREEVALUES = Int[] +# Thread safety for PYJLVALUES and PYJLFREEVALUES +const PYJLVALUES_LOCK = Threads.SpinLock() function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) @@ -31,8 +33,10 @@ end function _pyjl_dealloc(o::C.PyPtr) idx = UnsafePtr{PyJuliaValueObject}(o).value[] if idx != 0 - PYJLVALUES[idx] = nothing - push!(PYJLFREEVALUES, idx) + Base.@lock PYJLVALUES_LOCK begin + PYJLVALUES[idx] = nothing + push!(PYJLFREEVALUES, idx) + end end UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o) ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o) @@ -40,11 +44,14 @@ function _pyjl_dealloc(o::C.PyPtr) end const PYJLMETHODS = Vector{Any}() +const PYJLMETHODS_LOCK = Threads.SpinLock() function PyJulia_MethodNum(f) @nospecialize f - push!(PYJLMETHODS, f) - return length(PYJLMETHODS) + Base.@lock PYJLMETHODS_LOCK begin + push!(PYJLMETHODS, f) + return length(PYJLMETHODS) + end end function _pyjl_isnull(o::C.PyPtr, ::C.PyPtr) @@ -58,12 +65,13 @@ function _pyjl_callmethod(o::C.PyPtr, args::C.PyPtr) @assert nargs > 0 num = C.PyLong_AsLongLong(C.PyTuple_GetItem(args, 0)) num == -1 && return C.PyNULL - f = PYJLMETHODS[num] + f = Base.@lock PYJLMETHODS_LOCK PYJLMETHODS[num] # this form gets defined in jlwrap/base.jl return _pyjl_callmethod(f, o, args, nargs)::C.PyPtr end const PYJLBUFCACHE = Dict{Ptr{Cvoid},Any}() +const PYJLBUFCACHE_LOCK = Threads.SpinLock() @kwdef struct PyBufferInfo{N} # data @@ -177,7 +185,9 @@ function _pyjl_get_buffer_impl( # internal cptr = Base.pointer_from_objref(c) - PYJLBUFCACHE[cptr] = c + Base.@lock PYJLBUFCACHE_LOCK begin + PYJLBUFCACHE[cptr] = c + end b.internal[] = cptr # obj @@ -195,7 +205,7 @@ function _pyjl_get_buffer(o::C.PyPtr, buf::Ptr{C.Py_buffer}, flags::Cint) C.Py_DecRef(num_) num == -1 && return Cint(-1) try - f = PYJLMETHODS[num] + f = Base.@lock PYJLMETHODS_LOCK PYJLMETHODS[num] x = PyJuliaValue_GetValue(o) return _pyjl_get_buffer_impl(o, buf, flags, x, f)::Cint catch exc @@ -209,7 +219,9 @@ function _pyjl_get_buffer(o::C.PyPtr, buf::Ptr{C.Py_buffer}, flags::Cint) end function _pyjl_release_buffer(xo::C.PyPtr, buf::Ptr{C.Py_buffer}) - delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[!]) + Base.@lock PYJLBUFCACHE_LOCK begin + delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[!]) + end nothing end @@ -339,22 +351,31 @@ end PyJuliaValue_IsNull(o) = Base.GC.@preserve o UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] == 0 -PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]] +PyJuliaValue_GetValue(o) = Base.GC.@preserve o begin + idx = UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] + Base.@lock PYJLVALUES_LOCK begin + PYJLVALUES[idx] + end +end PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin o = C.asptr(_o) idx = UnsafePtr{PyJuliaValueObject}(o).value[] if idx == 0 - if isempty(PYJLFREEVALUES) - push!(PYJLVALUES, v) - idx = length(PYJLVALUES) - else - idx = pop!(PYJLFREEVALUES) - PYJLVALUES[idx] = v + Base.@lock PYJLVALUES_LOCK begin + if isempty(PYJLFREEVALUES) + push!(PYJLVALUES, v) + idx = length(PYJLVALUES) + else + idx = pop!(PYJLFREEVALUES) + PYJLVALUES[idx] = v + end end UnsafePtr{PyJuliaValueObject}(o).value[] = idx else - PYJLVALUES[idx] = v + Base.@lock PYJLVALUES_LOCK begin + PYJLVALUES[idx] = v + end end nothing end diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 47ffb084..f8250c52 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -84,6 +84,7 @@ function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssi pybuiltins.NotImplementedError, "__jl_callmethod not implemented for this many arguments", ) + return C.PyNULL end return getptr(incref(ans)) catch exc From 129fda4ce30d9fa22437e05f62883c83bf70db28 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 16:16:20 +0100 Subject: [PATCH 02/14] fix: safer PYJLVALUES --- src/JlWrap/C.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 2cde6494..e4afd2ec 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -16,11 +16,13 @@ const PyJuliaBase_Type = Ref(C.PyNULL) # we store the actual julia values here # the `value` field of `PyJuliaValueObject` indexes into here -const PYJLVALUES = [] +const PYJLVALUES = IdDict{Int,Any}() # unused indices in PYJLVALUES const PYJLFREEVALUES = Int[] # Thread safety for PYJLVALUES and PYJLFREEVALUES const PYJLVALUES_LOCK = Threads.SpinLock() +# Track next available index +const PYJLVALUES_NEXT_IDX = Ref(1) function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) @@ -34,7 +36,7 @@ function _pyjl_dealloc(o::C.PyPtr) idx = UnsafePtr{PyJuliaValueObject}(o).value[] if idx != 0 Base.@lock PYJLVALUES_LOCK begin - PYJLVALUES[idx] = nothing + delete!(PYJLVALUES, idx) push!(PYJLFREEVALUES, idx) end end @@ -364,12 +366,12 @@ PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin if idx == 0 Base.@lock PYJLVALUES_LOCK begin if isempty(PYJLFREEVALUES) - push!(PYJLVALUES, v) - idx = length(PYJLVALUES) + idx = PYJLVALUES_NEXT_IDX[] + PYJLVALUES_NEXT_IDX[] += 1 else idx = pop!(PYJLFREEVALUES) - PYJLVALUES[idx] = v end + PYJLVALUES[idx] = v end UnsafePtr{PyJuliaValueObject}(o).value[] = idx else From 32c5b7a9e38565f549e4db9a0c0ac833a8727868 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 16:23:06 +0100 Subject: [PATCH 03/14] switch to Lockable --- pysrc/juliacall/juliapkg.json | 4 ++- src/Convert/Convert.jl | 1 + src/Convert/pyconvert.jl | 59 +++++++++++++++++----------------- src/Core/Core.jl | 2 +- src/Core/Py.jl | 11 +++---- src/JlWrap/C.jl | 60 ++++++++++++++--------------------- src/Utils/Utils.jl | 26 +++++++++++++++ 7 files changed, 89 insertions(+), 74 deletions(-) diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index b05353fc..27b08dd9 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -3,7 +3,9 @@ "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", - "version": "=0.9.25" + "version": "=0.9.25", + "path": "/Users/mcranmer/PermaDocuments/PythonCall.jl/", + "dev": true }, "OpenSSL_jll": { "uuid": "458c3c95-2e84-50aa-8efc-19380b2a3a95", diff --git a/src/Convert/Convert.jl b/src/Convert/Convert.jl index a97f1a48..688d022e 100644 --- a/src/Convert/Convert.jl +++ b/src/Convert/Convert.jl @@ -9,6 +9,7 @@ using ..Core using ..Core: C, Utils, + Lockable, @autopy, getptr, incref, diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index ac2750d7..a1e6e9b2 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -12,8 +12,8 @@ struct PyConvertRule priority::PyConvertPriority end -const PYCONVERT_RULES = Dict{String,Vector{PyConvertRule}}() -const PYCONVERT_EXTRATYPES = Py[] +const PYCONVERT_RULES = Lockable(Dict{String,Vector{PyConvertRule}}()) +const PYCONVERT_EXTRATYPES = Lockable(Py[]) """ pyconvert_add_rule(tname::String, T::Type, func::Function, priority::PyConvertPriority=PYCONVERT_PRIORITY_NORMAL) @@ -69,11 +69,11 @@ function pyconvert_add_rule( priority::PyConvertPriority = PYCONVERT_PRIORITY_NORMAL, ) @nospecialize type func - push!( - get!(Vector{PyConvertRule}, PYCONVERT_RULES, pytypename), + Base.@lock PYCONVERT_RULES push!( + get!(Vector{PyConvertRule}, PYCONVERT_RULES[], pytypename), PyConvertRule(type, func, priority), ) - Base.@lock PYCONVERT_RULES_CACHE_LOCK empty!.(values(PYCONVERT_RULES_CACHE)) + Base.@lock PYCONVERT_RULES_CACHE empty!.(values(PYCONVERT_RULES_CACHE[])) return end @@ -163,7 +163,7 @@ function _pyconvert_get_rules(pytype::Py) omro = collect(pytype.__mro__) basetypes = Py[pytype] basemros = Vector{Py}[omro] - for xtype in PYCONVERT_EXTRATYPES + Base.@lock PYCONVERT_EXTRATYPES for xtype in PYCONVERT_EXTRATYPES[] # find the topmost supertype of xbase = PyNULL for base in omro @@ -248,9 +248,9 @@ function _pyconvert_get_rules(pytype::Py) mro = String[x for xs in xmro for x in xs] # get corresponding rules - rules = PyConvertRule[ + rules = Base.@lock PYCONVERT_RULES PyConvertRule[ rule for tname in mro for - rule in get!(Vector{PyConvertRule}, PYCONVERT_RULES, tname) + rule in get!(Vector{PyConvertRule}, PYCONVERT_RULES[], tname) ] # order the rules by priority, then by original order @@ -261,11 +261,10 @@ function _pyconvert_get_rules(pytype::Py) return rules end -const PYCONVERT_PREFERRED_TYPE = Dict{Py,Type}() -const PYCONVERT_PREFERRED_TYPE_LOCK = Threads.SpinLock() +const PYCONVERT_PREFERRED_TYPE = Lockable(Dict{Py,Type}()) pyconvert_preferred_type(pytype::Py) = - Base.@lock PYCONVERT_PREFERRED_TYPE_LOCK get!(PYCONVERT_PREFERRED_TYPE, pytype) do + Base.@lock PYCONVERT_PREFERRED_TYPE get!(PYCONVERT_PREFERRED_TYPE[], pytype) do if pyissubclass(pytype, pybuiltins.int) Union{Int,BigInt} else @@ -308,11 +307,10 @@ end pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) -const PYCONVERT_RULES_CACHE = Dict{Type,Dict{C.PyPtr,Vector{Function}}}() -const PYCONVERT_RULES_CACHE_LOCK = Threads.SpinLock() +const PYCONVERT_RULES_CACHE = Lockable(Dict{Type,Dict{C.PyPtr,Vector{Function}}}()) @generated pyconvert_rules_cache(::Type{T}) where {T} = - Base.@lock PYCONVERT_RULES_CACHE_LOCK get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE, T) + Base.@lock PYCONVERT_RULES_CACHE get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE[], T) function pyconvert_rule_fast(::Type{T}, x::Py) where {T} if T isa Union @@ -353,12 +351,13 @@ function pytryconvert(::Type{T}, x_) where {T} # get rules from the cache # TODO: we should hold weak references and clear the cache if types get deleted tptr = C.Py_Type(x) - trules = pyconvert_rules_cache(T) - rules = Base.@lock PYCONVERT_RULES_CACHE_LOCK get!(trules, tptr) do - t = pynew(incref(tptr)) - ans = pyconvert_get_rules(T, t)::Vector{Function} - pydel!(t) - ans + rules = Base.@lock PYCONVERT_RULES_CACHE let trules = pyconvert_rules_cache(T) + get!(trules, tptr) do + t = pynew(incref(tptr)) + ans = pyconvert_get_rules(T, t)::Vector{Function} + pydel!(t) + ans + end end # apply the rules @@ -420,15 +419,17 @@ pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin end function init_pyconvert() - push!(PYCONVERT_EXTRATYPES, pyimport("io" => "IOBase")) - push!( - PYCONVERT_EXTRATYPES, - pyimport("numbers" => ("Number", "Complex", "Real", "Rational", "Integral"))..., - ) - push!( - PYCONVERT_EXTRATYPES, - pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))..., - ) + Base.@lock PYCONVERT_EXTRATYPES begin + push!(PYCONVERT_EXTRATYPES[], pyimport("io" => "IOBase")) + push!( + PYCONVERT_EXTRATYPES[], + pyimport("numbers" => ("Number", "Complex", "Real", "Rational", "Integral"))..., + ) + push!( + PYCONVERT_EXTRATYPES[], + pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))..., + ) + end priority = PYCONVERT_PRIORITY_CANONICAL pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority) diff --git a/src/Core/Core.jl b/src/Core/Core.jl index 886eaff9..7c95326b 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -11,7 +11,7 @@ const ROOT_DIR = dirname(dirname(@__DIR__)) using ..PythonCall: PythonCall # needed for docstring cross-refs using ..C: C using ..GC: GC -using ..Utils: Utils +using ..Utils: Utils, Lockable using Base: @propagate_inbounds, @kwdef using Dates: Date, diff --git a/src/Core/Py.jl b/src/Core/Py.jl index dff462be..b2f8a861 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -56,8 +56,7 @@ decref(x::Py) = Base.GC.@preserve x (decref(getptr(x)); x) Base.unsafe_convert(::Type{C.PyPtr}, x::Py) = getptr(x) -const PYNULL_CACHE = Py[] -const PYNULL_CACHE_LOCK = Threads.SpinLock() +const PYNULL_CACHE = Lockable(Py[]) """ pynew([ptr]) @@ -70,11 +69,11 @@ points at, i.e. the new `Py` object owns a reference. Note that NULL Python objects are not safe in the sense that most API functions will probably crash your Julia session if you pass a NULL argument. """ -pynew() = Base.@lock PYNULL_CACHE_LOCK begin - if isempty(PYNULL_CACHE) +pynew() = Base.@lock PYNULL_CACHE begin + if isempty(PYNULL_CACHE[]) Py(Val(:new), C.PyNULL) else - pop!(PYNULL_CACHE) + pop!(PYNULL_CACHE[]) end end @@ -121,7 +120,7 @@ function pydel!(x::Py) C.Py_DecRef(ptr) setptr!(x, C.PyNULL) end - Base.@lock PYNULL_CACHE_LOCK push!(PYNULL_CACHE, x) + Base.@lock PYNULL_CACHE push!(PYNULL_CACHE[], x) return end diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index e4afd2ec..5067b38d 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -1,7 +1,7 @@ module Cjl using ...C: C -using ...Utils: Utils +using ...Utils: Utils, Lockable using Base: @kwdef using UnsafePointers: UnsafePtr using Serialization: serialize, deserialize @@ -16,13 +16,7 @@ const PyJuliaBase_Type = Ref(C.PyNULL) # we store the actual julia values here # the `value` field of `PyJuliaValueObject` indexes into here -const PYJLVALUES = IdDict{Int,Any}() -# unused indices in PYJLVALUES -const PYJLFREEVALUES = Int[] -# Thread safety for PYJLVALUES and PYJLFREEVALUES -const PYJLVALUES_LOCK = Threads.SpinLock() -# Track next available index -const PYJLVALUES_NEXT_IDX = Ref(1) +const PYJLVALUES = Lockable((; values=IdDict{Int,Any}(), free=Int[], next=Ref(1))) function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) @@ -35,9 +29,9 @@ end function _pyjl_dealloc(o::C.PyPtr) idx = UnsafePtr{PyJuliaValueObject}(o).value[] if idx != 0 - Base.@lock PYJLVALUES_LOCK begin - delete!(PYJLVALUES, idx) - push!(PYJLFREEVALUES, idx) + Base.@lock PYJLVALUES begin + delete!(PYJLVALUES[].values, idx) + push!(PYJLVALUES[].free, idx) end end UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o) @@ -45,14 +39,13 @@ function _pyjl_dealloc(o::C.PyPtr) nothing end -const PYJLMETHODS = Vector{Any}() -const PYJLMETHODS_LOCK = Threads.SpinLock() +const PYJLMETHODS = Lockable([]) function PyJulia_MethodNum(f) @nospecialize f - Base.@lock PYJLMETHODS_LOCK begin - push!(PYJLMETHODS, f) - return length(PYJLMETHODS) + Base.@lock PYJLMETHODS begin + push!(PYJLMETHODS[], f) + return length(PYJLMETHODS[]) end end @@ -67,13 +60,12 @@ function _pyjl_callmethod(o::C.PyPtr, args::C.PyPtr) @assert nargs > 0 num = C.PyLong_AsLongLong(C.PyTuple_GetItem(args, 0)) num == -1 && return C.PyNULL - f = Base.@lock PYJLMETHODS_LOCK PYJLMETHODS[num] + f = Base.@lock PYJLMETHODS PYJLMETHODS[][num] # this form gets defined in jlwrap/base.jl return _pyjl_callmethod(f, o, args, nargs)::C.PyPtr end -const PYJLBUFCACHE = Dict{Ptr{Cvoid},Any}() -const PYJLBUFCACHE_LOCK = Threads.SpinLock() +const PYJLBUFCACHE = Lockable(Dict{Ptr{Cvoid},Any}()) @kwdef struct PyBufferInfo{N} # data @@ -187,9 +179,7 @@ function _pyjl_get_buffer_impl( # internal cptr = Base.pointer_from_objref(c) - Base.@lock PYJLBUFCACHE_LOCK begin - PYJLBUFCACHE[cptr] = c - end + Base.@lock PYJLBUFCACHE PYJLBUFCACHE[][cptr] = c b.internal[] = cptr # obj @@ -207,7 +197,7 @@ function _pyjl_get_buffer(o::C.PyPtr, buf::Ptr{C.Py_buffer}, flags::Cint) C.Py_DecRef(num_) num == -1 && return Cint(-1) try - f = Base.@lock PYJLMETHODS_LOCK PYJLMETHODS[num] + f = Base.@lock PYJLMETHODS PYJLMETHODS[][num] x = PyJuliaValue_GetValue(o) return _pyjl_get_buffer_impl(o, buf, flags, x, f)::Cint catch exc @@ -221,9 +211,7 @@ function _pyjl_get_buffer(o::C.PyPtr, buf::Ptr{C.Py_buffer}, flags::Cint) end function _pyjl_release_buffer(xo::C.PyPtr, buf::Ptr{C.Py_buffer}) - Base.@lock PYJLBUFCACHE_LOCK begin - delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[!]) - end + Base.@lock PYJLBUFCACHE delete!(PYJLBUFCACHE[], UnsafePtr(buf).internal[!]) nothing end @@ -355,28 +343,26 @@ PyJuliaValue_IsNull(o) = Base.GC.@preserve o UnsafePtr{PyJuliaValueObject}(C.asp PyJuliaValue_GetValue(o) = Base.GC.@preserve o begin idx = UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] - Base.@lock PYJLVALUES_LOCK begin - PYJLVALUES[idx] - end + Base.@lock PYJLVALUES PYJLVALUES[].values[idx] end PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin o = C.asptr(_o) idx = UnsafePtr{PyJuliaValueObject}(o).value[] if idx == 0 - Base.@lock PYJLVALUES_LOCK begin - if isempty(PYJLFREEVALUES) - idx = PYJLVALUES_NEXT_IDX[] - PYJLVALUES_NEXT_IDX[] += 1 + Base.@lock PYJLVALUES begin + if isempty(PYJLVALUES[].free) + idx = PYJLVALUES[].next[] + PYJLVALUES[].next[] += 1 else - idx = pop!(PYJLFREEVALUES) + idx = pop!(PYJLVALUES[].free) end - PYJLVALUES[idx] = v + PYJLVALUES[].values[idx] = v end UnsafePtr{PyJuliaValueObject}(o).value[] = idx else - Base.@lock PYJLVALUES_LOCK begin - PYJLVALUES[idx] = v + Base.@lock PYJLVALUES begin + PYJLVALUES[].values[idx] = v end end nothing diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 6e7b3f4d..27490a14 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -308,4 +308,30 @@ function Base.iterate(x::StaticString{UInt32,N}, i::Int = 1) where {N} end end +@static if !isdefined(Base, :Lockable) + """ + Compat for `Base.Lockable` (introduced in Julia 1.11) + """ + struct Lockable{T, L<:AbstractLock} + value::T + lock::L + end + + Lockable(value) = Lockable(value, ReentrantLock()) + + function Base.lock(f, l::Lockable) + lock(l.lock) do + f(l.value) + end + end + + Base.lock(l::Lockable) = lock(l.lock) + Base.trylock(l::Lockable) = trylock(l.lock) + Base.unlock(l::Lockable) = unlock(l.lock) + Base.islocked(l::Lockable) = islocked(l.lock) + Base.getindex(l::Lockable) = (@assert islocked(l); l.value) +else + const Lockable = Base.Lockable +end + end From f48c80ade110c53cd80f72fd5ec28273c4a00d77 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 16:23:19 +0100 Subject: [PATCH 04/14] better names --- src/JlWrap/C.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 5067b38d..5a91d330 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -16,7 +16,7 @@ const PyJuliaBase_Type = Ref(C.PyNULL) # we store the actual julia values here # the `value` field of `PyJuliaValueObject` indexes into here -const PYJLVALUES = Lockable((; values=IdDict{Int,Any}(), free=Int[], next=Ref(1))) +const PYJLVALUES = Lockable((; values=IdDict{Int,Any}(), free_slots=Int[], next_slot=Ref(1))) function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) @@ -31,7 +31,7 @@ function _pyjl_dealloc(o::C.PyPtr) if idx != 0 Base.@lock PYJLVALUES begin delete!(PYJLVALUES[].values, idx) - push!(PYJLVALUES[].free, idx) + push!(PYJLVALUES[].free_slots, idx) end end UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o) @@ -351,11 +351,11 @@ PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin idx = UnsafePtr{PyJuliaValueObject}(o).value[] if idx == 0 Base.@lock PYJLVALUES begin - if isempty(PYJLVALUES[].free) - idx = PYJLVALUES[].next[] - PYJLVALUES[].next[] += 1 + if isempty(PYJLVALUES[].free_slots) + idx = PYJLVALUES[].next_slot[] + PYJLVALUES[].next_slot[] += 1 else - idx = pop!(PYJLVALUES[].free) + idx = pop!(PYJLVALUES[].free_slots) end PYJLVALUES[].values[idx] = v end From ea68cd6acca0858f597472ae81c83738b88690db Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 16:32:53 +0100 Subject: [PATCH 05/14] safer module access --- src/Core/builtins.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index ec7eb10b..62d58309 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -1206,7 +1206,7 @@ export pyfraction ### eval/exec -const MODULE_GLOBALS = Dict{Module,Py}() +const MODULE_GLOBALS = Lockable(Dict{Module,Py}()) function _pyeval_args(code, globals, locals) if code isa AbstractString @@ -1217,7 +1217,7 @@ function _pyeval_args(code, globals, locals) throw(ArgumentError("code must be a string or Python code")) end if globals isa Module - globals_ = get!(pydict, MODULE_GLOBALS, globals) + globals_ = Base.@lock MODULE_GLOBALS get!(pydict, MODULE_GLOBALS[], globals) elseif ispy(globals) globals_ = globals else From e5a5fcfa4b5ad5ee8e899eb571928eebb74338d5 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 16:34:46 +0100 Subject: [PATCH 06/14] undo changes to juliapkg.json --- pysrc/juliacall/juliapkg.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index 27b08dd9..b05353fc 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -3,9 +3,7 @@ "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", - "version": "=0.9.25", - "path": "/Users/mcranmer/PermaDocuments/PythonCall.jl/", - "dev": true + "version": "=0.9.25" }, "OpenSSL_jll": { "uuid": "458c3c95-2e84-50aa-8efc-19380b2a3a95", From 8d82e2a20b0854ce38d6e79ead13e56b905d23f7 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 16:40:22 +0100 Subject: [PATCH 07/14] AbstractLock undefined on 1.10 --- src/Utils/Utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 27490a14..c1f39b13 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -312,11 +312,11 @@ end """ Compat for `Base.Lockable` (introduced in Julia 1.11) """ - struct Lockable{T, L<:AbstractLock} + struct Lockable{T,L} value::T lock::L end - + Lockable(value) = Lockable(value, ReentrantLock()) function Base.lock(f, l::Lockable) From 53a8e80a65c11ef46850a301e11695fb4d0dddd9 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 17:37:08 +0100 Subject: [PATCH 08/14] avoid locking in generated function preamble --- src/Convert/pyconvert.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index a1e6e9b2..b7a20bb5 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -309,8 +309,12 @@ pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) const PYCONVERT_RULES_CACHE = Lockable(Dict{Type,Dict{C.PyPtr,Vector{Function}}}()) -@generated pyconvert_rules_cache(::Type{T}) where {T} = - Base.@lock PYCONVERT_RULES_CACHE get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE[], T) +function pyconvert_rules_cache(::Type{T}) where {T} + Base.@lock PYCONVERT_RULES_CACHE _pyconvert_rules_cache!(T) +end +@generated function _pyconvert_rules_cache!(::Type{T}) where {T} + get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE[], T) +end function pyconvert_rule_fast(::Type{T}, x::Py) where {T} if T isa Union From 82f7f4ad1b702cc9ec744565413987d77f580e9a Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 18:16:55 +0100 Subject: [PATCH 09/14] avoid generated function caching --- src/Convert/pyconvert.jl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index b7a20bb5..9aa3591e 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -307,13 +307,14 @@ end pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) -const PYCONVERT_RULES_CACHE = Lockable(Dict{Type,Dict{C.PyPtr,Vector{Function}}}()) +const PYCONVERT_RULES_CACHE = Lockable(IdDict{Any,Dict{C.PyPtr,Vector{Function}}}()) function pyconvert_rules_cache(::Type{T}) where {T} - Base.@lock PYCONVERT_RULES_CACHE _pyconvert_rules_cache!(T) -end -@generated function _pyconvert_rules_cache!(::Type{T}) where {T} - get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE[], T) + Base.@lock PYCONVERT_RULES_CACHE get!( + Dict{C.PyPtr,Vector{Function}}, + PYCONVERT_RULES_CACHE[], + T, + ) end function pyconvert_rule_fast(::Type{T}, x::Py) where {T} From 551bf3303744c9142fd8f3dd58bbe3511590820c Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sun, 6 Jul 2025 18:33:27 +0100 Subject: [PATCH 10/14] comment out unused methods; but leave in case needed --- src/Utils/Utils.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index c1f39b13..972b6216 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -319,14 +319,14 @@ end Lockable(value) = Lockable(value, ReentrantLock()) - function Base.lock(f, l::Lockable) - lock(l.lock) do - f(l.value) - end - end + # function Base.lock(f, l::Lockable) + # lock(l.lock) do + # f(l.value) + # end + # end Base.lock(l::Lockable) = lock(l.lock) - Base.trylock(l::Lockable) = trylock(l.lock) + # Base.trylock(l::Lockable) = trylock(l.lock) Base.unlock(l::Lockable) = unlock(l.lock) Base.islocked(l::Lockable) = islocked(l.lock) Base.getindex(l::Lockable) = (@assert islocked(l); l.value) From c05a22f0a3707d9d01746b1f6c1d450bb2d283a6 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 7 Jul 2025 16:43:27 +0100 Subject: [PATCH 11/14] add global locking utility --- src/Convert/Convert.jl | 1 + src/Convert/pyconvert.jl | 8 ++++---- src/Core/Core.jl | 2 +- src/Core/Py.jl | 2 +- src/Core/builtins.jl | 2 +- src/JlWrap/C.jl | 8 ++++---- src/Utils/Utils.jl | 2 ++ 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Convert/Convert.jl b/src/Convert/Convert.jl index 688d022e..7842d01d 100644 --- a/src/Convert/Convert.jl +++ b/src/Convert/Convert.jl @@ -10,6 +10,7 @@ using ..Core: C, Utils, Lockable, + GLOBAL_LOCK, @autopy, getptr, incref, diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 9aa3591e..d0b7c124 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -12,8 +12,8 @@ struct PyConvertRule priority::PyConvertPriority end -const PYCONVERT_RULES = Lockable(Dict{String,Vector{PyConvertRule}}()) -const PYCONVERT_EXTRATYPES = Lockable(Py[]) +const PYCONVERT_RULES = Lockable(Dict{String,Vector{PyConvertRule}}(), GLOBAL_LOCK) +const PYCONVERT_EXTRATYPES = Lockable(Py[], GLOBAL_LOCK) """ pyconvert_add_rule(tname::String, T::Type, func::Function, priority::PyConvertPriority=PYCONVERT_PRIORITY_NORMAL) @@ -261,7 +261,7 @@ function _pyconvert_get_rules(pytype::Py) return rules end -const PYCONVERT_PREFERRED_TYPE = Lockable(Dict{Py,Type}()) +const PYCONVERT_PREFERRED_TYPE = Lockable(Dict{Py,Type}(), GLOBAL_LOCK) pyconvert_preferred_type(pytype::Py) = Base.@lock PYCONVERT_PREFERRED_TYPE get!(PYCONVERT_PREFERRED_TYPE[], pytype) do @@ -307,7 +307,7 @@ end pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) -const PYCONVERT_RULES_CACHE = Lockable(IdDict{Any,Dict{C.PyPtr,Vector{Function}}}()) +const PYCONVERT_RULES_CACHE = Lockable(IdDict{Any,Dict{C.PyPtr,Vector{Function}}}(), GLOBAL_LOCK) function pyconvert_rules_cache(::Type{T}) where {T} Base.@lock PYCONVERT_RULES_CACHE get!( diff --git a/src/Core/Core.jl b/src/Core/Core.jl index 7c95326b..f742bbcf 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -11,7 +11,7 @@ const ROOT_DIR = dirname(dirname(@__DIR__)) using ..PythonCall: PythonCall # needed for docstring cross-refs using ..C: C using ..GC: GC -using ..Utils: Utils, Lockable +using ..Utils: Utils, Lockable, GLOBAL_LOCK using Base: @propagate_inbounds, @kwdef using Dates: Date, diff --git a/src/Core/Py.jl b/src/Core/Py.jl index b2f8a861..7d048047 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -56,7 +56,7 @@ decref(x::Py) = Base.GC.@preserve x (decref(getptr(x)); x) Base.unsafe_convert(::Type{C.PyPtr}, x::Py) = getptr(x) -const PYNULL_CACHE = Lockable(Py[]) +const PYNULL_CACHE = Lockable(Py[], GLOBAL_LOCK) """ pynew([ptr]) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 62d58309..d75f14e3 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -1206,7 +1206,7 @@ export pyfraction ### eval/exec -const MODULE_GLOBALS = Lockable(Dict{Module,Py}()) +const MODULE_GLOBALS = Lockable(Dict{Module,Py}(), GLOBAL_LOCK) function _pyeval_args(code, globals, locals) if code isa AbstractString diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 5a91d330..b8d8e0e9 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -1,7 +1,7 @@ module Cjl using ...C: C -using ...Utils: Utils, Lockable +using ...Utils: Utils, Lockable, GLOBAL_LOCK using Base: @kwdef using UnsafePointers: UnsafePtr using Serialization: serialize, deserialize @@ -16,7 +16,7 @@ const PyJuliaBase_Type = Ref(C.PyNULL) # we store the actual julia values here # the `value` field of `PyJuliaValueObject` indexes into here -const PYJLVALUES = Lockable((; values=IdDict{Int,Any}(), free_slots=Int[], next_slot=Ref(1))) +const PYJLVALUES = Lockable((; values=IdDict{Int,Any}(), free_slots=Int[], next_slot=Ref(1)), GLOBAL_LOCK) function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) @@ -39,7 +39,7 @@ function _pyjl_dealloc(o::C.PyPtr) nothing end -const PYJLMETHODS = Lockable([]) +const PYJLMETHODS = Lockable([], GLOBAL_LOCK) function PyJulia_MethodNum(f) @nospecialize f @@ -65,7 +65,7 @@ function _pyjl_callmethod(o::C.PyPtr, args::C.PyPtr) return _pyjl_callmethod(f, o, args, nargs)::C.PyPtr end -const PYJLBUFCACHE = Lockable(Dict{Ptr{Cvoid},Any}()) +const PYJLBUFCACHE = Lockable(Dict{Ptr{Cvoid},Any}(), GLOBAL_LOCK) @kwdef struct PyBufferInfo{N} # data diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 972b6216..2a424659 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -334,4 +334,6 @@ else const Lockable = Base.Lockable end +const GLOBAL_LOCK = ReentrantLock() + end From 4a772eb905fa93597e5ee2835afc5eb5fd64b09c Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 7 Jul 2025 17:23:53 +0100 Subject: [PATCH 12/14] remove check that GC queue must not be empty --- test/GC.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/GC.jl b/test/GC.jl index 475c547c..f3a5d56e 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -5,9 +5,6 @@ finalize(obj) end end - Threads.nthreads() > 1 && - VERSION >= v"1.10.0-" && - @test !isempty(PythonCall.GC.QUEUE.items) PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end @@ -19,9 +16,6 @@ end finalize(obj) end end - Threads.nthreads() > 1 && - VERSION >= v"1.10.0-" && - @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end From 95d9e4197c95021ad896c5c73ad97422595bf621 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 7 Jul 2025 17:30:35 +0100 Subject: [PATCH 13/14] Revert "remove check that GC queue must not be empty" This reverts commit 4a772eb905fa93597e5ee2835afc5eb5fd64b09c. --- test/GC.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/GC.jl b/test/GC.jl index f3a5d56e..475c547c 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -5,6 +5,9 @@ finalize(obj) end end + Threads.nthreads() > 1 && + VERSION >= v"1.10.0-" && + @test !isempty(PythonCall.GC.QUEUE.items) PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end @@ -16,6 +19,9 @@ end finalize(obj) end end + Threads.nthreads() > 1 && + VERSION >= v"1.10.0-" && + @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end From d798710eb155ca1b1089778adee4863e063aeaaa Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 7 Jul 2025 17:30:42 +0100 Subject: [PATCH 14/14] Revert "add global locking utility" This reverts commit c05a22f0a3707d9d01746b1f6c1d450bb2d283a6. --- src/Convert/Convert.jl | 1 - src/Convert/pyconvert.jl | 8 ++++---- src/Core/Core.jl | 2 +- src/Core/Py.jl | 2 +- src/Core/builtins.jl | 2 +- src/JlWrap/C.jl | 8 ++++---- src/Utils/Utils.jl | 2 -- 7 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/Convert/Convert.jl b/src/Convert/Convert.jl index 7842d01d..688d022e 100644 --- a/src/Convert/Convert.jl +++ b/src/Convert/Convert.jl @@ -10,7 +10,6 @@ using ..Core: C, Utils, Lockable, - GLOBAL_LOCK, @autopy, getptr, incref, diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index d0b7c124..9aa3591e 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -12,8 +12,8 @@ struct PyConvertRule priority::PyConvertPriority end -const PYCONVERT_RULES = Lockable(Dict{String,Vector{PyConvertRule}}(), GLOBAL_LOCK) -const PYCONVERT_EXTRATYPES = Lockable(Py[], GLOBAL_LOCK) +const PYCONVERT_RULES = Lockable(Dict{String,Vector{PyConvertRule}}()) +const PYCONVERT_EXTRATYPES = Lockable(Py[]) """ pyconvert_add_rule(tname::String, T::Type, func::Function, priority::PyConvertPriority=PYCONVERT_PRIORITY_NORMAL) @@ -261,7 +261,7 @@ function _pyconvert_get_rules(pytype::Py) return rules end -const PYCONVERT_PREFERRED_TYPE = Lockable(Dict{Py,Type}(), GLOBAL_LOCK) +const PYCONVERT_PREFERRED_TYPE = Lockable(Dict{Py,Type}()) pyconvert_preferred_type(pytype::Py) = Base.@lock PYCONVERT_PREFERRED_TYPE get!(PYCONVERT_PREFERRED_TYPE[], pytype) do @@ -307,7 +307,7 @@ end pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) -const PYCONVERT_RULES_CACHE = Lockable(IdDict{Any,Dict{C.PyPtr,Vector{Function}}}(), GLOBAL_LOCK) +const PYCONVERT_RULES_CACHE = Lockable(IdDict{Any,Dict{C.PyPtr,Vector{Function}}}()) function pyconvert_rules_cache(::Type{T}) where {T} Base.@lock PYCONVERT_RULES_CACHE get!( diff --git a/src/Core/Core.jl b/src/Core/Core.jl index f742bbcf..7c95326b 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -11,7 +11,7 @@ const ROOT_DIR = dirname(dirname(@__DIR__)) using ..PythonCall: PythonCall # needed for docstring cross-refs using ..C: C using ..GC: GC -using ..Utils: Utils, Lockable, GLOBAL_LOCK +using ..Utils: Utils, Lockable using Base: @propagate_inbounds, @kwdef using Dates: Date, diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 7d048047..b2f8a861 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -56,7 +56,7 @@ decref(x::Py) = Base.GC.@preserve x (decref(getptr(x)); x) Base.unsafe_convert(::Type{C.PyPtr}, x::Py) = getptr(x) -const PYNULL_CACHE = Lockable(Py[], GLOBAL_LOCK) +const PYNULL_CACHE = Lockable(Py[]) """ pynew([ptr]) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index d75f14e3..62d58309 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -1206,7 +1206,7 @@ export pyfraction ### eval/exec -const MODULE_GLOBALS = Lockable(Dict{Module,Py}(), GLOBAL_LOCK) +const MODULE_GLOBALS = Lockable(Dict{Module,Py}()) function _pyeval_args(code, globals, locals) if code isa AbstractString diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index b8d8e0e9..5a91d330 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -1,7 +1,7 @@ module Cjl using ...C: C -using ...Utils: Utils, Lockable, GLOBAL_LOCK +using ...Utils: Utils, Lockable using Base: @kwdef using UnsafePointers: UnsafePtr using Serialization: serialize, deserialize @@ -16,7 +16,7 @@ const PyJuliaBase_Type = Ref(C.PyNULL) # we store the actual julia values here # the `value` field of `PyJuliaValueObject` indexes into here -const PYJLVALUES = Lockable((; values=IdDict{Int,Any}(), free_slots=Int[], next_slot=Ref(1)), GLOBAL_LOCK) +const PYJLVALUES = Lockable((; values=IdDict{Int,Any}(), free_slots=Int[], next_slot=Ref(1))) function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) @@ -39,7 +39,7 @@ function _pyjl_dealloc(o::C.PyPtr) nothing end -const PYJLMETHODS = Lockable([], GLOBAL_LOCK) +const PYJLMETHODS = Lockable([]) function PyJulia_MethodNum(f) @nospecialize f @@ -65,7 +65,7 @@ function _pyjl_callmethod(o::C.PyPtr, args::C.PyPtr) return _pyjl_callmethod(f, o, args, nargs)::C.PyPtr end -const PYJLBUFCACHE = Lockable(Dict{Ptr{Cvoid},Any}(), GLOBAL_LOCK) +const PYJLBUFCACHE = Lockable(Dict{Ptr{Cvoid},Any}()) @kwdef struct PyBufferInfo{N} # data diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 2a424659..972b6216 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -334,6 +334,4 @@ else const Lockable = Base.Lockable end -const GLOBAL_LOCK = ReentrantLock() - end