Merge #1226

1226: [RFC] Allow Pkg.test to pass command-line arguments to test/runtests.jl r=fredrikekre a=DilumAluthge

Fixes #518 
Fixes #981 
Fixes #1241   



Co-authored-by: Dilum Aluthge <dilum@aluthge.com>

Author: bors[bot]

src/API.jl

@@ -229,7 +229,11 @@ test(pkg::Union{AbstractString, PackageSpec}; kwargs...) = test([pkg]; kwargs...
 test(pkgs::Vector{<:AbstractString}; kwargs...)          = test([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
 test(pkgs::Vector{PackageSpec}; kwargs...)               = test(Context(), pkgs; kwargs...)
 function test(ctx::Context, pkgs::Vector{PackageSpec};
-              coverage=false, test_fn=nothing, kwargs...)
+              coverage=false, test_fn=nothing,
+              julia_args::AbstractVector{<:Base.AbstractCmd}=Cmd[],
+              test_args::AbstractVector{<:AbstractString}=String[], kwargs...)
+    julia_args = convert(Vector{Cmd}, julia_args)
+    test_args = convert(Vector{String}, test_args)
     pkgs = deepcopy(pkgs) # deepcopy for avoid mutating PackageSpec members
     Context!(ctx; kwargs...)
     ctx.preview && preview_info()
@@ -242,7 +246,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
         manifest_resolve!(ctx.env, pkgs)
         ensure_resolved(ctx.env, pkgs)
     end
-    Operations.test(ctx, pkgs; coverage=coverage, test_fn=test_fn)
+    Operations.test(ctx, pkgs; coverage=coverage, test_fn=test_fn, julia_args=julia_args, test_args=test_args)
     return
 end
 

src/Operations.jl

@@ -1142,10 +1142,15 @@ function free(ctx::Context, pkgs::Vector{PackageSpec})
     end
 end
 
-function gen_test_code(testfile::String; coverage=false)
+function gen_test_code(testfile::String;
+        coverage=false,
+        julia_args::Vector{Cmd}=Cmd[],
+        test_args::Vector{String}=String[])
+    julia_args_str::Vector{String} = vcat([x.exec for x in julia_args]...)
     code = """
         $(Base.load_path_setup_code(false))
         cd($(repr(dirname(testfile))))
+        append!(empty!(ARGS), $(repr(test_args)))
         include($(repr(testfile)))
         """
     return ```
@@ -1157,7 +1162,8 @@ function gen_test_code(testfile::String; coverage=false)
         --inline=$(Bool(Base.JLOptions().can_inline) ? "yes" : "no")
         --startup-file=$(Base.JLOptions().startupfile == 1 ? "yes" : "no")
         --track-allocation=$(("none", "user", "all")[Base.JLOptions().malloc_log + 1])
-        --eval $code
+        $(julia_args_str)
+        --eval $(code)
     ```
 end
 
@@ -1240,7 +1246,10 @@ end
 
 testdir(source_path::String) = joinpath(source_path, "test")
 testfile(source_path::String) = joinpath(testdir(source_path), "runtests.jl")
-function test(ctx::Context, pkgs::Vector{PackageSpec}; coverage=false, test_fn=nothing)
+function test(ctx::Context, pkgs::Vector{PackageSpec};
+        coverage=false, test_fn=nothing,
+        julia_args::Vector{Cmd}=Cmd[],
+        test_args::Vector{String}=String[])
     ctx.preview || Pkg.instantiate(ctx)
 
     # load manifest data
@@ -1273,7 +1282,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; coverage=false, test_fn=n
     for (pkg, source_path) in zip(pkgs, source_paths)
         if !isfile(projectfile_path(testdir(source_path)))
             backwards_compatibility_for_test(ctx, pkg, testfile(source_path),
-                                             pkgs_errored, coverage)
+                                             pkgs_errored, coverage; julia_args=julia_args, test_args=test_args)
             continue
         end
 
@@ -1287,7 +1296,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; coverage=false, test_fn=n
             test_fn !== nothing && test_fn()
             Display.status(Context(), mode=PKGMODE_PROJECT)
             try
-                run(gen_test_code(testfile(source_path); coverage=coverage))
+                run(gen_test_code(testfile(source_path); coverage=coverage, julia_args=julia_args, test_args=test_args))
                 printpkgstyle(ctx, :Testing, pkg.name * " tests passed ")
             catch err
                 push!(pkgs_errored, pkg.name)

src/Pkg.jl

@@ -137,11 +137,18 @@ See also [`PackageSpec`](@ref), [`PackageMode`](@ref), [`UpgradeLevel`](@ref).
 """
 const update = API.up
 
-
 """
-    Pkg.test(; coverage::Bool=false)
-    Pkg.test(pkg::Union{String, Vector{String}; coverage::Bool=false)
-    Pkg.test(pkgs::Union{PackageSpec, Vector{PackageSpec}}; coverage::Bool=false)
+    Pkg.test(; kwargs...)
+    Pkg.test(pkg::Union{String, Vector{String}; kwargs...)
+    Pkg.test(pkgs::Union{PackageSpec, Vector{PackageSpec}}; kwargs...)
+
+**Keyword arguments:**
+  - `coverage::Bool=false`: enable or disable generation of coverage statistics.
+  - `julia_args`: vector of `Cmd` passed to the test process.
+  - `test_args`: vector of test arguments (`ARGS`) available in the test process.
+
+!!! compat "Julia 1.3"
+    `julia_args` and `test_args` requires at least Julia 1.3.
 
 Run the tests for package `pkg`, or for the current project (which thus needs to be a package) if no
 positional argument is given to `Pkg.test`. A package is tested by running its
@@ -162,9 +169,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 test = ["Test"]
 ```
 
-Coverage statistics for the packages may be generated by
-passing `coverage=true`. The default behavior is to not run coverage.
-
 The tests are executed in a new process with `check-bounds=yes` and by default `startup-file=no`.
 If using the startup file (`~/.julia/config/startup.jl`) is desired, start julia with `--startup-file=yes`.
 Inlining of functions during testing can be disabled (for better coverage accuracy)

src/backwards_compatible_isolation.jl

@@ -572,7 +572,7 @@ end
 
 function backwards_compatibility_for_test(
     ctx::Context, pkg::PackageSpec, testfile::String, pkgs_errored::Vector{String},
-    coverage
+    coverage; julia_args=Cmd[], test_args=String[]
 )
     printpkgstyle(ctx, :Testing, pkg.name)
     if ctx.preview
@@ -582,8 +582,10 @@ function backwards_compatibility_for_test(
     code = """
         $(Base.load_path_setup_code(false))
         cd($(repr(dirname(testfile))))
+        append!(empty!(ARGS), $(repr(test_args)))
         include($(repr(testfile)))
         """
+    julia_args_str = vcat([x.exec for x in julia_args]...)
     cmd = ```
         $(Base.julia_cmd())
         --code-coverage=$(coverage ? "user" : "none")
@@ -593,6 +595,7 @@ function backwards_compatibility_for_test(
         --inline=$(Bool(Base.JLOptions().can_inline) ? "yes" : "no")
         --startup-file=$(Base.JLOptions().startupfile == 1 ? "yes" : "no")
         --track-allocation=$(("none", "user", "all")[Base.JLOptions().malloc_log + 1])
+        $(julia_args_str)
         --eval $code
     ```
     run_test = () -> begin

test/api.jl

@@ -256,5 +256,16 @@ end
     end end
 end
 
+@testset "Pkg.test" begin
+    temp_pkg_dir() do tmp
+        copy_test_package(tmp, "TestArguments")
+        Pkg.activate(joinpath(tmp, "TestArguments"))
+        # test the old code path (no test/Project.toml)
+        Pkg.test("TestArguments"; test_args=["a", "b"], julia_args = [`--quiet --check-bounds=no`])
+        # test new code path
+        touch(joinpath(tmp, "TestArguments", "test", "Project.toml"))
+        Pkg.test("TestArguments"; test_args=["a", "b"], julia_args = [`--quiet --check-bounds=no`])
+    end
+end
 
 end # module APITests

test/test_packages/TestArguments/Project.toml

@@ -0,0 +1,3 @@
+name = "TestArguments"
+uuid = "265b0eca-b78c-42af-9929-ddebf847c026"
+version = "0.1.0"

test/test_packages/TestArguments/src/TestArguments.jl

@@ -0,0 +1,3 @@
+module TestArguments
+
+end # module

test/test_packages/TestArguments/test/runtests.jl

@@ -0,0 +1,3 @@
+@assert ARGS == ["a", "b"]
+@assert Base.JLOptions().quiet == 1 # --quiet
+@assert Base.JLOptions().check_bounds == 2 # overriden default when testing