From 3c7fff90294ec326aadb1064542169987d0bacda Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Thu, 12 Dec 2019 11:50:04 +0900 Subject: [PATCH 1/9] Create test_distribute.jl --- test/test_distribute.jl | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/test_distribute.jl diff --git a/test/test_distribute.jl b/test/test_distribute.jl new file mode 100644 index 0000000..e0b1dc7 --- /dev/null +++ b/test/test_distribute.jl @@ -0,0 +1,23 @@ +using Pkg, Test, DistStat + +type=[Float64,Float32] + +if haskey(Pkg.installed(), "CuArrays") + using CuArrays + ArrayType = CuArray +else + ArrayType = Array +end + +for T in type + + data =ArrayType{T}(reshape(collect(1:42),6,7)) + data_dist1 = distribute(data) + data_dist2 = distribute(ArrayType{T}(transpose(data))) + cols1=data_dist1.partitioning[DistStat.Rank()+1][2] + cols2=data_dist2.partitioning[DistStat.Rank()+1][2] + + println(@test data_dist1.localarray==data[:,cols1]) + println(@test data_dist2.localarray==(ArrayType{T}(transpose(data)))[:,cols2]) + +end From bd097a59c44ac7d8c3a3485fb3b66259df72cfe2 Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Thu, 12 Dec 2019 14:18:04 +0900 Subject: [PATCH 2/9] Create test_aux.jl --- test/test_aux.jl | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 test/test_aux.jl diff --git a/test/test_aux.jl b/test/test_aux.jl new file mode 100644 index 0000000..150734c --- /dev/null +++ b/test/test_aux.jl @@ -0,0 +1,34 @@ +using DistStat, Random, Test, Pkg + +type=[Float32,Float64] + +if haskey(Pkg.installed(), "CuArrays") + using CuArrays + ArrayType = CuArray +else + ArrayType = Array +end + +for T in type + A=ArrayType{T}(undef,7,10) + A_dist=distribute(A) + fill!(A, 1.0) + fill!(A_dist,1.0) + cols1=A_dist.partitioning[DistStat.Rank()+1][2] + + println(@test isapprox(A_dist.localarray,A[:,cols1])) + + B=reshape(collect(1:70), 7, 10) + B_dist=distribute(B) + cols2=B_dist.partitioning[DistStat.Rank()+1][2] + + println(@test isapprox(B_dist.localarray,B[:,cols2])) + + C_dist = MPIArray{T, 2, ArrayType}(undef, 7, 9) + + C_dist1=randn!(C_dist; seed=0) + C_dist2=randn!(C_dist; seed=0, common_init=true) + + println(@test C_dist1==C_dist2) + +end From 08131fea6c66e260e55dbd5dc20c241b5940254a Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Wed, 18 Dec 2019 12:57:39 +0900 Subject: [PATCH 3/9] Update test_distribute.jl --- test/test_distribute.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_distribute.jl b/test/test_distribute.jl index e0b1dc7..fe64e1f 100644 --- a/test/test_distribute.jl +++ b/test/test_distribute.jl @@ -2,7 +2,7 @@ using Pkg, Test, DistStat type=[Float64,Float32] -if haskey(Pkg.installed(), "CuArrays") +if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray" using CuArrays ArrayType = CuArray else From 5aadd206e0685c74b7b055dd99a85a8b8c2c9bb7 Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Wed, 18 Dec 2019 13:38:12 +0900 Subject: [PATCH 4/9] Update test_aux.jl --- test/test_aux.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test_aux.jl b/test/test_aux.jl index 150734c..d16f4ca 100644 --- a/test/test_aux.jl +++ b/test/test_aux.jl @@ -2,7 +2,7 @@ using DistStat, Random, Test, Pkg type=[Float32,Float64] -if haskey(Pkg.installed(), "CuArrays") +if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray" using CuArrays ArrayType = CuArray else @@ -25,10 +25,12 @@ for T in type println(@test isapprox(B_dist.localarray,B[:,cols2])) C_dist = MPIArray{T, 2, ArrayType}(undef, 7, 9) + cols3=C_dist.partitioning[DistStat.Rank()+1][2] + randn!(C_dist; seed=0,common_init=true) - C_dist1=randn!(C_dist; seed=0) - C_dist2=randn!(C_dist; seed=0, common_init=true) - - println(@test C_dist1==C_dist2) + C=ArrayType{T}(undef,size(C_dist)) + Random.seed!(0) + randn!(C) + println(@test isapprox(C_dist.localarray,C[:,cols3])) end From 4e0c09ffe8a65dbb91b1df71fd273605004b73fd Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Wed, 18 Dec 2019 15:08:42 +0900 Subject: [PATCH 5/9] Create test_dot.jl --- test/test_dot.jl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 test/test_dot.jl diff --git a/test/test_dot.jl b/test/test_dot.jl new file mode 100644 index 0000000..fce51d3 --- /dev/null +++ b/test/test_dot.jl @@ -0,0 +1,24 @@ +using DistStat, Random, Test, LinearAlgebra + +type=[Float32,Float64] + +if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray" + using CuArrays + ArrayType = CuArray +else + ArrayType = Array +end + +for T in type + A=ArrayType{T}(reshape(collect(1:36),4,9)) + B=ArrayType{T}(reshape(collect(-7:28),4,9)) + + A_dist=distribute(A); B_dist=distribute(B) + + println(@test isapprox(LinearAlgebra.dot(A_dist,B_dist),LinearAlgebra.dot(A,B))) + + A_vec=vec(A); B_vec=vec(B) + + println(@test isapprox(LinearAlgebra.dot(A_dist,B_dist),LinearAlgebra.dot(A_vec,B_vec))) + +end From ee095d1de61b305efe1e5e46952781eaa1d1f9b5 Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Wed, 18 Dec 2019 16:10:59 +0900 Subject: [PATCH 6/9] Create test_opnorm2.jl --- test/test_opnorm2.jl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 test/test_opnorm2.jl diff --git a/test/test_opnorm2.jl b/test/test_opnorm2.jl new file mode 100644 index 0000000..e9b1c31 --- /dev/null +++ b/test/test_opnorm2.jl @@ -0,0 +1,20 @@ +using DistStat, LinearAlgebra, Test + +type=[Float32,Float64] + +if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray" + using CuArrays + ArrayType = CuArray +else + ArrayType = Array +end + +for T in type + A=ArrayType{T}(reshape(collect(1:45),5,9)) + A_dist=distribute(A) + + println(@test isapprox(opnorm(A_dist,1),opnorm(A,1))) + println(@test isapprox(opnorm(A_dist,2),opnorm(A,2))) + println(@test isapprox(opnorm(A_dist,Inf),opnorm(A,Inf))) + +end From 1d523e5b82cac2cf12b4046d01ce65c6a83dd5d3 Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Wed, 18 Dec 2019 22:40:30 +0900 Subject: [PATCH 7/9] Create test_mul2.jl --- test/test_mul2.jl | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 test/test_mul2.jl diff --git a/test/test_mul2.jl b/test/test_mul2.jl new file mode 100644 index 0000000..9f39b75 --- /dev/null +++ b/test/test_mul2.jl @@ -0,0 +1,37 @@ +using DistStat, LinearAlgebra, Test, Random + +type=[Float32,Float64] + +if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray" + using CuArrays + ArrayType = CuArray +else + ArrayType = Array +end + +for T in type + + A=ArrayType{T}(reshape(collect(1:63),7,9)) + B=ArrayType{T}(reshape(collect(-31:31),7,9)) + A_dist=distribute(A) + B_dist=distribute(B) + B_distt=distribute(ArrayType{T}(transpose(B))) + + C=MPIMatrix{T,ArrayType}(undef,9,9) + cols1=B_dist.partitioning[DistStat.Rank()+1][2] + cols2=B_distt.partitioning[DistStat.Rank()+1][2] + + result1=LinearAlgebra.mul!(C,transpose(A),B_dist) + ans1=transpose(A)*B + + println(@test isapprox(result1.localarray,ans1[:,cols1])) + + result2=LinearAlgebra.mul!(transpose(C),transpose(A_dist),B) + println(@test isapprox(result2.localarray,ArrayType{T}(transpose(ans1))[:,cols1])) + + D=Vector{Float64}(reshape() + E=Vector{Float64}(undef,9) + LinearAlgebra.mul!(E,transpose(A_dist),D) + println(@test isapprox(E,transpose(A)*D) + +end From 28dd64dc522bed2991a4eff5e852a8184487bdd5 Mon Sep 17 00:00:00 2001 From: Seungbongjung <54697679+Seungbongjung@users.noreply.github.com> Date: Wed, 18 Dec 2019 22:41:25 +0900 Subject: [PATCH 8/9] Update test_mul2.jl --- test/test_mul2.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/test_mul2.jl b/test/test_mul2.jl index 9f39b75..5911869 100644 --- a/test/test_mul2.jl +++ b/test/test_mul2.jl @@ -29,9 +29,4 @@ for T in type result2=LinearAlgebra.mul!(transpose(C),transpose(A_dist),B) println(@test isapprox(result2.localarray,ArrayType{T}(transpose(ans1))[:,cols1])) - D=Vector{Float64}(reshape() - E=Vector{Float64}(undef,9) - LinearAlgebra.mul!(E,transpose(A_dist),D) - println(@test isapprox(E,transpose(A)*D) - end From 9d643ba87ef0cdf1462346d94b993657b6b58642 Mon Sep 17 00:00:00 2001 From: Seyoon Date: Thu, 19 Dec 2019 17:25:07 +0900 Subject: [PATCH 9/9] fix bug in mul! / modifications for CuArrays --- src/DistStat.jl | 2 +- src/distlinalg.jl | 2 +- test/test_aux.jl | 11 ++++++----- test/test_distribute.jl | 4 ++-- test/test_dot.jl | 4 ++-- test/test_mul2.jl | 11 +++++++++-- test/test_opnorm2.jl | 6 +++--- 7 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/DistStat.jl b/src/DistStat.jl index 4b11b3c..00d096a 100644 --- a/src/DistStat.jl +++ b/src/DistStat.jl @@ -10,7 +10,7 @@ function __init__() @require CUDAnative="be33ccc6-a3ff-5ff2-a52e-74243cff1e17" begin include("cuda.jl") set_device!() - CuArrays.allowscalar(false) + #CuArrays.allowscalar(false) end end diff --git a/src/distlinalg.jl b/src/distlinalg.jl index 24df01b..88a2584 100644 --- a/src/distlinalg.jl +++ b/src/distlinalg.jl @@ -174,7 +174,7 @@ end function LinearAlgebra.mul!(C::AbstractVector{T}, A::Transpose{T, MPIMatrix{T,AT}}, B::AbstractVector{T}) where {T,AT} localA = get_local(A) fill!(C, zero(T)) - LinearAlgebra.mul!(C[transpose(A).partitioning[Rank()+1][2]], localA, B[transpose(A).partitioning[Rank()+1][1]]) + LinearAlgebra.mul!(@view(C[transpose(A).partitioning[Rank()+1][2]]), localA, B[transpose(A).partitioning[Rank()+1][1]]) sync() Allreduce!(C) C diff --git a/test/test_aux.jl b/test/test_aux.jl index d16f4ca..1a79e13 100644 --- a/test/test_aux.jl +++ b/test/test_aux.jl @@ -16,21 +16,22 @@ for T in type fill!(A_dist,1.0) cols1=A_dist.partitioning[DistStat.Rank()+1][2] - println(@test isapprox(A_dist.localarray,A[:,cols1])) + @test isapprox(A_dist.localarray,A[:,cols1]) - B=reshape(collect(1:70), 7, 10) + B=ArrayType{T}(reshape(collect(1:70), 7, 10)) B_dist=distribute(B) cols2=B_dist.partitioning[DistStat.Rank()+1][2] - println(@test isapprox(B_dist.localarray,B[:,cols2])) + @test isapprox(B_dist.localarray,B[:,cols2]) C_dist = MPIArray{T, 2, ArrayType}(undef, 7, 9) cols3=C_dist.partitioning[DistStat.Rank()+1][2] randn!(C_dist; seed=0,common_init=true) - C=ArrayType{T}(undef,size(C_dist)) + C=Array{T}(undef,size(C_dist)) Random.seed!(0) randn!(C) - println(@test isapprox(C_dist.localarray,C[:,cols3])) + + @test isapprox(C_dist.localarray, ArrayType{T}(C[:,cols3])) end diff --git a/test/test_distribute.jl b/test/test_distribute.jl index fe64e1f..e18179c 100644 --- a/test/test_distribute.jl +++ b/test/test_distribute.jl @@ -17,7 +17,7 @@ for T in type cols1=data_dist1.partitioning[DistStat.Rank()+1][2] cols2=data_dist2.partitioning[DistStat.Rank()+1][2] - println(@test data_dist1.localarray==data[:,cols1]) - println(@test data_dist2.localarray==(ArrayType{T}(transpose(data)))[:,cols2]) + @test data_dist1.localarray==data[:,cols1] + @test data_dist2.localarray==(ArrayType{T}(transpose(data)))[:,cols2] end diff --git a/test/test_dot.jl b/test/test_dot.jl index fce51d3..b03f643 100644 --- a/test/test_dot.jl +++ b/test/test_dot.jl @@ -15,10 +15,10 @@ for T in type A_dist=distribute(A); B_dist=distribute(B) - println(@test isapprox(LinearAlgebra.dot(A_dist,B_dist),LinearAlgebra.dot(A,B))) + @test isapprox(LinearAlgebra.dot(A_dist,B_dist),LinearAlgebra.dot(A,B)) A_vec=vec(A); B_vec=vec(B) - println(@test isapprox(LinearAlgebra.dot(A_dist,B_dist),LinearAlgebra.dot(A_vec,B_vec))) + @test isapprox(LinearAlgebra.dot(A_dist,B_dist),LinearAlgebra.dot(A_vec,B_vec)) end diff --git a/test/test_mul2.jl b/test/test_mul2.jl index 5911869..64b0482 100644 --- a/test/test_mul2.jl +++ b/test/test_mul2.jl @@ -24,9 +24,16 @@ for T in type result1=LinearAlgebra.mul!(C,transpose(A),B_dist) ans1=transpose(A)*B - println(@test isapprox(result1.localarray,ans1[:,cols1])) + @test isapprox(result1.localarray, ans1[:,cols1]) result2=LinearAlgebra.mul!(transpose(C),transpose(A_dist),B) - println(@test isapprox(result2.localarray,ArrayType{T}(transpose(ans1))[:,cols1])) + @test isapprox(result2.localarray, ArrayType{T}(transpose(ans1))[:,cols1]) + + + B_vec = ArrayType{T}(collect(1:7)) + C_vec = ArrayType{T}(undef, 9) + LinearAlgebra.mul!(C_vec, transpose(A_dist), B_vec) + C_true = transpose(A) * B_vec + @test isapprox(C_vec, C_true) end diff --git a/test/test_opnorm2.jl b/test/test_opnorm2.jl index e9b1c31..578603a 100644 --- a/test/test_opnorm2.jl +++ b/test/test_opnorm2.jl @@ -13,8 +13,8 @@ for T in type A=ArrayType{T}(reshape(collect(1:45),5,9)) A_dist=distribute(A) - println(@test isapprox(opnorm(A_dist,1),opnorm(A,1))) - println(@test isapprox(opnorm(A_dist,2),opnorm(A,2))) - println(@test isapprox(opnorm(A_dist,Inf),opnorm(A,Inf))) + @test isapprox(opnorm(A_dist,1),opnorm(A,1)) + @test isapprox(opnorm(A_dist,2),opnorm(A,2)) + @test isapprox(opnorm(A_dist,Inf),opnorm(A,Inf)) end