diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst index e746244b653d8..8a400065376f4 100644 --- a/libcxx/docs/ReleaseNotes/21.rst +++ b/libcxx/docs/ReleaseNotes/21.rst @@ -44,7 +44,8 @@ Implemented Papers Improvements and New Features ----------------------------- -- TODO +- The ``std::ranges::copy`` and ``std::ranges::copy_n`` algorithms have been optimized for ``std::vector::iterator``\s, + resulting in a performance improvement of up to 2000x. Deprecations and Removals diff --git a/libcxx/include/__algorithm/copy.h b/libcxx/include/__algorithm/copy.h index 962aa90059d57..7454c874a4d93 100644 --- a/libcxx/include/__algorithm/copy.h +++ b/libcxx/include/__algorithm/copy.h @@ -13,8 +13,10 @@ #include <__algorithm/for_each_segment.h> #include <__algorithm/min.h> #include <__config> +#include <__fwd/bit_reference.h> #include <__iterator/iterator_traits.h> #include <__iterator/segmented_iterator.h> +#include <__memory/pointer_traits.h> #include <__type_traits/common_type.h> #include <__type_traits/enable_if.h> #include <__utility/move.h> @@ -29,9 +31,129 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result); + template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> __copy(_InIter, _Sent, _OutIter); +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned( + __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { + using _In = __bit_iterator<_Cp, _IsConst>; + using difference_type = typename _In::difference_type; + using __storage_type = typename _In::__storage_type; + + const int __bits_per_word = _In::__bits_per_word; + difference_type __n = __last - __first; + if (__n > 0) { + // do first word + if (__first.__ctz_ != 0) { + unsigned __clz = __bits_per_word - __first.__ctz_; + difference_type __dn = std::min(static_cast(__clz), __n); + __n -= __dn; + __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn)); + __storage_type __b = *__first.__seg_ & __m; + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b; + __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word; + __result.__ctz_ = static_cast((__dn + __result.__ctz_) % __bits_per_word); + ++__first.__seg_; + // __first.__ctz_ = 0; + } + // __first.__ctz_ == 0; + // do middle words + __storage_type __nw = __n / __bits_per_word; + std::copy(std::__to_address(__first.__seg_), + std::__to_address(__first.__seg_ + __nw), + std::__to_address(__result.__seg_)); + __n -= __nw * __bits_per_word; + __result.__seg_ += __nw; + // do last word + if (__n > 0) { + __first.__seg_ += __nw; + __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n); + __storage_type __b = *__first.__seg_ & __m; + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b; + __result.__ctz_ = static_cast(__n); + } + } + return __result; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned( + __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { + using _In = __bit_iterator<_Cp, _IsConst>; + using difference_type = typename _In::difference_type; + using __storage_type = typename _In::__storage_type; + + const int __bits_per_word = _In::__bits_per_word; + difference_type __n = __last - __first; + if (__n > 0) { + // do first word + if (__first.__ctz_ != 0) { + unsigned __clz_f = __bits_per_word - __first.__ctz_; + difference_type __dn = std::min(static_cast(__clz_f), __n); + __n -= __dn; + __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn)); + __storage_type __b = *__first.__seg_ & __m; + unsigned __clz_r = __bits_per_word - __result.__ctz_; + __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r); + __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn)); + *__result.__seg_ &= ~__m; + if (__result.__ctz_ > __first.__ctz_) + *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_); + else + *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_); + __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word; + __result.__ctz_ = static_cast((__ddn + __result.__ctz_) % __bits_per_word); + __dn -= __ddn; + if (__dn > 0) { + __m = ~__storage_type(0) >> (__bits_per_word - __dn); + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn); + __result.__ctz_ = static_cast(__dn); + } + ++__first.__seg_; + // __first.__ctz_ = 0; + } + // __first.__ctz_ == 0; + // do middle words + unsigned __clz_r = __bits_per_word - __result.__ctz_; + __storage_type __m = ~__storage_type(0) << __result.__ctz_; + for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) { + __storage_type __b = *__first.__seg_; + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b << __result.__ctz_; + ++__result.__seg_; + *__result.__seg_ &= __m; + *__result.__seg_ |= __b >> __clz_r; + } + // do last word + if (__n > 0) { + __m = ~__storage_type(0) >> (__bits_per_word - __n); + __storage_type __b = *__first.__seg_ & __m; + __storage_type __dn = std::min(__n, static_cast(__clz_r)); + __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn)); + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b << __result.__ctz_; + __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word; + __result.__ctz_ = static_cast((__dn + __result.__ctz_) % __bits_per_word); + __n -= __dn; + if (__n > 0) { + __m = ~__storage_type(0) >> (__bits_per_word - __n); + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b >> __dn; + __result.__ctz_ = static_cast(__n); + } + } + } + return __result; +} + struct __copy_impl { template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> @@ -95,6 +217,16 @@ struct __copy_impl { } } + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> > + operator()(__bit_iterator<_Cp, _IsConst> __first, + __bit_iterator<_Cp, _IsConst> __last, + __bit_iterator<_Cp, false> __result) const { + if (__first.__ctz_ == __result.__ctz_) + return std::make_pair(__last, std::__copy_aligned(__first, __last, __result)); + return std::make_pair(__last, std::__copy_unaligned(__first, __last, __result)); + } + // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. template ::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> @@ -110,7 +242,7 @@ __copy(_InIter __first, _Sent __last, _OutIter __result) { } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) { return std::__copy(__first, __last, __result).second; } diff --git a/libcxx/include/__bit_reference b/libcxx/include/__bit_reference index 67abb023122ed..46d2b2f7ed948 100644 --- a/libcxx/include/__bit_reference +++ b/libcxx/include/__bit_reference @@ -10,6 +10,7 @@ #ifndef _LIBCPP___BIT_REFERENCE #define _LIBCPP___BIT_REFERENCE +#include <__algorithm/copy.h> #include <__algorithm/copy_n.h> #include <__algorithm/min.h> #include <__bit/countr.h> @@ -24,6 +25,7 @@ #include <__type_traits/conditional.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/void_t.h> +#include <__utility/pair.h> #include <__utility/swap.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -183,130 +185,6 @@ private: __mask_(__m) {} }; -// copy - -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned( - __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { - using _In = __bit_iterator<_Cp, _IsConst>; - using difference_type = typename _In::difference_type; - using __storage_type = typename _In::__storage_type; - - const int __bits_per_word = _In::__bits_per_word; - difference_type __n = __last - __first; - if (__n > 0) { - // do first word - if (__first.__ctz_ != 0) { - unsigned __clz = __bits_per_word - __first.__ctz_; - difference_type __dn = std::min(static_cast(__clz), __n); - __n -= __dn; - __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz - __dn)); - __storage_type __b = *__first.__seg_ & __m; - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b; - __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word; - __result.__ctz_ = static_cast((__dn + __result.__ctz_) % __bits_per_word); - ++__first.__seg_; - // __first.__ctz_ = 0; - } - // __first.__ctz_ == 0; - // do middle words - __storage_type __nw = __n / __bits_per_word; - std::copy_n(std::__to_address(__first.__seg_), __nw, std::__to_address(__result.__seg_)); - __n -= __nw * __bits_per_word; - __result.__seg_ += __nw; - // do last word - if (__n > 0) { - __first.__seg_ += __nw; - __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n); - __storage_type __b = *__first.__seg_ & __m; - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b; - __result.__ctz_ = static_cast(__n); - } - } - return __result; -} - -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned( - __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { - using _In = __bit_iterator<_Cp, _IsConst>; - using difference_type = typename _In::difference_type; - using __storage_type = typename _In::__storage_type; - - const int __bits_per_word = _In::__bits_per_word; - difference_type __n = __last - __first; - if (__n > 0) { - // do first word - if (__first.__ctz_ != 0) { - unsigned __clz_f = __bits_per_word - __first.__ctz_; - difference_type __dn = std::min(static_cast(__clz_f), __n); - __n -= __dn; - __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn)); - __storage_type __b = *__first.__seg_ & __m; - unsigned __clz_r = __bits_per_word - __result.__ctz_; - __storage_type __ddn = std::min<__storage_type>(__dn, __clz_r); - __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn)); - *__result.__seg_ &= ~__m; - if (__result.__ctz_ > __first.__ctz_) - *__result.__seg_ |= __b << (__result.__ctz_ - __first.__ctz_); - else - *__result.__seg_ |= __b >> (__first.__ctz_ - __result.__ctz_); - __result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word; - __result.__ctz_ = static_cast((__ddn + __result.__ctz_) % __bits_per_word); - __dn -= __ddn; - if (__dn > 0) { - __m = ~__storage_type(0) >> (__bits_per_word - __dn); - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b >> (__first.__ctz_ + __ddn); - __result.__ctz_ = static_cast(__dn); - } - ++__first.__seg_; - // __first.__ctz_ = 0; - } - // __first.__ctz_ == 0; - // do middle words - unsigned __clz_r = __bits_per_word - __result.__ctz_; - __storage_type __m = ~__storage_type(0) << __result.__ctz_; - for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) { - __storage_type __b = *__first.__seg_; - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b << __result.__ctz_; - ++__result.__seg_; - *__result.__seg_ &= __m; - *__result.__seg_ |= __b >> __clz_r; - } - // do last word - if (__n > 0) { - __m = ~__storage_type(0) >> (__bits_per_word - __n); - __storage_type __b = *__first.__seg_ & __m; - __storage_type __dn = std::min(__n, static_cast(__clz_r)); - __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn)); - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b << __result.__ctz_; - __result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word; - __result.__ctz_ = static_cast((__dn + __result.__ctz_) % __bits_per_word); - __n -= __dn; - if (__n > 0) { - __m = ~__storage_type(0) >> (__bits_per_word - __n); - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b >> __dn; - __result.__ctz_ = static_cast(__n); - } - } - } - return __result; -} - -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false> -copy(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { - if (__first.__ctz_ == __result.__ctz_) - return std::__copy_aligned(__first, __last, __result); - return std::__copy_unaligned(__first, __last, __result); -} - // copy_backward template @@ -989,8 +867,9 @@ private: _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_unaligned( __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result); template - _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> - copy(__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result); + _LIBCPP_CONSTEXPR_SINCE_CXX20 friend pair<__bit_iterator<_Dp, _IC>, __bit_iterator<_Dp, false> > + __copy_impl::operator()( + __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result) const; template _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_aligned( __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result); diff --git a/libcxx/include/bitset b/libcxx/include/bitset index 10576eb80bf2e..2914dee3d5292 100644 --- a/libcxx/include/bitset +++ b/libcxx/include/bitset @@ -129,6 +129,7 @@ template struct hash>; #if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS) # include <__cxx03/bitset> #else +# include <__algorithm/copy.h> # include <__algorithm/count.h> # include <__algorithm/fill.h> # include <__algorithm/fill_n.h> diff --git a/libcxx/test/benchmarks/algorithms/copy.bench.cpp b/libcxx/test/benchmarks/algorithms/copy.bench.cpp new file mode 100644 index 0000000000000..b6f0f15eb7703 --- /dev/null +++ b/libcxx/test/benchmarks/algorithms/copy.bench.cpp @@ -0,0 +1,89 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20 + +#include +#include +#include + +static void bm_ranges_copy_vb(benchmark::State& state, bool aligned) { + auto n = state.range(); + std::vector in(n, true); + std::vector out(aligned ? n : n + 8); + benchmark::DoNotOptimize(&in); + auto dst = aligned ? out.begin() : out.begin() + 4; + for (auto _ : state) { + benchmark::DoNotOptimize(std::ranges::copy(in, dst)); + benchmark::DoNotOptimize(&out); + } +} + +static void bm_ranges_copy_n_vb(benchmark::State& state, bool aligned) { + auto n = state.range(); + std::vector in(n, true); + std::vector out(aligned ? n : n + 8); + benchmark::DoNotOptimize(&in); + auto src = in.begin(); + auto dst = aligned ? out.begin() : out.begin() + 4; + for (auto _ : state) { + benchmark::DoNotOptimize(std::ranges::copy_n(src, n, dst)); + benchmark::DoNotOptimize(&out); + } +} + +static void bm_copy_vb(benchmark::State& state, bool aligned) { + auto n = state.range(); + std::vector in(n, true); + std::vector out(aligned ? n : n + 8); + benchmark::DoNotOptimize(&in); + auto beg = in.begin(); + auto end = in.end(); + auto dst = aligned ? out.begin() : out.begin() + 4; + for (auto _ : state) { + benchmark::DoNotOptimize(std::copy(beg, end, dst)); + benchmark::DoNotOptimize(&out); + } +} + +static void bm_copy_n_vb(benchmark::State& state, bool aligned) { + auto n = state.range(); + std::vector in(n, true); + std::vector out(aligned ? n : n + 8); + benchmark::DoNotOptimize(&in); + auto src = in.begin(); + auto dst = aligned ? out.begin() : out.begin() + 4; + for (auto _ : state) { + benchmark::DoNotOptimize(std::copy_n(src, n, dst)); + benchmark::DoNotOptimize(&out); + } +} + +static void bm_ranges_copy_vb_aligned(benchmark::State& state) { bm_ranges_copy_vb(state, true); } +static void bm_ranges_copy_vb_unaligned(benchmark::State& state) { bm_ranges_copy_vb(state, false); } +static void bm_ranges_copy_n_vb_aligned(benchmark::State& state) { bm_ranges_copy_n_vb(state, true); } +static void bm_ranges_copy_n_vb_unaligned(benchmark::State& state) { bm_ranges_copy_n_vb(state, false); } + +static void bm_copy_vb_aligned(benchmark::State& state) { bm_copy_vb(state, true); } +static void bm_copy_vb_unaligned(benchmark::State& state) { bm_copy_vb(state, false); } +static void bm_copy_n_vb_aligned(benchmark::State& state) { bm_copy_n_vb(state, true); } +static void bm_copy_n_vb_unaligned(benchmark::State& state) { bm_copy_n_vb(state, false); } + +// Test std::ranges::copy for vector::iterator +BENCHMARK(bm_ranges_copy_vb_aligned)->Range(8, 1 << 16)->DenseRange(102400, 204800, 4096); +BENCHMARK(bm_ranges_copy_n_vb_aligned)->Range(8, 1 << 20); +BENCHMARK(bm_ranges_copy_vb_unaligned)->Range(8, 1 << 20); +BENCHMARK(bm_ranges_copy_n_vb_unaligned)->Range(8, 1 << 20); + +// Test std::copy for vector::iterator +BENCHMARK(bm_copy_vb_aligned)->Range(8, 1 << 20); +BENCHMARK(bm_copy_n_vb_aligned)->Range(8, 1 << 20); +BENCHMARK(bm_copy_vb_unaligned)->Range(8, 1 << 20); +BENCHMARK(bm_copy_n_vb_unaligned)->Range(8, 1 << 20); + +BENCHMARK_MAIN(); diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp index b5f0a32b986a0..1ca397c92a334 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp @@ -14,6 +14,7 @@ #include #include +#include #include "test_macros.h" #include "test_iterators.h" @@ -59,6 +60,26 @@ struct TestInIters { } }; +TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) { + std::vector in(N, false); + for (std::size_t i = 0; i < N; i += 2) + in[i] = true; + + { // Test copy with aligned bytes + std::vector out(N); + std::copy(in.begin(), in.end(), out.begin()); + assert(in == out); + } + { // Test copy with unaligned bytes + std::vector out(N + 8); + std::copy(in.begin(), in.end(), out.begin() + 4); + for (std::size_t i = 0; i < N; ++i) + assert(out[i + 4] == in[i]); + } + + return true; +} + TEST_CONSTEXPR_CXX20 bool test() { types::for_each(types::cpp17_input_iterator_list(), TestInIters()); @@ -78,13 +99,23 @@ TEST_CONSTEXPR_CXX20 bool test() { assert(std::equal(a, a + 10, expected)); } + { // Test vector::iterator optimization + assert(test_vector_bool(8)); + assert(test_vector_bool(19)); + assert(test_vector_bool(32)); + assert(test_vector_bool(49)); + assert(test_vector_bool(64)); + assert(test_vector_bool(199)); + assert(test_vector_bool(256)); + } + return true; } int main(int, char**) { test(); -#if TEST_STD_VER > 17 +#if TEST_STD_VER >= 20 static_assert(test()); #endif diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp index b0acc1060101c..889e71f4eceb9 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp @@ -14,6 +14,7 @@ #include #include +#include #include "test_macros.h" #include "test_iterators.h" @@ -37,20 +38,18 @@ class Derived : public PaddedBase { }; template -TEST_CONSTEXPR_CXX20 void -test_copy_n() -{ +TEST_CONSTEXPR_CXX20 void test_copy_n() { { const unsigned N = 1000; - int ia[N] = {}; + int ia[N] = {}; for (unsigned i = 0; i < N; ++i) - ia[i] = i; + ia[i] = i; int ib[N] = {0}; - OutIter r = std::copy_n(InIter(ia), UDI(N/2), OutIter(ib)); - assert(base(r) == ib+N/2); - for (unsigned i = 0; i < N/2; ++i) - assert(ia[i] == ib[i]); + OutIter r = std::copy_n(InIter(ia), UDI(N / 2), OutIter(ib)); + assert(base(r) == ib + N / 2); + for (unsigned i = 0; i < N / 2; ++i) + assert(ia[i] == ib[i]); } { // Make sure that padding bits aren't copied @@ -70,53 +69,80 @@ test_copy_n() } } -TEST_CONSTEXPR_CXX20 bool -test() -{ - test_copy_n, cpp17_output_iterator >(); - test_copy_n, cpp17_input_iterator >(); - test_copy_n, forward_iterator >(); - test_copy_n, bidirectional_iterator >(); - test_copy_n, random_access_iterator >(); - test_copy_n, int*>(); - - test_copy_n, cpp17_output_iterator >(); - test_copy_n, cpp17_input_iterator >(); - test_copy_n, forward_iterator >(); - test_copy_n, bidirectional_iterator >(); - test_copy_n, random_access_iterator >(); - test_copy_n, int*>(); - - test_copy_n, cpp17_output_iterator >(); - test_copy_n, cpp17_input_iterator >(); - test_copy_n, forward_iterator >(); - test_copy_n, bidirectional_iterator >(); - test_copy_n, random_access_iterator >(); - test_copy_n, int*>(); - - test_copy_n, cpp17_output_iterator >(); - test_copy_n, cpp17_input_iterator >(); - test_copy_n, forward_iterator >(); - test_copy_n, bidirectional_iterator >(); - test_copy_n, random_access_iterator >(); - test_copy_n, int*>(); - - test_copy_n >(); - test_copy_n >(); - test_copy_n >(); - test_copy_n >(); - test_copy_n >(); - test_copy_n(); +TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) { + std::vector in(N, false); + for (std::size_t i = 0; i < N; i += 2) + in[i] = true; + + { // Test copy with aligned bytes + std::vector out(N); + std::copy_n(in.begin(), N, out.begin()); + assert(in == out); + } + { // Test copy with unaligned bytes + std::vector out(N + 8); + std::copy_n(in.begin(), N, out.begin() + 4); + for (std::size_t i = 0; i < N; ++i) + assert(out[i + 4] == in[i]); + } + + return true; +} + +TEST_CONSTEXPR_CXX20 bool test() { + test_copy_n, cpp17_output_iterator >(); + test_copy_n, cpp17_input_iterator >(); + test_copy_n, forward_iterator >(); + test_copy_n, bidirectional_iterator >(); + test_copy_n, random_access_iterator >(); + test_copy_n, int*>(); + + test_copy_n, cpp17_output_iterator >(); + test_copy_n, cpp17_input_iterator >(); + test_copy_n, forward_iterator >(); + test_copy_n, bidirectional_iterator >(); + test_copy_n, random_access_iterator >(); + test_copy_n, int*>(); + + test_copy_n, cpp17_output_iterator >(); + test_copy_n, cpp17_input_iterator >(); + test_copy_n, forward_iterator >(); + test_copy_n, bidirectional_iterator >(); + test_copy_n, random_access_iterator >(); + test_copy_n, int*>(); + + test_copy_n, cpp17_output_iterator >(); + test_copy_n, cpp17_input_iterator >(); + test_copy_n, forward_iterator >(); + test_copy_n, bidirectional_iterator >(); + test_copy_n, random_access_iterator >(); + test_copy_n, int*>(); + + test_copy_n >(); + test_copy_n >(); + test_copy_n >(); + test_copy_n >(); + test_copy_n >(); + test_copy_n(); + + { // Test vector::iterator optimization + assert(test_vector_bool(8)); + assert(test_vector_bool(19)); + assert(test_vector_bool(32)); + assert(test_vector_bool(49)); + assert(test_vector_bool(64)); + assert(test_vector_bool(199)); + assert(test_vector_bool(256)); + } return true; } -int main(int, char**) -{ - test(); +int main(int, char**) { + test(); #if TEST_STD_VER > 17 - static_assert(test()); + static_assert(test()); #endif return 0; diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp index 2507e594fe944..68356c80ba7f6 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp @@ -26,6 +26,7 @@ #include "almost_satisfies_types.h" #include "test_iterators.h" +#include "test_macros.h" #include "type_algorithms.h" template > @@ -99,6 +100,28 @@ constexpr void test_iterators() { } // clang-format on +#if TEST_STD_VER >= 23 +constexpr bool test_vector_bool(std::size_t N) { + std::vector in(N, false); + for (std::size_t i = 0; i < N; i += 2) + in[i] = true; + + { // Test copy with aligned bytes + std::vector out(N); + std::ranges::copy(in, out.begin()); + assert(in == out); + } + { // Test copy with unaligned bytes + std::vector out(N + 8); + std::ranges::copy(in, out.begin() + 4); + for (std::size_t i = 0; i < N; ++i) + assert(out[i + 4] == in[i]); + } + + return true; +} +#endif + constexpr bool test() { types::for_each(types::forward_iterator_list{}, []() { test_iterators, Out, sentinel_wrapper>>(); @@ -204,6 +227,18 @@ constexpr bool test() { } } +#if TEST_STD_VER >= 23 + { // Test vector::iterator optimization + assert(test_vector_bool(8)); + assert(test_vector_bool(19)); + assert(test_vector_bool(32)); + assert(test_vector_bool(49)); + assert(test_vector_bool(64)); + assert(test_vector_bool(199)); + assert(test_vector_bool(256)); + } +#endif + return true; } diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp index d2a2b7c488830..c7031f63a02f6 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_n.pass.cpp @@ -19,8 +19,10 @@ #include #include #include +#include #include "almost_satisfies_types.h" +#include "test_macros.h" #include "test_iterators.h" template @@ -41,10 +43,10 @@ static_assert(std::is_same_v, std::ranges::i template constexpr void test_iterators() { { // simple test - std::array in {1, 2, 3, 4}; + std::array in{1, 2, 3, 4}; std::array out; std::same_as> auto ret = - std::ranges::copy_n(In(in.data()), in.size(), Out(out.data())); + std::ranges::copy_n(In(in.data()), in.size(), Out(out.data())); assert(in == out); assert(base(ret.in) == in.data() + in.size()); assert(base(ret.out) == out.data() + out.size()); @@ -70,13 +72,37 @@ constexpr void test_in_iterators() { template constexpr void test_proxy_in_iterators() { - test_iterators>, Out, sentinel_wrapper>>>(); + test_iterators>, + Out, + sentinel_wrapper>>>(); test_iterators>, Out>(); test_iterators>, Out>(); test_iterators>, Out>(); test_iterators>, Out>(); } +#if TEST_STD_VER >= 23 +constexpr bool test_vector_bool(std::size_t N) { + std::vector in(N, false); + for (std::size_t i = 0; i < N; i += 2) + in[i] = true; + + { // Test copy with aligned bytes + std::vector out(N); + std::ranges::copy_n(in.begin(), N, out.begin()); + assert(in == out); + } + { // Test copy with unaligned bytes + std::vector out(N + 8); + std::ranges::copy_n(in.begin(), N, out.begin() + 4); + for (std::size_t i = 0; i < N; ++i) + assert(out[i + 4] == in[i]); + } + + return true; +}; +#endif + constexpr bool test() { test_in_iterators>(); test_in_iterators>(); @@ -92,8 +118,8 @@ constexpr bool test() { { // check that every element is copied exactly once struct CopyOnce { - bool copied = false; - constexpr CopyOnce() = default; + bool copied = false; + constexpr CopyOnce() = default; constexpr CopyOnce(const CopyOnce& other) = delete; constexpr CopyOnce& operator=(const CopyOnce& other) { assert(!other.copied); @@ -101,14 +127,26 @@ constexpr bool test() { return *this; } }; - std::array in {}; - std::array out {}; + std::array in{}; + std::array out{}; auto ret = std::ranges::copy_n(in.begin(), in.size(), out.begin()); assert(ret.in == in.end()); assert(ret.out == out.end()); assert(std::all_of(out.begin(), out.end(), [](const auto& e) { return e.copied; })); } +#if TEST_STD_VER >= 23 + { // Test vector::iterator optimization + assert(test_vector_bool(8)); + assert(test_vector_bool(19)); + assert(test_vector_bool(32)); + assert(test_vector_bool(49)); + assert(test_vector_bool(64)); + assert(test_vector_bool(199)); + assert(test_vector_bool(256)); + } +#endif + return true; }