  C++    
 


   .    .

     C++ (C++20, C++23,  C++26)  ,     C++      senior.      ,       ThreadSanitizer.     .

   key-value  shardkv:  14               std::execution.

 : , , , condition variables;     C++;   lock-free ; hazard pointers  RCU;  ,   senders/receivers;      .         ,  42     .

 ,      senior C++.





  C++    





   


    ,     C++,  RAII,    move-,         std::thread,  .     ,      ,       senior-.

 :    ,      ThreadSanitizer    happens-before.        ThreadSanitizer  AddressSanitizer;       .

  : ;      ;   ;   ;    ( ,  ,  );       .              .

  shardkv  in-memory key-value ,   14                ,  , CI    .       ,   :     ,    ,    .

   : GCC 14+  Clang 18+  -std=c++23, CMake 3.25+, Ninja;   1013  Asio (standalone)  stdexec,      FetchContent.          ,       ;           ,   .

     17,           7.      senior.




 1. std::thread, std::jthread  stop_token:       std::terminate


   .    .

:    .      ,    ,       .

 :    C++   . :      ,    .    ,    ,    ,            .

   :

 ,               ;

 ,   std::thread  std::terminate   detach()    ,   ;

           ;

  std::jthread     stop_token;

      ,    ;

     : TCP- shardkv    ,    .

 .       C++23   GCC 14+  Clang 18+   -std=c++23.    std::println   <print>.       ThreadSanitizer  AddressSanitizer    .




1.1.       


    :   ,   ,  .       :  ,  ,   .   , ,          .

            .        .          .

 Linux std::thread     pthread,  pthread     clone.    ,  :

         ;

   ( Linux   8   ,     );

     (thread_local);

  ,        .

    ,  :



















 1.1     (ex01_cost.cpp)



    release-   55   .            . , ?      .      .

  :      .       ,         ,      .     ,      7.  ,   ,       .

 :     ,  . std::thread::hardware_concurrency()    ,   ,   :     0,   .    ,  ,    ,     ,      .




1.2. join, detach  std::terminate


  :    std::thread - .       ,       :

 join()    ;

 detach()   ,      ,       .

     ?










 1.2       (ex02_terminate.cpp)



   std::terminate.     ,   .  std::thread,   joinable() == true,   .    ,     :   (   ,   ),   (      ,   )   .   :     UB.




 detach()    


,   terminate,   : ,   detach,   .   .   .

 detach()      ,    .  ,   ,  ,    ,  :

   ,   ,     ;

 ,     ,    ;

    main     ,       (   1.9).

  use-after-free,        .  .  .

 detach   ,   : ,      ,   ,   ,       .    .   :  detach()     ,      .




1.3.   join:      


, :   join().     :



















 1.3      join()   (ex03_exception_join.cpp)



  bad()     join()  ,    .    ,  std::thread      terminate. ,      ,    .

, ,   try/catch   join()   .     C++,   C++ ,   ,  RAII-.    .




1.4. std::jthread: ,     


 C++20  std::jthread (joining thread).    std::thread:

    terminate,      (request_stop()),   join();

      : std::stop_source  std::stop_token;

        std::stop_token, jthread     .

 good()      jthread.   ,      .  terminate.

 ,      :    std::jthread. std::thread   ,      terminate,   ( )      .

 : join   ,    .      ,  jthread   .   ,  ,    ,      ( 1.6).




1.5.  : ,   


    (,  decay-copy)       ,   .     ,      rvalue.      .



















 1.4             (ex04_args.cpp)



   .    std::vector<int>&,      :    rvalue,       .   ,   .     ,   std::ref       ,    .

     .    const std::string&,    char*,    ,    std::string  ,    .       (, - ,      detach),     . :  ,    .

Move-only  . std::unique_ptr   std::move         :    ,       .

       .  [&]   ,     ,  std::ref:  ,     .  jthread        :  jthread  ,   ,   .             1.3.




1.6.  : stop_token, stop_source, stop_callback


      .  C++   ,   . ,   ,    ,  ,    .         .

  C++   ,    ,     .     :

 std::stop_source  .  request_stop()  .    ,  .

 std::stop_token  ,   stop_source  get_token().   stop_requested()  ,   .   ,       .

 std::stop_callback    .  ,      .

  jthread   stop_source.        ,    :



















 1.5    (ex05_stop.cpp)



    .

     ,   .  while (!st.stop_requested()) ,        .        (     recv()    accept()),    ,    .      .     stop_callback:           ,  -,  condition variable ( 3).       .

  stop_callback.     ,   request_stop(),    ,  .       ,   ,   stop_callback.  stop_callback ,        .     .

 stop_source    .               .     :       .




1.7.   


      ,  std::terminate.    main :     ,          .

,   .    std::exception_ptr  ,      :



















 1.6  ,    ,    (ex06_exceptions.cpp)



     catch (),   std::current_exception(),      join()   std::rethrow_exception().   join()?   join() ,  ,   ,  ,   .     error  main    .     happens-before,    5    .

  7        std::future,       ,  ,   .




1.8. thread_local  hardware_concurrency


  thread_local       .        ,      :










 1.7        (ex07_thread_local.cpp)



  :      (    ),   ,      ,    (std::mt19937  ,     rand  ).

  :     ,     64     10    640 .   thread_local-    ,      detach    .




1.9.     


 ,   ,   .    detach()       -. main()  .   :

   main   std::exit();

 std::exit()      ,     ;

             ;

       ,     ,     .

 :    ,         .     .      .

  :

 :       main.   ( - )    stop_token    .  jthread    .

 : std::quick_exit()       (  ,   std::at_quick_exit).   ,    ,     .

 : ,   .




1.10.   


 ,      pull request'.       Request changes.

 detach()  .      ,    .

 std::thread    C++20.  std::jthread,    .

  [&]   ,      .   :  ,   ,   .

 ,   .     stop_token       .

 ,    .  std::terminate,    -.

 sleep_for   .  100 ,      ,  .

     .   ,   .

     join()   .  data race,     int.




1.11.    


 ,    join()  detach()  std::thread? (middle)   std::terminate.    ,   :      ,    .

 std::jthread   std::thread? (middle)   request_stop()  join()  terminate;     stop_source/stop_token;   stop_token   .

  ? (middle)  :    stop_token     stop_callback.      C++ ,   .     ,    .

 detach() ? (middle)  ,   ,  ,     ,   .         .

       ? (middle) std::exception_ptr  std::current_exception()/std::rethrow_exception(),  std::promise/std::future,     .

          ? (junior/middle) std::ref      .  ,     . :    char*  std::string,      .

   ? (senior)      hardware_concurrency().     I/O ,    I/O ( 10).         ,   .

 ,         main? (senior) exit()   ,    : UB.          .




1.12.   shardkv,  1:   


  . : shardkv  in-memory key-value       Redis.           ,      senior-.     ,   .

    1:

  : CMake   debug, release, tsan, asan;

    std::unordered_map;

 : PING, GET, SET, DEL, SETNX, INCR;

 TCP-   ,   std::jthread   ;

  :  ,      .

   :      .      unordered_map.  data race,    2     ,  ThreadSanitizer  .

 : .    .




  


    .        accept(),      recv().  stop_requested()     . :

   stop_callback,   shutdown()   .  Linux   accept()  ,    ;

     std::stop_source  .       ,   accept()    .      accept            ,    ;

     stop_callback,   shutdown()  . recv()  0,   ;

  Connection   ,     run()      jthread.

      serve(): stop_callback   ,  ,  .  stop_callback ,      .  shutdown()       .

  :       (reap_finished).  done         ,   std::atomic   release/acquire.   ,      5,     :        .




 


 :



















 1.8  TCP- shardkv,  1 (server.hpp)



:





































 1.9   (server.cpp)



 .  SIGTERM     4,     Enter:



















 1.10    shardkv,  1 (main.cpp)




, , 


    ThreadSanitizer,   :










 1.11      TSan,   



       :     ,    ,    .   ,   ,  ,     .

 :










 1.12      nc




  2


    TSan    ,    INCR  :










 1.13      INCR



   :










 1.14   TSan   



   .      unordered_map.  release-       ,       .             ,    .




   1


 ,     ,   .     : ,   RAII-  .



















 1.15   shardkv,  1 (storage.hpp)










 1.16   (storage.cpp)










 1.17    shardkv (protocol.hpp)




























 1.18   (protocol.cpp)




























 1.19   RAII-  POSIX- (socket.hpp)





































 1.20   (socket.cpp)



















 1.21    CMake (CMakeLists.txt)



















 1.22    CMake (CMakePresets.json)



















 1.23   (test_protocol.cpp)




























 1.24   (test_server.cpp)




   


     :  ,      TSan  ASan    (ctest --repeat until-fail:10).  ,   ,   .




 1.1 ().   


    .    .    ,  .










 1.25     1.1 (task1_1_bugs.cpp)



   .      ,       .     std::jthread ,    ,  log_async -    .

:     ASan  TSan;            (   ).




 1.2 (). parallel_sum   


 :










 1.26    parallel_sum



  transform       .    threads   ,     .       ,   .  transform     , parallel_sum         .

  :  data, threads == 0, threads ,  .

:    std::transform_reduce   ; ,   transform    ,    ;          .




 1.3 (). ,    


  TickService:

     N   N ;

       std::vector<std::uint64_t>,    ,    ;

  stop_and_total()       ;

    ,   stop_and_total()  ;

      std::stop_source.

 :        ,     . :  ,             ,    . ,     ,    .

:      1000     ASan  TSan;  stop_and_total()      ;   stop_and_total()   .

              .




 2. Data race,     deadlock


   int,     ?      .

     shardkv   :      std::unordered_map.    .    ThreadSanitizer  data race.   ,    ,   ,   ,    .

  .     : ,    ,       .    .

   :

   data race  race condition (    );

    lock_guard, unique_lock, scoped_lock  shared_lock;

 ,      ,    ;

   ,      ;

          ;

 ,     ,    ;

   shardkv      .




2.1. Data race:   UB,      


      :



















 2.1           (ex01_race.cpp)



      ++counter.  4 000 000.      -O0  2 176 855.    .

? ++counter     ,  :      ,  ,  .    41,    42,   42.   .

        , ,  ,   .  C++  data race :         ,       ,     ,      happens-before (   ).    :   data race   .

    .  .   ,   ,   .             .      .    .   -O2       4 000 000:     counter += 1000000,      .   .  UB,   .

  ,   :










 2.2     bool



  ,   ready   (  ,   ),    ,   1  .  -O0 ,  -O2 .        .

: data race     ,     ,     . Ÿ       . Ÿ .




Data race  race condition   


 Data race    :     .  UB.  ThreadSanitizer.

 Race condition   :    ,     ,      .     data race:    ,     . TSan    .

 race condition  data race    2.4.   :      .




2.2. std::mutex  RAII-


 (mutual exclusion)  ,          . ,    , .        :          .



















 2.3    ,   (ex02_mutex.cpp)



   4 000 000.     ,   ,  :

  :     ;

 : ,     unlock(),   ,     lock().       .               .      happens-before,     5.

  m.lock()  m.unlock()  :       -   return,    ,    .    RAII-:

 std::lock_guard    ,   .    ,   .      .

 std::unique_lock        ,   (std::defer_lock),    ,  .   condition_variable ( 3).   lock_guard:   .

 std::scoped_lock (C++17)       ,   .      lock_guard,     .

 std::shared_lock    shared_mutex ( 2.6).

 CTAD (   )     : std::lock_guard lock(m)  std::lock_guard<std::mutex> lock(m).




  


   Linux   :         ,    (    futex,    4).      (contention):      ,     , ,  .   :

    . ,      (, ,  )    (,   ),    .   shardkv    :         .




2.3.  ,   


  :       , -  .    ,    .            ,   .

:          ,   ,       .              :




























 2.4   ,    (ex03_synchronized.cpp)



Synchronized<T>   ,     :       with_lock,    .     folly (folly::Synchronized)  Boost (boost::synchronized_value).

    :         ,       .      data race. : -   ,   .      shardkv: get()   ,   const std::string&.

        .      ,       ,     ,   (std::mutex ).      ,      ( 2.5). :     ,   .




2.4.  :    ,   


   race condition  data race:




























 2.5    ,      (ex04_check_then_act.cpp)



 WalletBad    . Data race , TSan .    ,      ,        .      100,    30.  ?140.

   ,   .         .    ,    ,  : try_withdraw      .

            empty(), top()  pop():










 2.6        



    ,     .      : std::optional<T> try_pop().        2.2.

     :

 - (empty, size, contains, count),      ;

 ,     ;

     ,     .

 shardkv    INCR  SETNX      ,    get() + set()   .




2.5. Deadlock:        


  ,            .   ,   ,   CPU.   .     ,   ,    .

 ,       :

1.  :      .

2.   :       .

3.   :   ,    .

4.  : A  B, B  A (  ).

     ,  . ,     :   .



















 2.7   ,  ,   (ex05_deadlock.cpp)



 merge_bad     .  1  merge_bad(a, b),  2  merge_bad(b, a).  1  a   b,  2  b   a. .

   :

 std::scoped_lock    ( std::lock).   ,    ,        :     ,       .     ,      .

   .    (,   ),        .   . ,          ,   .

  .    ,  ,    N,       N.   ,   -  .       2.3.

     .   .      .

   merge(a, a):   std::mutex,     ,   UB (   ).   merge        .




2.6. shared_mutex:  ,  


       ,       ,    . std::shared_mutex     :

  (std::shared_lock)     ;

  (std::unique_lock  std::lock_guard)   ,  .



















 2.8   ,   (ex06_shared_mutex.cpp)



   ,    :

 shared_mutex  :       ,            (    4);

     (  -),   shared_mutex    ;

        :    .

: shared_mutex ,   ,  ,   .      .   4     shardkv.




2.7. recursive_mutex   


std::recursive_mutex        (    ).   ,         ,    .

     :   ,        .   A      ,     B,    , recursive_mutex    ,  .

 :     ( )   (,    ,    _locked).   ,     .




2.8. : timed_mutex   


    :      .    std::timed_mutex   try_lock_for  try_lock_until:




























 2.9        (ex07_timeouts_once.cpp)



  .

 steady_clock  . std::chrono::system_clock    :    NTP,      .   12:00:05    ,    . steady_clock    . try_lock_for     steady_clock,    try_lock_until    ,       steady_clock.

     .         ,    ,  -    .    ,    .

    .    GCC  libtsan      TSan: unlock of an unlocked mutex. : try_lock_for  libstdc++  pthread_mutex_clocklock,  libtsan       ,    .   ,   .      :










 2.10    TSan (tsan.supp)










 2.11     



    :        ,    .     TSan  CI    13.




2.9.  


  (    )     .    ,    ,    .

 C++11     (   ):

   .  ,       ,        ;   .   magic statics.   ,    ;

 std::call_once  std::once_flag        ,      .

   ,    :     (double-checked locking)   .  data race     .    ,       5.  : magic statics.




2.10.     


  :   const-   ,     ;         .        .



















 2.12      ,    (ex08_stdlib.cpp)



:

 .   std::vector  std::map    ,       .    ,     : operator[]  std::map  ,   .

 std::cout    . Data race  ,       . std::osyncstream (C++20)       .

 rand().    ,  .    <random>,     (thread_local).

 localtime(), gmtime(), strtok(), setlocale(), getenv()   setenv().      .    std::chrono  std::format,  strtok  std::string_view  std::views::split.

 std::shared_ptr.  :   ,       .       5.




2.11. ThreadSanitizer:  


ThreadSanitizer (TSan)  ,   GCC  Clang,   data race    .           ,       happens-before.

 :










 2.13      TSan 



     tsan:










 2.14   tsan:     



  :

 WARNING: ThreadSanitizer: data race   ;

 Write of size 8 by thread T2      ;

 Previous read/write by thread T1    ,  ;

 Location is      ( , ,   );

 Thread T2 created at      .

    TSan:

    ,      .      , TSan .    TSan     ;

    race condition  data race ( WalletBad);

    515       510       ,     CI;

 TSan    ASan   ,      ;

   ,   ( 2.8).       ,    ,    .




2.12.   


   bool/int,   .  data race  ,  UB,    .

 ,   ,   .         .

     ,  - .

   +     . empty() + pop(), contains() + insert(), get() + set().

  ,       .

        lock_guard.  std::scoped_lock.

    : ,  , ,   .

 recursive_mutex   .

   system_clock.

  m.lock()/m.unlock()  RAII-.




2.13.    


  data race      race condition? (middle) Data race       ,      ;   UB. Race condition       ;     data race.       .

 ++counter         ? (middle/senior)  UB:     ,   ,  , ,    .       .

 lock_guard   unique_lock  scoped_lock? (junior/middle) lock_guard   RAII; unique_lock   ( ,  unlock, ,  ,   condition_variable); scoped_lock     .

      . (middle)   ;     : std::scoped_lock,   ,  ,     .

    empty(), top(), pop()    ? (middle)    ,      .   try_pop(),  std::optional<T>.

 shared_mutex   mutex? (senior)        :        , -     .  : ,   .

     ? (middle) ,  C++11 (magic statics). : ,  double-checked locking     data race.

   std::vector    ? (junior/middle) ,  const-,    . : operator[]  std::map .




2.14.   shardkv,  2:     


 :

 ,  TSan    1 (      1);

  Storage :     ;

 INCR  SETNX   Storage         ;

  -  TSan;

      kvbench    .

 :

 8   5000 INCR   40 000;

 8    SETNX      , 200  ;

 6    TCP-  INCR     ;

    TSan  ASan, 10  .

 :        .




 





























 2.15   shardkv,  2 (storage.hpp)



















 2.16   (storage.cpp)



       :

  mutable,      const-;

        ,       ;

 get()  ,   .

 server.cpp   1      execute:   .




-





























 2.17   (test_storage_stress.cpp)



  SetnxHasExactlyOneWinner  std::atomic<int>   .    int,       ;      5.




kvbench,  1






































 2.18    shardkv (kvbench.cpp)



       .  :        ,       join  .    ,   .    .




 


 release,   kvbench   ,  mix (90% GET, 10% SET), 20 000   :










 2.19   release   



   :










 2.20  kvbench  1:     



 :

  4    ;

   ,   16   ;

   ,   :  8  16     ,  ops/s .

 ,      :        ,    kvbench       .       -  CPU,           .       8+ :     ,      .  ,    ,   CPU,     13 (off-CPU ).    4      .

    bench/results/step02.txt.      /,        .




   2


        1,  CMakeLists.txt ( kvbench  -):




























 2.21    CMake (CMakeLists.txt)




   


  :  ,      TSan  ASan   .




 2.1 (). ,  




















 2.22     2.1 (task2_1_bank.cpp)



   (    ).       ,    .   transfer  :

1.  std::scoped_lock;

2.       id .

,    .  :     transfer(alice, alice, 10)?

:    ,       2 000 000; TSan ;        ,   UB.




 2.2 (). ThreadSafeStack    


  ThreadSafeStack<T>  :










 2.23   ThreadSafeStack<T>



:

   empty() + top() + pop(),    ;

       T            ;

      .

 -: 4    100 000  , 4   ,    .        .

: -   TSan 10  ;   ,  size()       .




 2.3 (). hierarchical_mutex


  hierarchical_mutex:    . : ,     N,         N.     std::logic_error ,  ,      - .

:

  lock(), unlock(), try_lock(),     std::lock_guard  std::unique_lock;

      thread_local ;

  unlock()      (      );

 ,  ,        .

:      ;     ;       lock/unlock;      TSan.

              .




 3. Condition variables   


 sleep(100),   .   .        .

        .        :  ,     ;     ;   ,   .    while (!ready) sleep_for()    :      ,  CPU    .

   condition variable ( ).          :       ,         .     ,   ,       ,   shardkv     .

   :

 ,  condition variable       ;

   spurious wakeup  lost wakeup;

 ,   notify_one,  notify_all       ;

        ,   ;

  condition_variable_any   stop_token;

        ;

   shardkv EXPIRE/TTL   ,      .




3.1.   


Condition variable    ,    .              .   ,     :       ,  condition variable   ,    .

    :



















 3.1      (ex01_event.cpp)



  :

1.  ( ready_)   ,       .    bool.

2.            .

3.   wait  ,   .

   cv.wait(lock, pred):










 3.2    cv.wait(lock, pred)



   :       ,      .  wait   unique_lock:       ,  lock_guard   .




3.2.   





 1: wait    spurious wakeup


 condition variable        notify.   spurious wakeup ( ).     :    .     .

   :










 3.3  if  while



      , ,   .  :    wait   (   while).      .

     ,   .      ,     ,   :       .    ,   .




 2: lost wakeup  ,    


Condition variable   .  notify ,    ,   .        ,          .             .

   :










 3.4   :    



 B            A.   B  ready    ,        : A      ,  wait     .

 :     ,  ready   std::atomic<bool>.    data race,        .   3.1     .

:  ,    condition variable,     ,   .  ,   .




 3:      


  :    ,     ;    ,     .        ,  data race  .  condition variable       .




3.3. notify_one, notify_all    


notify_one    (   ). ,        :     .

notify_all  . ,     :  ,  ,    .         :  32         32,   , 31     .   thundering herd ( ),         CPU.

    : notify_one ,   notify_all.          ,     .

    ?   ,     .    notify  ,      ,    ,   ,    .    :










 3.5     ,  



  .     condition variable    (,   ,     ,  cv),  notify         cv.      .  ,      ,    .




3.4.   : wait_for  wait_until


    .       :

 wait_for(lock, duration, pred)      ;

 wait_until(lock, time_point, pred)     .

    bool: true,    , false,   ,      .



















 3.6     (ex03_deadline.cpp)



   .  ,   100   ,      .      wait_for(100ms),     300 .              .     :     .

     :             wait_for    ,   ,          . wait_until      . (       ,       .)

,    ,  steady_clock ( 2,  2.8).




3.5. condition_variable_any  stop_token


  1   : ,    ,     ,    .  condition variable  C++20   . std::condition_variable_any     condition_variable,       , ,    stop_token:










 3.7    stop_token



  ,       .     :   false, ,    ( ),   .

    stop_callback,   condition variable  request_stop()    ,         1.     condition_variable_any    condition_variable (    ).     ,        .




3.6.  


        :    ,   .    ,  :





































 3.8  -      (ex02_blocking_queue.cpp)



   :

 pop  std::optional<T>.             (  2).

 push  , close  .      ;     .

  close   .  ,      ,  pop  nullopt,      .      .

 pop  stop_token.   ,       .

       .       ;  ,    join,  .

     :  .    ,  ,    .        :   ,     ,     OOM killer.  ,    ,     3.2,       (backpressure)    12.




3.7. Graceful shutdown:    


       . ,     ,    (      SIGKILL)    ,   ,   .  ,   ,    ,   -  .

     ?  ? :

1.    .   (   ).

2.  . close()   .

3.  .   ,    ,  ,  pop  nullopt.

4.  . join (     jthread).

5.      ,   .

       ,   3  stop_token:   ,  .   ,  :     ,     .

  .       ,    (push  false),      .    join   use-after-free,       1.




3.8.   


 wait   (  if,   while).

    ,    .

 notify_one    ,     .

 notify_all      .

 ,          .

 wait_for        .

 ,      .

       .

 sleep_for  : ,   .




3.9.    


 condition variable  ? (middle)          .    lost wakeup.   , cv  .

  spurious wakeup     ? (middle)   notify,  .        ( wait  ).

  lost wakeup? (middle/senior) , ,     . ,      ,   .  :       , .

notify_one  notify_all? (middle) notify_one,      ; notify_all,     (, ). : thundering herd.

    ? (senior)  ;    (    );     cv       .

 wait_until  wait_for? (senior)         .      .

    ? (senior)   ,  ,  ,   (   stop_token),  ,   .   ,    .




3.10.   shardkv,  3: TTL  ,     


 :

  EXPIRE  PEXPIRE (     ), TTL  PTTL ( )   Redis: ?2   , ?1     ;

 SET   , INCR ;

  :      ,    ;

  :     ,       .     TTL,     ,     .

              .      7,    :         ,       3.2.




   


        .     :      ,      .  :

    - (std::priority_queue),    ;

    wait_until    ;

      , expire()    notify,     ;

     ,    :     ;

   stop_token ,        .

:     .   TTL,      SET (TTL )    TTL.     priority_queue ,        :        expires_at .      .




 






































 3.9   shardkv,  3 (storage.hpp)























































 3.10   (storage.cpp)



   :

 run_expirer     ,  . wait_until     ,    .   (purge_expired_locked)   ,      ,   .  Redis   ,     ;      ,     .

 expire          notify   .

   wait_until     ,    .       ,     : purge,   .

 get   const.    ,   get   .  map_ mutable   const   : const-,    ,     ,   ,  ,    .




  


    EXPIRE, PEXPIRE, TTL, PTTL (     ).  main.cpp   :










 3.11    storage  expirer (main.cpp)



      ,    1:     ,  .











































 3.12   (test_ttl.cpp)



    eventually. ,   ,    flaky-.  : sleep_for(100ms),  .   CI- 100   ,   .  :       .   ,    ;   ,  ,    .

 EarlierDeadlineWakesExpirer    :    ,    ,    10 .  expire   ,    .  ExpirerStopsPromptly ,  stop_token  ,    .

    TSan  ASan 10  .







  TTL     GET  SET?   2   3   kvbench (4 ,  mix,   ):










 3.1    TTL:  2   3, ops/s



:    ,          ( 102  141 ). ,       ,  TTL - .     .       ,  141  106       +33%.              .




   





 3.1 ().  




















 3.13     3.1 (task3_1_lost_wakeup.cpp)



  .     ,       .  , notify      ?

     (   3.2),    .      (  )  .

:    100 000   , 10  ,  TSan;    ,    notify.




 3.2 (). BoundedQueue


     BoundedQueue<T>  ,   :










 3.14   BoundedQueue<T>



:

 push ,   ,  ,   ;

  close()    ,  pop  nullopt;

 ,   ,  close()    false;

 ,  condition variable    .

:     (FIFO   ),    ,   try_pop_for,       ; - 4  4 ,  8,      ;   TSan 10  .




 3.3 (). -  


     ping  pong, N  ,  : ping, pong, ping, pong

 : K    0..K?1       (0, 1, 2, , K?1, 0, 1, ),  N .

:    (sleep       ),   .

:           ;  K = 8, N = 10 000  CPU     ( ,   ); TSan . ,      notify_all   notify_one,  ,         .

              .




 4.   :       std::mutex


 ,  ,      .     -.

      : , ,  .  ,    .   , ,   ,      ,      ;     ,    ;   SIGTERM          malloc.

       : , , , .   ,              C++ ( 5),    ,    senior-          8 .

   :

 ,         -,   ;

     false sharing;

 ,   store buffer       ;

 ,     Linux (futex)    ;

            ;

       ,   ;

 ,   fork()   ;

       ;

   shardkv        SIGTERM.




4.1.   -


      100 .          .        :

 L1     ,  ,    ;

 L2      ,      ,   ;

 L3      ,  ,   ;

   ,  .

,   :       -.  x86-64   ARM  64  ( Apple M-  128).     ,    ,    .     ,         .

  :



















 4.1     -,    (ex02_cache_stride.cpp)



  :










 4.2      



  1   64    64  ,      4 .       ,    -,       . (  64   ,   :     ,     .    ,    .)

   : ,   ,    ( , std::vector  std::list).       ,   .




4.2.    false sharing


     ,  ,          ?   :       .  x86      MESI.          :

 Modified          ;

 Exclusive       ,  ;

 Shared      ,   ;

 Invalid   .

   ,      Modified,          .        ,     :       .   ping-pong -.

 :          .     ,     .    false sharing ( ):




























 4.3       (ex01_false_sharing.cpp)



   :










 4.4  false sharing:    



 ,    ,       ,     3,46 .    ,    ,   -: alignas(64)   padding.




hardware_destructive_interference_size


 C++17   std::hardware_destructive_interference_size  <new>     ,   false sharing.  :

       (  __cpp_lib_hardware_interference_size);

 GCC     (-Winterference-size):     -mtune/-mcpu,      ABI (,      ),     .

       :










 4.5   -  ,    (cacheline.hpp)




  false sharing


      : , ,  ;

           ( ,          4);

       ;

      ,    (   ,  std::shared_ptr    ).

     :    64   8    8        .   ,     .




4.3. Store buffer:    


        :      .   ,       store buffer   ,      .     (    )  .

:          ,      .   (store-load reordering):










 4.6        



 :     r1, r2   1.    x86   r1 == 0  r2 == 0:      store buffer',      .         4.2.

    . x86  :    ,    . ARM  POWER   .  ,     x86,     ARM-  Apple Silicon.

       .   C++   :  ,     (memory_order),        .      .     5.




4.4.     Linux: futex


   ,     ,  ?  Linux   futex (fast userspace mutex). :

          (0  , 1  , 2     );

   :    compare-and-swap 0 ? 1,    .   ;

   :  CAS  ,  (    )    futex(FUTEX_WAIT),    ;

 :   0;       futex(FUTEX_WAKE),   .

:



















 4.7          (ex03_mutex_cost.cpp)



(   ;          .)      :       23 ,        .   ,     ,   :  ,    .

   :    ,     .  ,     ,  ,    .    ,     4 .




4.5.    


 .    p  ,   (1 ? p)  ,     N :

S(N) = 1 / ((1  p) + p / N)

 95%  ,         1 / 0,05 = 20.   90%    10.    (       !)  ,      .

 :    shardkv   2    .       .    ,       .

       :           ( ,  ),      .       .

  ,     . :      ? :  ?       ?        ,       .




4.6. NUMA


        :      .     (   )  .   NUMA (non-uniform memory access).

  :

 Linux        ,       (first touch).      ,      ,            ;

 ,      ,   ,     ;

 false sharing     ,    .

: numactl     , lscpu  numactl --hardware   .        NUMA-  (  ),     .         .




4.7. ,     


 C++        .   ,  native_handle()     ( Linux  pthread_t),     API:



















 4.8       native_handle() (ex04_affinity.cpp)



  (affinity) :

      :           (  real-time    12);

  thread-per-core,        (  12);

 NUMA:      .

       .  Linux    .        ,    . ,   +10%  ,   ?50%  ,     .

 :    Linux (SCHED_OTHER)    ; real-time  (SCHED_FIFO, SCHED_RR) ,         real-time       .    12.




4.8. fork()   


fork()   .          ,   fork().     .  ,    ,   .

 :

1.  A    malloc (    ).

2.  B     fork().

3.      ,   A,    ,  .

4.    malloc   .

     : ,  ,   .

:     fork()       async-signal-safe    exec().        exec.     ,  posix_spawn()      .   fork  exec (,   ,   Redis),         ,      ,    .




4.9.   


   UNIX,    .       .

  . -  (, SIGTERM  kill)   ,     .     .

     . ,  malloc       .        async-signal-safe  (    man signal-safety):  malloc,  printf,  .     ,     ,   :    ,    .

    .  :     volatile std::sig_atomic_t   lock-free std::atomic,  .      volatile    (   5).




 :   


     :

1.    main,    ,     pthread_sigmask.   ,   ,  ,   ;

2.   ,     sigwait ( sigtimedwait);

3. sigwait       ,     :   , , ;

4.           , request_stop()   stop_source.




























 4.9      (ex05_signal_thread.cpp)



 sigtimedwait,   sigwait:      ,       (, -  ). sigwait   ,   jthread    join.

            stop_token.   :  -    ,   .




4.10.   


          .

             .

 ,          ( x86  ).

        ,   .

         .

 fork()   exec()   .

 printf, malloc,      .

    signal()      .




4.11.    


  false sharing    ? (middle/senior)         -,    .    (perf c2c  Linux)      ,   .     .

  -      ? (middle)       ( 64 );     ,       .

      ? (senior) Store buffer,  ,  . x86  store-load, ARM  .   C++  .

  ? (senior)         (futex),        .   ,    .




  .


   .

   ,     (https://www.litres.ru/book/aleksey-primak/mnogopotochnost-na-c-cherez-praktiku-i-zamery-74702598/)  .

      Visa, MasterCard, Maestro,    ,   ,     ,  PayPal, WebMoney, ., QIWI ,       .


