extra_ops_msvc_x86.hpp 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336
  1. /*
  2. * Distributed under the Boost Software License, Version 1.0.
  3. * (See accompanying file LICENSE_1_0.txt or copy at
  4. * http://www.boost.org/LICENSE_1_0.txt)
  5. *
  6. * Copyright (c) 2017 Andrey Semashev
  7. */
  8. /*!
  9. * \file atomic/detail/extra_ops_msvc_x86.hpp
  10. *
  11. * This header contains implementation of the extra atomic operations for x86.
  12. */
  13. #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_
  14. #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_
  15. #include <cstddef>
  16. #include <boost/memory_order.hpp>
  17. #include <boost/atomic/detail/config.hpp>
  18. #include <boost/atomic/detail/interlocked.hpp>
  19. #include <boost/atomic/detail/storage_traits.hpp>
  20. #include <boost/atomic/detail/extra_operations_fwd.hpp>
  21. #include <boost/atomic/detail/extra_ops_generic.hpp>
  22. #include <boost/atomic/capabilities.hpp>
  23. #ifdef BOOST_HAS_PRAGMA_ONCE
  24. #pragma once
  25. #endif
  26. #if defined(BOOST_MSVC)
  27. #pragma warning(push)
  28. // frame pointer register 'ebx' modified by inline assembly code
  29. #pragma warning(disable: 4731)
  30. #endif
  31. namespace boost {
  32. namespace atomics {
  33. namespace detail {
  34. #if defined(_M_IX86)
  35. template< typename Base, bool Signed >
  36. struct extra_operations< Base, 1u, Signed, true > :
  37. public generic_extra_operations< Base, 1u, Signed >
  38. {
  39. typedef generic_extra_operations< Base, 1u, Signed > base_type;
  40. typedef typename base_type::storage_type storage_type;
  41. static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  42. {
  43. base_type::fence_before(order);
  44. storage_type old_val;
  45. __asm
  46. {
  47. mov ecx, storage
  48. movzx eax, byte ptr [ecx]
  49. align 16
  50. again:
  51. mov edx, eax
  52. neg dl
  53. lock cmpxchg byte ptr [ecx], dl
  54. jne again
  55. mov old_val, al
  56. };
  57. base_type::fence_after(order);
  58. return old_val;
  59. }
  60. static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  61. {
  62. base_type::fence_before(order);
  63. storage_type new_val;
  64. __asm
  65. {
  66. mov ecx, storage
  67. movzx eax, byte ptr [ecx]
  68. align 16
  69. again:
  70. mov edx, eax
  71. neg dl
  72. lock cmpxchg byte ptr [ecx], dl
  73. jne again
  74. mov new_val, dl
  75. };
  76. base_type::fence_after(order);
  77. return new_val;
  78. }
  79. static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  80. {
  81. base_type::fence_before(order);
  82. bool result;
  83. __asm
  84. {
  85. mov ecx, storage
  86. movzx eax, byte ptr [ecx]
  87. align 16
  88. again:
  89. mov edx, eax
  90. neg dl
  91. lock cmpxchg byte ptr [ecx], dl
  92. jne again
  93. test dl, dl
  94. setnz result
  95. };
  96. base_type::fence_after(order);
  97. return result;
  98. }
  99. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  100. {
  101. base_type::fence_before(order);
  102. __asm
  103. {
  104. mov ecx, storage
  105. movzx eax, byte ptr [ecx]
  106. align 16
  107. again:
  108. mov edx, eax
  109. neg dl
  110. lock cmpxchg byte ptr [ecx], dl
  111. jne again
  112. };
  113. base_type::fence_after(order);
  114. }
  115. static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  116. {
  117. base_type::fence_before(order);
  118. __asm
  119. {
  120. mov edi, storage
  121. movzx ecx, v
  122. xor edx, edx
  123. movzx eax, byte ptr [edi]
  124. align 16
  125. again:
  126. mov dl, al
  127. and dl, cl
  128. lock cmpxchg byte ptr [edi], dl
  129. jne again
  130. mov v, dl
  131. };
  132. base_type::fence_after(order);
  133. return v;
  134. }
  135. static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  136. {
  137. base_type::fence_before(order);
  138. __asm
  139. {
  140. mov edi, storage
  141. movzx ecx, v
  142. xor edx, edx
  143. movzx eax, byte ptr [edi]
  144. align 16
  145. again:
  146. mov dl, al
  147. or dl, cl
  148. lock cmpxchg byte ptr [edi], dl
  149. jne again
  150. mov v, dl
  151. };
  152. base_type::fence_after(order);
  153. return v;
  154. }
  155. static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  156. {
  157. base_type::fence_before(order);
  158. __asm
  159. {
  160. mov edi, storage
  161. movzx ecx, v
  162. xor edx, edx
  163. movzx eax, byte ptr [edi]
  164. align 16
  165. again:
  166. mov dl, al
  167. xor dl, cl
  168. lock cmpxchg byte ptr [edi], dl
  169. jne again
  170. mov v, dl
  171. };
  172. base_type::fence_after(order);
  173. return v;
  174. }
  175. static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  176. {
  177. base_type::fence_before(order);
  178. storage_type old_val;
  179. __asm
  180. {
  181. mov ecx, storage
  182. movzx eax, byte ptr [ecx]
  183. align 16
  184. again:
  185. mov edx, eax
  186. not dl
  187. lock cmpxchg byte ptr [ecx], dl
  188. jne again
  189. mov old_val, al
  190. };
  191. base_type::fence_after(order);
  192. return old_val;
  193. }
  194. static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  195. {
  196. base_type::fence_before(order);
  197. storage_type new_val;
  198. __asm
  199. {
  200. mov ecx, storage
  201. movzx eax, byte ptr [ecx]
  202. align 16
  203. again:
  204. mov edx, eax
  205. not dl
  206. lock cmpxchg byte ptr [ecx], dl
  207. jne again
  208. mov new_val, dl
  209. };
  210. base_type::fence_after(order);
  211. return new_val;
  212. }
  213. static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  214. {
  215. base_type::fence_before(order);
  216. bool result;
  217. __asm
  218. {
  219. mov ecx, storage
  220. movzx eax, byte ptr [ecx]
  221. align 16
  222. again:
  223. mov edx, eax
  224. not dl
  225. lock cmpxchg byte ptr [ecx], dl
  226. jne again
  227. test dl, dl
  228. setnz result
  229. };
  230. base_type::fence_after(order);
  231. return result;
  232. }
  233. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  234. {
  235. base_type::fence_before(order);
  236. __asm
  237. {
  238. mov ecx, storage
  239. movzx eax, byte ptr [ecx]
  240. align 16
  241. again:
  242. mov edx, eax
  243. not dl
  244. lock cmpxchg byte ptr [ecx], dl
  245. jne again
  246. };
  247. base_type::fence_after(order);
  248. }
  249. static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  250. {
  251. base_type::fence_before(order);
  252. __asm
  253. {
  254. mov edx, storage
  255. movzx eax, v
  256. lock add byte ptr [edx], al
  257. };
  258. base_type::fence_after(order);
  259. }
  260. static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  261. {
  262. base_type::fence_before(order);
  263. __asm
  264. {
  265. mov edx, storage
  266. movzx eax, v
  267. lock sub byte ptr [edx], al
  268. };
  269. base_type::fence_after(order);
  270. }
  271. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  272. {
  273. base_type::fence_before(order);
  274. __asm
  275. {
  276. mov edx, storage
  277. lock neg byte ptr [edx]
  278. };
  279. base_type::fence_after(order);
  280. }
  281. static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  282. {
  283. base_type::fence_before(order);
  284. __asm
  285. {
  286. mov edx, storage
  287. movzx eax, v
  288. lock and byte ptr [edx], al
  289. };
  290. base_type::fence_after(order);
  291. }
  292. static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  293. {
  294. base_type::fence_before(order);
  295. __asm
  296. {
  297. mov edx, storage
  298. movzx eax, v
  299. lock or byte ptr [edx], al
  300. };
  301. base_type::fence_after(order);
  302. }
  303. static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  304. {
  305. base_type::fence_before(order);
  306. __asm
  307. {
  308. mov edx, storage
  309. movzx eax, v
  310. lock xor byte ptr [edx], al
  311. };
  312. base_type::fence_after(order);
  313. }
  314. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  315. {
  316. base_type::fence_before(order);
  317. __asm
  318. {
  319. mov edx, storage
  320. lock not byte ptr [edx]
  321. };
  322. base_type::fence_after(order);
  323. }
  324. static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  325. {
  326. base_type::fence_before(order);
  327. bool result;
  328. __asm
  329. {
  330. mov edx, storage
  331. movzx eax, v
  332. lock add byte ptr [edx], al
  333. setnz result
  334. };
  335. base_type::fence_after(order);
  336. return result;
  337. }
  338. static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  339. {
  340. base_type::fence_before(order);
  341. bool result;
  342. __asm
  343. {
  344. mov edx, storage
  345. movzx eax, v
  346. lock sub byte ptr [edx], al
  347. setnz result
  348. };
  349. base_type::fence_after(order);
  350. return result;
  351. }
  352. static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  353. {
  354. base_type::fence_before(order);
  355. bool result;
  356. __asm
  357. {
  358. mov edx, storage
  359. movzx eax, v
  360. lock and byte ptr [edx], al
  361. setnz result
  362. };
  363. base_type::fence_after(order);
  364. return result;
  365. }
  366. static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  367. {
  368. base_type::fence_before(order);
  369. bool result;
  370. __asm
  371. {
  372. mov edx, storage
  373. movzx eax, v
  374. lock or byte ptr [edx], al
  375. setnz result
  376. };
  377. base_type::fence_after(order);
  378. return result;
  379. }
  380. static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  381. {
  382. base_type::fence_before(order);
  383. bool result;
  384. __asm
  385. {
  386. mov edx, storage
  387. movzx eax, v
  388. lock xor byte ptr [edx], al
  389. setnz result
  390. };
  391. base_type::fence_after(order);
  392. return result;
  393. }
  394. };
  395. template< typename Base, bool Signed >
  396. struct extra_operations< Base, 2u, Signed, true > :
  397. public generic_extra_operations< Base, 2u, Signed >
  398. {
  399. typedef generic_extra_operations< Base, 2u, Signed > base_type;
  400. typedef typename base_type::storage_type storage_type;
  401. static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  402. {
  403. base_type::fence_before(order);
  404. storage_type old_val;
  405. __asm
  406. {
  407. mov ecx, storage
  408. movzx eax, word ptr [ecx]
  409. align 16
  410. again:
  411. mov edx, eax
  412. neg dx
  413. lock cmpxchg word ptr [ecx], dx
  414. jne again
  415. mov old_val, ax
  416. };
  417. base_type::fence_after(order);
  418. return old_val;
  419. }
  420. static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  421. {
  422. base_type::fence_before(order);
  423. storage_type new_val;
  424. __asm
  425. {
  426. mov ecx, storage
  427. movzx eax, word ptr [ecx]
  428. align 16
  429. again:
  430. mov edx, eax
  431. neg dx
  432. lock cmpxchg word ptr [ecx], dx
  433. jne again
  434. mov new_val, dx
  435. };
  436. base_type::fence_after(order);
  437. return new_val;
  438. }
  439. static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  440. {
  441. base_type::fence_before(order);
  442. bool result;
  443. __asm
  444. {
  445. mov ecx, storage
  446. movzx eax, word ptr [ecx]
  447. align 16
  448. again:
  449. mov edx, eax
  450. neg dx
  451. lock cmpxchg word ptr [ecx], dx
  452. jne again
  453. test dx, dx
  454. setnz result
  455. };
  456. base_type::fence_after(order);
  457. return result;
  458. }
  459. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  460. {
  461. base_type::fence_before(order);
  462. __asm
  463. {
  464. mov ecx, storage
  465. movzx eax, word ptr [ecx]
  466. align 16
  467. again:
  468. mov edx, eax
  469. neg dx
  470. lock cmpxchg word ptr [ecx], dx
  471. jne again
  472. };
  473. base_type::fence_after(order);
  474. }
  475. static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  476. {
  477. base_type::fence_before(order);
  478. __asm
  479. {
  480. mov edi, storage
  481. movzx ecx, v
  482. xor edx, edx
  483. movzx eax, word ptr [edi]
  484. align 16
  485. again:
  486. mov dx, ax
  487. and dx, cx
  488. lock cmpxchg word ptr [edi], dx
  489. jne again
  490. mov v, dx
  491. };
  492. base_type::fence_after(order);
  493. return v;
  494. }
  495. static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  496. {
  497. base_type::fence_before(order);
  498. __asm
  499. {
  500. mov edi, storage
  501. movzx ecx, v
  502. xor edx, edx
  503. movzx eax, word ptr [edi]
  504. align 16
  505. again:
  506. mov dx, ax
  507. or dx, cx
  508. lock cmpxchg word ptr [edi], dx
  509. jne again
  510. mov v, dx
  511. };
  512. base_type::fence_after(order);
  513. return v;
  514. }
  515. static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  516. {
  517. base_type::fence_before(order);
  518. __asm
  519. {
  520. mov edi, storage
  521. movzx ecx, v
  522. xor edx, edx
  523. movzx eax, word ptr [edi]
  524. align 16
  525. again:
  526. mov dx, ax
  527. xor dx, cx
  528. lock cmpxchg word ptr [edi], dx
  529. jne again
  530. mov v, dx
  531. };
  532. base_type::fence_after(order);
  533. return v;
  534. }
  535. static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  536. {
  537. base_type::fence_before(order);
  538. storage_type old_val;
  539. __asm
  540. {
  541. mov ecx, storage
  542. movzx eax, word ptr [ecx]
  543. align 16
  544. again:
  545. mov edx, eax
  546. not dx
  547. lock cmpxchg word ptr [ecx], dx
  548. jne again
  549. mov old_val, ax
  550. };
  551. base_type::fence_after(order);
  552. return old_val;
  553. }
  554. static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  555. {
  556. base_type::fence_before(order);
  557. storage_type new_val;
  558. __asm
  559. {
  560. mov ecx, storage
  561. movzx eax, word ptr [ecx]
  562. align 16
  563. again:
  564. mov edx, eax
  565. not dx
  566. lock cmpxchg word ptr [ecx], dx
  567. jne again
  568. mov new_val, dx
  569. };
  570. base_type::fence_after(order);
  571. return new_val;
  572. }
  573. static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  574. {
  575. base_type::fence_before(order);
  576. bool result;
  577. __asm
  578. {
  579. mov ecx, storage
  580. movzx eax, word ptr [ecx]
  581. align 16
  582. again:
  583. mov edx, eax
  584. not dx
  585. lock cmpxchg word ptr [ecx], dx
  586. jne again
  587. test dx, dx
  588. setnz result
  589. };
  590. base_type::fence_after(order);
  591. return result;
  592. }
  593. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  594. {
  595. base_type::fence_before(order);
  596. __asm
  597. {
  598. mov ecx, storage
  599. movzx eax, word ptr [ecx]
  600. align 16
  601. again:
  602. mov edx, eax
  603. not dx
  604. lock cmpxchg word ptr [ecx], dx
  605. jne again
  606. };
  607. base_type::fence_after(order);
  608. }
  609. static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  610. {
  611. base_type::fence_before(order);
  612. __asm
  613. {
  614. mov edx, storage
  615. movzx eax, v
  616. lock add word ptr [edx], ax
  617. };
  618. base_type::fence_after(order);
  619. }
  620. static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  621. {
  622. base_type::fence_before(order);
  623. __asm
  624. {
  625. mov edx, storage
  626. movzx eax, v
  627. lock sub word ptr [edx], ax
  628. };
  629. base_type::fence_after(order);
  630. }
  631. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  632. {
  633. base_type::fence_before(order);
  634. __asm
  635. {
  636. mov edx, storage
  637. lock neg word ptr [edx]
  638. };
  639. base_type::fence_after(order);
  640. }
  641. static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  642. {
  643. base_type::fence_before(order);
  644. __asm
  645. {
  646. mov edx, storage
  647. movzx eax, v
  648. lock and word ptr [edx], ax
  649. };
  650. base_type::fence_after(order);
  651. }
  652. static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  653. {
  654. base_type::fence_before(order);
  655. __asm
  656. {
  657. mov edx, storage
  658. movzx eax, v
  659. lock or word ptr [edx], ax
  660. };
  661. base_type::fence_after(order);
  662. }
  663. static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  664. {
  665. base_type::fence_before(order);
  666. __asm
  667. {
  668. mov edx, storage
  669. movzx eax, v
  670. lock xor word ptr [edx], ax
  671. };
  672. base_type::fence_after(order);
  673. }
  674. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  675. {
  676. base_type::fence_before(order);
  677. __asm
  678. {
  679. mov edx, storage
  680. lock not word ptr [edx]
  681. };
  682. base_type::fence_after(order);
  683. }
  684. static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  685. {
  686. base_type::fence_before(order);
  687. bool result;
  688. __asm
  689. {
  690. mov edx, storage
  691. movzx eax, v
  692. lock add word ptr [edx], ax
  693. setnz result
  694. };
  695. base_type::fence_after(order);
  696. return result;
  697. }
  698. static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  699. {
  700. base_type::fence_before(order);
  701. bool result;
  702. __asm
  703. {
  704. mov edx, storage
  705. movzx eax, v
  706. lock sub word ptr [edx], ax
  707. setnz result
  708. };
  709. base_type::fence_after(order);
  710. return result;
  711. }
  712. static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  713. {
  714. base_type::fence_before(order);
  715. bool result;
  716. __asm
  717. {
  718. mov edx, storage
  719. movzx eax, v
  720. lock and word ptr [edx], ax
  721. setnz result
  722. };
  723. base_type::fence_after(order);
  724. return result;
  725. }
  726. static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  727. {
  728. base_type::fence_before(order);
  729. bool result;
  730. __asm
  731. {
  732. mov edx, storage
  733. movzx eax, v
  734. lock or word ptr [edx], ax
  735. setnz result
  736. };
  737. base_type::fence_after(order);
  738. return result;
  739. }
  740. static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  741. {
  742. base_type::fence_before(order);
  743. bool result;
  744. __asm
  745. {
  746. mov edx, storage
  747. movzx eax, v
  748. lock xor word ptr [edx], ax
  749. setnz result
  750. };
  751. base_type::fence_after(order);
  752. return result;
  753. }
  754. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  755. {
  756. base_type::fence_before(order);
  757. bool result;
  758. __asm
  759. {
  760. mov edx, storage
  761. mov eax, bit_number
  762. lock bts word ptr [edx], ax
  763. setc result
  764. };
  765. base_type::fence_after(order);
  766. return result;
  767. }
  768. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  769. {
  770. base_type::fence_before(order);
  771. bool result;
  772. __asm
  773. {
  774. mov edx, storage
  775. mov eax, bit_number
  776. lock btr word ptr [edx], ax
  777. setc result
  778. };
  779. base_type::fence_after(order);
  780. return result;
  781. }
  782. static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  783. {
  784. base_type::fence_before(order);
  785. bool result;
  786. __asm
  787. {
  788. mov edx, storage
  789. mov eax, bit_number
  790. lock btc word ptr [edx], ax
  791. setc result
  792. };
  793. base_type::fence_after(order);
  794. return result;
  795. }
  796. };
  797. #endif // defined(_M_IX86)
  798. #if defined(_M_IX86) || (defined(BOOST_ATOMIC_INTERLOCKED_BTS) && defined(BOOST_ATOMIC_INTERLOCKED_BTR))
  799. template< typename Base, bool Signed >
  800. struct extra_operations< Base, 4u, Signed, true > :
  801. public generic_extra_operations< Base, 4u, Signed >
  802. {
  803. typedef generic_extra_operations< Base, 4u, Signed > base_type;
  804. typedef typename base_type::storage_type storage_type;
  805. #if defined(_M_IX86)
  806. static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  807. {
  808. base_type::fence_before(order);
  809. storage_type old_val;
  810. __asm
  811. {
  812. mov ecx, storage
  813. mov eax, dword ptr [ecx]
  814. align 16
  815. again:
  816. mov edx, eax
  817. neg edx
  818. lock cmpxchg dword ptr [ecx], edx
  819. jne again
  820. mov old_val, eax
  821. };
  822. base_type::fence_after(order);
  823. return old_val;
  824. }
  825. static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  826. {
  827. base_type::fence_before(order);
  828. storage_type new_val;
  829. __asm
  830. {
  831. mov ecx, storage
  832. mov eax, dword ptr [ecx]
  833. align 16
  834. again:
  835. mov edx, eax
  836. neg edx
  837. lock cmpxchg dword ptr [ecx], edx
  838. jne again
  839. mov new_val, edx
  840. };
  841. base_type::fence_after(order);
  842. return new_val;
  843. }
  844. static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  845. {
  846. base_type::fence_before(order);
  847. bool result;
  848. __asm
  849. {
  850. mov ecx, storage
  851. mov eax, dword ptr [ecx]
  852. align 16
  853. again:
  854. mov edx, eax
  855. neg edx
  856. lock cmpxchg dword ptr [ecx], edx
  857. jne again
  858. test edx, edx
  859. setnz result
  860. };
  861. base_type::fence_after(order);
  862. return result;
  863. }
  864. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  865. {
  866. base_type::fence_before(order);
  867. __asm
  868. {
  869. mov ecx, storage
  870. mov eax, dword ptr [ecx]
  871. align 16
  872. again:
  873. mov edx, eax
  874. neg edx
  875. lock cmpxchg dword ptr [ecx], edx
  876. jne again
  877. };
  878. base_type::fence_after(order);
  879. }
  880. static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  881. {
  882. base_type::fence_before(order);
  883. __asm
  884. {
  885. mov edi, storage
  886. mov ecx, v
  887. xor edx, edx
  888. mov eax, dword ptr [edi]
  889. align 16
  890. again:
  891. mov edx, eax
  892. and edx, ecx
  893. lock cmpxchg dword ptr [edi], edx
  894. jne again
  895. mov v, edx
  896. };
  897. base_type::fence_after(order);
  898. return v;
  899. }
  900. static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  901. {
  902. base_type::fence_before(order);
  903. __asm
  904. {
  905. mov edi, storage
  906. mov ecx, v
  907. xor edx, edx
  908. mov eax, dword ptr [edi]
  909. align 16
  910. again:
  911. mov edx, eax
  912. or edx, ecx
  913. lock cmpxchg dword ptr [edi], edx
  914. jne again
  915. mov v, edx
  916. };
  917. base_type::fence_after(order);
  918. return v;
  919. }
  920. static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  921. {
  922. base_type::fence_before(order);
  923. __asm
  924. {
  925. mov edi, storage
  926. mov ecx, v
  927. xor edx, edx
  928. mov eax, dword ptr [edi]
  929. align 16
  930. again:
  931. mov edx, eax
  932. xor edx, ecx
  933. lock cmpxchg dword ptr [edi], edx
  934. jne again
  935. mov v, edx
  936. };
  937. base_type::fence_after(order);
  938. return v;
  939. }
  940. static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  941. {
  942. base_type::fence_before(order);
  943. storage_type old_val;
  944. __asm
  945. {
  946. mov ecx, storage
  947. mov eax, dword ptr [ecx]
  948. align 16
  949. again:
  950. mov edx, eax
  951. not edx
  952. lock cmpxchg dword ptr [ecx], edx
  953. jne again
  954. mov old_val, eax
  955. };
  956. base_type::fence_after(order);
  957. return old_val;
  958. }
  959. static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  960. {
  961. base_type::fence_before(order);
  962. storage_type new_val;
  963. __asm
  964. {
  965. mov ecx, storage
  966. mov eax, dword ptr [ecx]
  967. align 16
  968. again:
  969. mov edx, eax
  970. not edx
  971. lock cmpxchg dword ptr [ecx], edx
  972. jne again
  973. mov new_val, edx
  974. };
  975. base_type::fence_after(order);
  976. return new_val;
  977. }
  978. static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  979. {
  980. base_type::fence_before(order);
  981. bool result;
  982. __asm
  983. {
  984. mov ecx, storage
  985. mov eax, dword ptr [ecx]
  986. align 16
  987. again:
  988. mov edx, eax
  989. not edx
  990. lock cmpxchg dword ptr [ecx], edx
  991. jne again
  992. test edx, edx
  993. setnz result
  994. };
  995. base_type::fence_after(order);
  996. return result;
  997. }
  998. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  999. {
  1000. base_type::fence_before(order);
  1001. __asm
  1002. {
  1003. mov ecx, storage
  1004. mov eax, dword ptr [ecx]
  1005. align 16
  1006. again:
  1007. mov edx, eax
  1008. not edx
  1009. lock cmpxchg dword ptr [ecx], edx
  1010. jne again
  1011. };
  1012. base_type::fence_after(order);
  1013. }
  1014. static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1015. {
  1016. base_type::fence_before(order);
  1017. __asm
  1018. {
  1019. mov edx, storage
  1020. mov eax, v
  1021. lock add dword ptr [edx], eax
  1022. };
  1023. base_type::fence_after(order);
  1024. }
  1025. static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1026. {
  1027. base_type::fence_before(order);
  1028. __asm
  1029. {
  1030. mov edx, storage
  1031. mov eax, v
  1032. lock sub dword ptr [edx], eax
  1033. };
  1034. base_type::fence_after(order);
  1035. }
  1036. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  1037. {
  1038. base_type::fence_before(order);
  1039. __asm
  1040. {
  1041. mov edx, storage
  1042. lock neg dword ptr [edx]
  1043. };
  1044. base_type::fence_after(order);
  1045. }
  1046. static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1047. {
  1048. base_type::fence_before(order);
  1049. __asm
  1050. {
  1051. mov edx, storage
  1052. mov eax, v
  1053. lock and dword ptr [edx], eax
  1054. };
  1055. base_type::fence_after(order);
  1056. }
  1057. static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1058. {
  1059. base_type::fence_before(order);
  1060. __asm
  1061. {
  1062. mov edx, storage
  1063. mov eax, v
  1064. lock or dword ptr [edx], eax
  1065. };
  1066. base_type::fence_after(order);
  1067. }
  1068. static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1069. {
  1070. base_type::fence_before(order);
  1071. __asm
  1072. {
  1073. mov edx, storage
  1074. mov eax, v
  1075. lock xor dword ptr [edx], eax
  1076. };
  1077. base_type::fence_after(order);
  1078. }
  1079. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  1080. {
  1081. base_type::fence_before(order);
  1082. __asm
  1083. {
  1084. mov edx, storage
  1085. lock not dword ptr [edx]
  1086. };
  1087. base_type::fence_after(order);
  1088. }
  1089. static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1090. {
  1091. base_type::fence_before(order);
  1092. bool result;
  1093. __asm
  1094. {
  1095. mov edx, storage
  1096. mov eax, v
  1097. lock add dword ptr [edx], eax
  1098. setnz result
  1099. };
  1100. base_type::fence_after(order);
  1101. return result;
  1102. }
  1103. static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1104. {
  1105. base_type::fence_before(order);
  1106. bool result;
  1107. __asm
  1108. {
  1109. mov edx, storage
  1110. mov eax, v
  1111. lock sub dword ptr [edx], eax
  1112. setnz result
  1113. };
  1114. base_type::fence_after(order);
  1115. return result;
  1116. }
  1117. static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1118. {
  1119. base_type::fence_before(order);
  1120. bool result;
  1121. __asm
  1122. {
  1123. mov edx, storage
  1124. mov eax, v
  1125. lock and dword ptr [edx], eax
  1126. setnz result
  1127. };
  1128. base_type::fence_after(order);
  1129. return result;
  1130. }
  1131. static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1132. {
  1133. base_type::fence_before(order);
  1134. bool result;
  1135. __asm
  1136. {
  1137. mov edx, storage
  1138. mov eax, v
  1139. lock or dword ptr [edx], eax
  1140. setnz result
  1141. };
  1142. base_type::fence_after(order);
  1143. return result;
  1144. }
  1145. static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  1146. {
  1147. base_type::fence_before(order);
  1148. bool result;
  1149. __asm
  1150. {
  1151. mov edx, storage
  1152. mov eax, v
  1153. lock xor dword ptr [edx], eax
  1154. setnz result
  1155. };
  1156. base_type::fence_after(order);
  1157. return result;
  1158. }
  1159. static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  1160. {
  1161. base_type::fence_before(order);
  1162. bool result;
  1163. __asm
  1164. {
  1165. mov edx, storage
  1166. mov eax, bit_number
  1167. lock btc dword ptr [edx], eax
  1168. setc result
  1169. };
  1170. base_type::fence_after(order);
  1171. return result;
  1172. }
  1173. #endif // defined(_M_IX86)
  1174. #if defined(BOOST_ATOMIC_INTERLOCKED_BTS)
  1175. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
  1176. {
  1177. return !!BOOST_ATOMIC_INTERLOCKED_BTS(&storage, bit_number);
  1178. }
  1179. #elif defined(_M_IX86)
  1180. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  1181. {
  1182. base_type::fence_before(order);
  1183. bool result;
  1184. __asm
  1185. {
  1186. mov edx, storage
  1187. mov eax, bit_number
  1188. lock bts dword ptr [edx], eax
  1189. setc result
  1190. };
  1191. base_type::fence_after(order);
  1192. return result;
  1193. }
  1194. #endif
  1195. #if defined(BOOST_ATOMIC_INTERLOCKED_BTR)
  1196. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
  1197. {
  1198. return !!BOOST_ATOMIC_INTERLOCKED_BTR(&storage, bit_number);
  1199. }
  1200. #elif defined(_M_IX86)
  1201. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  1202. {
  1203. base_type::fence_before(order);
  1204. bool result;
  1205. __asm
  1206. {
  1207. mov edx, storage
  1208. mov eax, bit_number
  1209. lock btr dword ptr [edx], eax
  1210. setc result
  1211. };
  1212. base_type::fence_after(order);
  1213. return result;
  1214. }
  1215. #endif
  1216. };
  1217. #endif // defined(_M_IX86) || (defined(BOOST_ATOMIC_INTERLOCKED_BTS) && defined(BOOST_ATOMIC_INTERLOCKED_BTR))
  1218. #if defined(BOOST_ATOMIC_INTERLOCKED_BTS64) && defined(BOOST_ATOMIC_INTERLOCKED_BTR64)
  1219. template< typename Base, bool Signed >
  1220. struct extra_operations< Base, 8u, Signed, true > :
  1221. public generic_extra_operations< Base, 8u, Signed >
  1222. {
  1223. typedef generic_extra_operations< Base, 8u, Signed > base_type;
  1224. typedef typename base_type::storage_type storage_type;
  1225. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  1226. {
  1227. return !!BOOST_ATOMIC_INTERLOCKED_BTS64(&storage, bit_number);
  1228. }
  1229. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  1230. {
  1231. return !!BOOST_ATOMIC_INTERLOCKED_BTR64(&storage, bit_number);
  1232. }
  1233. };
  1234. #endif // defined(BOOST_ATOMIC_INTERLOCKED_BTS64) && defined(BOOST_ATOMIC_INTERLOCKED_BTR64)
  1235. } // namespace detail
  1236. } // namespace atomics
  1237. } // namespace boost
  1238. #if defined(BOOST_MSVC)
  1239. #pragma warning(pop)
  1240. #endif
  1241. #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_