From ca62e58b621c8423338a9fe404100726c4f134e6 Mon Sep 17 00:00:00 2001
From: midipix <writeonce@midipix.org>
Date: Sep 26 2015 15:16:39 +0000
Subject: + fix the atomic store functions to include a full memory barrier.

+ rewrite at_store_64 for i386 as a wrapper around a 64-bit cas operation.

---

diff --git a/include/ntapi/bits/i386/nt_atomic_i386_asm__gcc.h b/include/ntapi/bits/i386/nt_atomic_i386_asm__gcc.h
index 04c0a8f..23260e9 100644
--- a/include/ntapi/bits/i386/nt_atomic_i386_asm__gcc.h
+++ b/include/ntapi/bits/i386/nt_atomic_i386_asm__gcc.h
@@ -424,7 +424,9 @@ static __inline__ void at_store(
 	intptr_t		val)
 {
 	__asm__(
-		"mov %1, %0"
+		"mov %1, %0;"
+		"lock;"
+		"orl $0,%0;"
 		: "=m" (*dst)
 		: "r"  (val)
 		: "memory");
@@ -436,7 +438,9 @@ static __inline__ void at_store_32(
 	int32_t			val)
 {
 	__asm__(
-		"mov %1, %0"
+		"mov %1, %0;"
+		"lock;"
+		"orl $0,%0;"
 		: "=m" (*dst)
 		: "r"  (val)
 		: "memory");
@@ -445,15 +449,24 @@ static __inline__ void at_store_32(
 
 static __inline__ void at_store_64(
 	volatile int64_t *	dst,
-	int64_t			val)
+	int64_t			xchg)
 {
-	__asm__(
-		"mov %1, %0"
-		: "=m" (*dst)
-		: "r"  (val)
-		: "memory");
-}
+	int64_t	cmp;
+	int64_t prev;
 
+	do {
+		prev = *dst;
+		cmp  = prev;
+
+		__atomic_compare_exchange_n(
+			dst,
+			&cmp,
+			xchg,
+			0,
+			__ATOMIC_SEQ_CST,
+			__ATOMIC_SEQ_CST);
+	} while (cmp != prev);
+}
 
 static __inline__ int at_bsf(
 	unsigned int *		index,
diff --git a/include/ntapi/bits/x86_64/nt_atomic_x86_64_asm__gcc.h b/include/ntapi/bits/x86_64/nt_atomic_x86_64_asm__gcc.h
index 84b9575..d4d4eb0 100644
--- a/include/ntapi/bits/x86_64/nt_atomic_x86_64_asm__gcc.h
+++ b/include/ntapi/bits/x86_64/nt_atomic_x86_64_asm__gcc.h
@@ -458,7 +458,8 @@ static __inline__ void at_store(
 	intptr_t		val)
 {
 	__asm__(
-		"mov %1, %0"
+		"mov %1, %0;"
+		"mfence;"
 		: "=m" (*dst)
 		: "r"  (val)
 		: "memory");
@@ -470,7 +471,8 @@ static __inline__ void at_store_32(
 	int32_t			val)
 {
 	__asm__(
-		"mov %1, %0"
+		"mov %1, %0;"
+		"mfence;"
 		: "=m" (*dst)
 		: "r"  (val)
 		: "memory");
@@ -482,7 +484,8 @@ static __inline__ void at_store_64(
 	int64_t			val)
 {
 	__asm__(
-		"mov %1, %0"
+		"mov %1, %0;"
+		"mfence;"
 		: "=m" (*dst)
 		: "r"  (val)
 		: "memory");