#include <psxtypes/psxtypes.h>
static __inline__ void at_locked_inc(
intptr_t volatile * ptr)
{
__asm__(
"lock;"
"incl %0"
: "=m" (*ptr)
: "m" (*ptr)
: "memory");
}
static __inline__ void at_locked_inc_32(
int32_t volatile * ptr)
{
__asm__(
"lock;"
"incl %0"
: "=m" (*ptr)
: "m" (*ptr)
: "memory");
}
static __inline__ void at_locked_inc_64(
int64_t volatile * ptr)
{
__sync_fetch_and_add(ptr,1);
}
static __inline__ void at_locked_dec(
intptr_t volatile * ptr)
{
__asm__(
"lock;"
"decl %0"
: "=m" (*ptr)
: "m" (*ptr)
: "memory");
}
static __inline__ void at_locked_dec_32(
int32_t volatile * ptr)
{
__asm__(
"lock;"
"decl %0"
: "=m" (*ptr)
: "m" (*ptr)
: "memory");
}
static __inline__ void at_locked_dec_64(
int64_t volatile * ptr)
{
__sync_fetch_and_sub(ptr,1);
}
static __inline__ void at_locked_add(
intptr_t volatile * ptr,
intptr_t val)
{
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
}
static __inline__ void at_locked_add_32(
int32_t volatile * ptr,
int32_t val)
{
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
}
static __inline__ void at_locked_add_64(
int64_t volatile * ptr,
int64_t val)
{
__sync_fetch_and_add(ptr,val);
}
static __inline__ void at_locked_sub(
intptr_t volatile * ptr,
intptr_t val)
{
val = -val;
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
}
static __inline__ void at_locked_sub_32(
int32_t volatile * ptr,
int32_t val)
{
val = -val;
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
}
static __inline__ void at_locked_sub_64(
int64_t volatile * ptr,
int64_t val)
{
__sync_fetch_and_sub(ptr,val);
}
static __inline__ intptr_t at_locked_xadd(
intptr_t volatile * ptr,
intptr_t val)
{
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
return val;
}
static __inline__ int32_t at_locked_xadd_32(
int32_t volatile * ptr,
int32_t val)
{
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
return val;
}
static __inline__ int64_t at_locked_xadd_64(
int64_t volatile * ptr,
int64_t val)
{
return __sync_fetch_and_add(ptr,val);
}
static __inline__ intptr_t at_locked_xsub(
intptr_t volatile * ptr,
intptr_t val)
{
val = -val;
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
return val;
}
static __inline__ int32_t at_locked_xsub_32(
int32_t volatile * ptr,
int32_t val)
{
val = -val;
__asm__(
"lock;"
"xaddl %1, %0"
: "=m" (*ptr), "=r" (val)
: "1" (val)
: "memory");
return val;
}
static __inline__ int64_t at_locked_xsub_64(
int64_t volatile * ptr,
int64_t val)
{
return __sync_fetch_and_sub(ptr,val);
}
static __inline__ intptr_t at_locked_cas(
intptr_t volatile * dst,
intptr_t cmp,
intptr_t xchg)
{
intptr_t ret;
__asm__(
"lock;"
"cmpxchg %3, %0"
: "=m" (*dst), "=a" (ret)
: "a" (cmp), "r" (xchg)
: "memory");
return ret;
}
static __inline__ int32_t at_locked_cas_32(
int32_t volatile * dst,
int32_t cmp,
int32_t xchg)
{
int32_t ret;
__asm__(
"lock;"
"cmpxchg %3, %0"
: "=m" (*dst), "=a" (ret)
: "a" (cmp), "r" (xchg)
: "memory");
return ret;
}
static __inline__ int64_t at_locked_cas_64(
int64_t volatile * dst,
int64_t cmp,
int64_t xchg)
{
unsigned edx, eax;
unsigned ecx, ebx;
eax = (unsigned)cmp;
edx = (uint64_t)cmp >> 32;
ebx = (unsigned)xchg;
ecx = (uint64_t)xchg >> 32;
__asm__ volatile (
"lock;"
"cmpxchg8b %6"
: "=a" (eax), "=d" (edx)
: "a" (eax), "d" (edx), "b" (ebx), "c" (ecx), "m" (*dst)
: "memory");
return ((int64_t)edx << 32) + eax;
}
static __inline__ void at_locked_and(
intptr_t volatile * dst,
intptr_t mask)
{
__asm__(
"lock;"
"andl %1, %0"
: "=m" (*dst)
: "r" (mask)
: "memory");
}
static __inline__ void at_locked_and_32(
int32_t volatile * dst,
int32_t mask)
{
__asm__(
"lock;"
"andl %1, %0"
: "=m" (*dst)
: "r" (mask)
: "memory");
}
static __inline__ void at_locked_and_64(
int64_t volatile * dst,
int64_t mask)
{
int64_t ret;
int64_t cmp;
int64_t xchg;
do {
cmp = *dst;
xchg = cmp & mask;
ret = at_locked_cas_64(dst,cmp,xchg);
} while (ret != cmp);
}
static __inline__ void at_locked_or(
intptr_t volatile * dst,
intptr_t mask)
{
__asm__(
"lock;"
"orl %1, %0"
: "=m" (*dst)
: "r" (mask)
: "memory");
}
static __inline__ void at_locked_or_32(
int32_t volatile * dst,
int32_t mask)
{
__asm__(
"lock;"
"orl %1, %0"
: "=m" (*dst)
: "r" (mask)
: "memory");
}
static __inline__ void at_locked_or_64(
int64_t volatile * dst,
int64_t mask)
{
int64_t ret;
int64_t cmp;
int64_t xchg;
do {
cmp = *dst;
xchg = cmp | mask;
ret = at_locked_cas_64(dst,cmp,xchg);
} while (ret != cmp);
}
static __inline__ void at_locked_xor(
intptr_t volatile * dst,
intptr_t mask)
{
__asm__(
"lock;"
"xorl %1, %0"
: "=m" (*dst)
: "r" (mask)
: "memory");
}
static __inline__ void at_locked_xor_32(
int32_t volatile * dst,
int32_t mask)
{
__asm__(
"lock;"
"xorl %1, %0"
: "=m" (*dst)
: "r" (mask)
: "memory");
}
static __inline__ void at_locked_xor_64(
int64_t volatile * dst,
int64_t mask)
{
int64_t ret;
int64_t cmp;
int64_t xchg;
do {
cmp = *dst;
xchg = cmp ^ mask;
ret = at_locked_cas_64(dst,cmp,xchg);
} while (ret != cmp);
}
static __inline__ void at_store(
intptr_t volatile * dst,
intptr_t val)
{
__asm__(
"mov %1, %0;"
"lock;"
"orl $0,%0;"
: "=m" (*dst)
: "r" (val)
: "memory");
}
static __inline__ void at_store_32(
int32_t volatile * dst,
int32_t val)
{
__asm__(
"mov %1, %0;"
"lock;"
"orl $0,%0;"
: "=m" (*dst)
: "r" (val)
: "memory");
}
static __inline__ void at_store_64(
int64_t volatile * dst,
int64_t xchg)
{
int64_t cmp;
int64_t prev;
do {
prev = *dst;
cmp = prev;
prev = at_locked_cas_64(
dst,cmp,xchg);
} while (cmp != prev);
}
static __inline__ int at_bsf(
unsigned int * index,
uintptr_t mask)
{
if (mask) {
__asm__(
"bsf %1, %0"
: "=r" (mask)
: "r" (mask));
*index = (int)mask;
return 1;
} else
return 0;
}
static __inline__ int at_bsr(
unsigned int * index,
uintptr_t mask)
{
if (mask) {
__asm__(
"bsr %1, %0"
: "=r" (mask)
: "r" (mask));
*index = (int)mask;
return 1;
} else
return 0;
}
static __inline__ size_t at_popcount(
uintptr_t mask)
{
__asm__(
"popcnt %0, %0"
: "=r" (mask)
: "0" (mask)
: "memory");
return mask;
}
static __inline__ size_t at_popcount_16(
uint16_t mask)
{
__asm__(
"popcnt %0, %0"
: "=r" (mask)
: "0" (mask)
: "memory");
return mask;
}
static __inline__ size_t at_popcount_32(
uint32_t mask)
{
__asm__(
"popcnt %0, %0"
: "=r" (mask)
: "0" (mask)
: "memory");
return mask;
}
static __inline__ size_t at_popcount_64(
uint64_t mask)
{
int ret = at_popcount_32(mask >> 32);
return ret + ((mask << 32) >> 32);
}