Commit 46efdbbc by Ian Lance Taylor

runtime: rewrite lfstack packing/unpacking to look more like that in Go

    Reviewed-on: https://go-review.googlesource.com/13037

From-SVN: r228057
parent a9c23810
e069d4417a692c1261df99fe3323277e1a0193d2 2087b95180caea3477647c449772b7fecc01a71c
The first line of this file holds the git revision number of the last The first line of this file holds the git revision number of the last
merge done from the gofrontend repository. merge done from the gofrontend repository.
...@@ -9,25 +9,41 @@ package runtime ...@@ -9,25 +9,41 @@ package runtime
#include "arch.h" #include "arch.h"
#if __SIZEOF_POINTER__ == 8 #if __SIZEOF_POINTER__ == 8
// Amd64 uses 48-bit virtual addresses, 47-th bit is used as kernel/user flag.
// So we use 17msb of pointers as ABA counter.
# define PTR_BITS 47
#else
# define PTR_BITS 32
#endif
#define PTR_MASK ((1ull<<PTR_BITS)-1)
#define CNT_MASK (0ull-1)
#if __SIZEOF_POINTER__ == 8 && (defined(__sparc__) || (defined(__sun__) && defined(__amd64__)))
// SPARC64 and Solaris on AMD64 uses all 64 bits of virtual addresses. // SPARC64 and Solaris on AMD64 uses all 64 bits of virtual addresses.
// Use low-order three bits as ABA counter. // Use low-order three bits as ABA counter.
// http://docs.oracle.com/cd/E19120-01/open.solaris/816-5138/6mba6ua5p/index.html // http://docs.oracle.com/cd/E19120-01/open.solaris/816-5138/6mba6ua5p/index.html
#undef PTR_BITS # if defined(__sparc__) || (defined(__sun__) && defined(__amd64__))
#undef CNT_MASK static inline uint64 lfPack(LFNode *node, uintptr cnt) {
#undef PTR_MASK return ((uint64)(node)) | ((cnt)&7);
#define PTR_BITS 0 }
#define CNT_MASK 7 static inline LFNode* lfUnpack(uint64 val) {
#define PTR_MASK ((0ull-1)<<3) return (LFNode*)(val&~7);
}
# else
# if defined(__aarch64__)
// Depending on the kernel options, pointers on arm64 can have up to 48 significant
// bits (see https://www.kernel.org/doc/Documentation/arm64/memory.txt).
# define PTR_BITS 48
# else
// Amd64 uses 48-bit virtual addresses, 47-th bit is used as kernel/user flag.
// So we use 17msb of pointers as ABA counter.
# define PTR_BITS 47
# endif
# endif
# define CNT_BITS (64 - PTR_BITS + 3)
static inline uint64 lfPack(LFNode *node, uintptr cnt) {
return ((uint64)(node)<<(64-PTR_BITS)) | (cnt&(((1<<CNT_BITS)-1)));
}
static inline LFNode* lfUnpack(uint64 val) {
return (LFNode*)((val >> CNT_BITS) << 3);
}
#else
static inline uint64 lfPack(LFNode *node, uintptr cnt) {
return ((uint64)(uintptr)(node)<<32) | cnt;
}
static inline LFNode* lfUnpack(uint64 val) {
return (LFNode*)(uintptr)(val >> 32);
}
#endif #endif
void void
...@@ -35,16 +51,16 @@ runtime_lfstackpush(uint64 *head, LFNode *node) ...@@ -35,16 +51,16 @@ runtime_lfstackpush(uint64 *head, LFNode *node)
{ {
uint64 old, new; uint64 old, new;
if((uintptr)node != ((uintptr)node&PTR_MASK)) { if(node != lfUnpack(lfPack(node, 0))) {
runtime_printf("p=%p\n", node); runtime_printf("p=%p\n", node);
runtime_throw("runtime_lfstackpush: invalid pointer"); runtime_throw("runtime_lfstackpush: invalid pointer");
} }
node->pushcnt++; node->pushcnt++;
new = (uint64)(uintptr)node|(((uint64)node->pushcnt&CNT_MASK)<<PTR_BITS); new = lfPack(node, node->pushcnt);
for(;;) { for(;;) {
old = runtime_atomicload64(head); old = runtime_atomicload64(head);
node->next = (LFNode*)(uintptr)(old&PTR_MASK); node->next = lfUnpack(old);
if(runtime_cas64(head, old, new)) if(runtime_cas64(head, old, new))
break; break;
} }
...@@ -60,11 +76,11 @@ runtime_lfstackpop(uint64 *head) ...@@ -60,11 +76,11 @@ runtime_lfstackpop(uint64 *head)
old = runtime_atomicload64(head); old = runtime_atomicload64(head);
if(old == 0) if(old == 0)
return nil; return nil;
node = (LFNode*)(uintptr)(old&PTR_MASK); node = lfUnpack(old);
node2 = runtime_atomicloadp(&node->next); node2 = runtime_atomicloadp(&node->next);
new = 0; new = 0;
if(node2 != nil) if(node2 != nil)
new = (uint64)(uintptr)node2|(((uint64)node2->pushcnt&CNT_MASK)<<PTR_BITS); new = lfPack(node2, node2->pushcnt);
if(runtime_cas64(head, old, new)) if(runtime_cas64(head, old, new))
return node; return node;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment