fix cpu read benchmark
This commit is contained in:
1 parent
e528efc6d1
commit
e5765e70ab
1 file changed
+185
-125
@@ -8,7 +8,6 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
#include <math.h>
|
#include <math.h>
|
||||||
#include <pthread.h>
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -24,110 +23,174 @@
|
|||||||
#define ALIGN_PADDING 0x100000
|
#define ALIGN_PADDING 0x100000
|
||||||
#define CACHE_LINE_SIZE 128
|
#define CACHE_LINE_SIZE 128
|
||||||
|
|
||||||
struct f_data {
|
#define BW_MAX_THREADS 4
|
||||||
void (*func)(int64_t *, int64_t *, int);
|
|
||||||
int64_t *arg1;
|
|
||||||
int64_t *arg2;
|
|
||||||
int arg3;
|
|
||||||
};
|
|
||||||
|
|
||||||
static pthread_cond_t p_ready, p_start;
|
typedef void (*bench_kernel_fn)(int64_t *dst, int64_t *src, long size);
|
||||||
static pthread_mutex_t p_lock;
|
|
||||||
static pthread_t *p_worker = NULL;
|
|
||||||
static struct f_data *worker_data = NULL;
|
|
||||||
static int p_worker_not_ready, p_workers_ready;
|
|
||||||
|
|
||||||
static void *thread_func(void *data) {
|
struct bw_pool;
|
||||||
struct f_data *d = data;
|
|
||||||
pthread_mutex_lock(&p_lock);
|
|
||||||
p_worker_not_ready--;
|
|
||||||
if (!p_worker_not_ready)
|
|
||||||
pthread_cond_signal(&p_ready);
|
|
||||||
while (p_workers_ready != 1)
|
|
||||||
pthread_cond_wait(&p_start, &p_lock);
|
|
||||||
pthread_mutex_unlock(&p_lock);
|
|
||||||
(d->func)(d->arg1, d->arg2, d->arg3);
|
|
||||||
pthread_exit(NULL);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void parallel_run(void) {
|
typedef struct {
|
||||||
pthread_mutex_lock(&p_lock);
|
Thread thread;
|
||||||
p_workers_ready = 1;
|
Semaphore start;
|
||||||
pthread_mutex_unlock(&p_lock);
|
struct bw_pool *pool;
|
||||||
pthread_cond_broadcast(&p_start);
|
bench_kernel_fn func;
|
||||||
}
|
int64_t *dst;
|
||||||
|
int64_t *src;
|
||||||
|
long size;
|
||||||
|
int core;
|
||||||
|
bool created;
|
||||||
|
} bw_worker_t;
|
||||||
|
|
||||||
static void parallel_init(int threads) {
|
typedef struct bw_pool {
|
||||||
pthread_attr_t attr;
|
bw_worker_t workers[BW_MAX_THREADS];
|
||||||
pthread_cond_init(&p_ready, NULL);
|
Semaphore done;
|
||||||
pthread_cond_init(&p_start, NULL);
|
int threads;
|
||||||
pthread_mutex_init(&p_lock, NULL);
|
bool ok;
|
||||||
p_worker_not_ready = threads;
|
} bw_pool_t;
|
||||||
p_workers_ready = 0;
|
|
||||||
pthread_attr_init(&attr);
|
static void bw_worker_entry(void *arg) {
|
||||||
if (!p_worker || !worker_data) {
|
bw_worker_t *w = (bw_worker_t *)arg;
|
||||||
p_worker = malloc(threads * sizeof(pthread_t));
|
bw_pool_t *p = w->pool;
|
||||||
worker_data = malloc(threads * sizeof(struct f_data));
|
/* Best-effort pin to own core. */
|
||||||
|
svcSetThreadCoreMask(CUR_THREAD_HANDLE, w->core, (u64)1 << w->core);
|
||||||
|
for (;;) {
|
||||||
|
semaphoreWait(&w->start);
|
||||||
|
bench_kernel_fn f = w->func;
|
||||||
|
if (!f)
|
||||||
|
break;
|
||||||
|
f(w->dst, w->src, w->size);
|
||||||
|
semaphoreSignal(&p->done);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool bw_pool_init(bw_pool_t *p, int threads) {
|
||||||
|
memset(p, 0, sizeof(*p));
|
||||||
|
if (threads > BW_MAX_THREADS)
|
||||||
|
threads = BW_MAX_THREADS;
|
||||||
|
p->threads = threads;
|
||||||
|
semaphoreInit(&p->done, 0);
|
||||||
|
s32 prio = 0x2C;
|
||||||
|
svcGetThreadPriority(&prio, CUR_THREAD_HANDLE);
|
||||||
|
for (int i = 0; i < threads; i++) {
|
||||||
|
bw_worker_t *w = &p->workers[i];
|
||||||
|
w->core = i;
|
||||||
|
w->func = NULL;
|
||||||
|
w->pool = p;
|
||||||
|
semaphoreInit(&w->start, 0);
|
||||||
|
if (R_FAILED(threadCreate(&w->thread, bw_worker_entry, w, NULL, 0x8000, prio, i)))
|
||||||
|
return false;
|
||||||
|
w->created = true;
|
||||||
|
if (R_FAILED(threadStart(&w->thread)))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
p->ok = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void bw_pool_free(bw_pool_t *p) {
|
||||||
|
for (int i = 0; i < p->threads; i++) {
|
||||||
|
bw_worker_t *w = &p->workers[i];
|
||||||
|
if (w->created) {
|
||||||
|
w->func = NULL; /* shutdown */
|
||||||
|
semaphoreSignal(&w->start);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (int i = 0; i < p->threads; i++) {
|
||||||
|
bw_worker_t *w = &p->workers[i];
|
||||||
|
if (w->created) {
|
||||||
|
threadWaitForExit(&w->thread);
|
||||||
|
threadClose(&w->thread);
|
||||||
|
w->created = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p->ok = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static double bw_run_once(bw_pool_t *p, bench_kernel_fn f, int64_t *dstbuf, int64_t *srcbuf, long size) {
|
||||||
|
int threads = p->threads;
|
||||||
|
for (int i = 0; i < threads; i++) {
|
||||||
|
bw_worker_t *w = &p->workers[i];
|
||||||
|
w->func = f;
|
||||||
|
w->dst = dstbuf + (size * i) / (long)sizeof(int64_t);
|
||||||
|
w->src = srcbuf + (size * i) / (long)sizeof(int64_t);
|
||||||
|
w->size = size;
|
||||||
|
}
|
||||||
|
uint64_t t0 = armGetSystemTick();
|
||||||
for (int i = 0; i < threads; i++)
|
for (int i = 0; i < threads; i++)
|
||||||
pthread_create(p_worker + i, &attr, thread_func, worker_data + i);
|
semaphoreSignal(&p->workers[i].start);
|
||||||
pthread_mutex_lock(&p_lock);
|
for (int i = 0; i < threads; i++)
|
||||||
while (p_worker_not_ready != 0)
|
semaphoreWait(&p->done);
|
||||||
pthread_cond_wait(&p_ready, &p_lock);
|
uint64_t t1 = armGetSystemTick();
|
||||||
pthread_mutex_unlock(&p_lock);
|
return (double)armTicksToNs(t1 - t0) / 1000000000.0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void aligned_block_copy(int64_t *__restrict dst_, int64_t *__restrict src, int size) {
|
#define BW_GPR_CLOBBERS \
|
||||||
volatile int64_t *dst = dst_;
|
"x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18"
|
||||||
int64_t t1, t2, t3, t4;
|
|
||||||
while ((size -= 64) >= 0) {
|
|
||||||
t1 = *src++;
|
|
||||||
t2 = *src++;
|
|
||||||
t3 = *src++;
|
|
||||||
t4 = *src++;
|
|
||||||
*dst++ = t1;
|
|
||||||
*dst++ = t2;
|
|
||||||
*dst++ = t3;
|
|
||||||
*dst++ = t4;
|
|
||||||
t1 = *src++;
|
|
||||||
t2 = *src++;
|
|
||||||
t3 = *src++;
|
|
||||||
t4 = *src++;
|
|
||||||
*dst++ = t1;
|
|
||||||
*dst++ = t2;
|
|
||||||
*dst++ = t3;
|
|
||||||
*dst++ = t4;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void aligned_block_fetch(int64_t *__restrict dst, int64_t *__restrict src_, int size) {
|
static void aligned_block_copy(int64_t *dst, int64_t *src, long size) {
|
||||||
volatile int64_t *src = src_;
|
__asm__ __volatile__(
|
||||||
(void)dst;
|
"1:\n"
|
||||||
while ((size -= 64) >= 0) {
|
"ldp x3, x4, [%[s], #0x00]\n"
|
||||||
*src++;
|
"ldp x5, x6, [%[s], #0x10]\n"
|
||||||
*src++;
|
"ldp x7, x8, [%[s], #0x20]\n"
|
||||||
*src++;
|
"ldp x9, x10, [%[s], #0x30]\n"
|
||||||
*src++;
|
"ldp x11, x12, [%[s], #0x40]\n"
|
||||||
*src++;
|
"ldp x13, x14, [%[s], #0x50]\n"
|
||||||
*src++;
|
"ldp x15, x16, [%[s], #0x60]\n"
|
||||||
*src++;
|
"ldp x17, x18, [%[s], #0x70]\n"
|
||||||
*src++;
|
"add %[s], %[s], #0x80\n"
|
||||||
}
|
"stp x3, x4, [%[d], #0x00]\n"
|
||||||
|
"stp x5, x6, [%[d], #0x10]\n"
|
||||||
|
"stp x7, x8, [%[d], #0x20]\n"
|
||||||
|
"stp x9, x10, [%[d], #0x30]\n"
|
||||||
|
"stp x11, x12, [%[d], #0x40]\n"
|
||||||
|
"stp x13, x14, [%[d], #0x50]\n"
|
||||||
|
"stp x15, x16, [%[d], #0x60]\n"
|
||||||
|
"stp x17, x18, [%[d], #0x70]\n"
|
||||||
|
"add %[d], %[d], #0x80\n"
|
||||||
|
"subs %[n], %[n], #0x80\n"
|
||||||
|
"b.gt 1b\n"
|
||||||
|
: [s] "+r"(src), [d] "+r"(dst), [n] "+r"(size)
|
||||||
|
:
|
||||||
|
: BW_GPR_CLOBBERS, "cc", "memory");
|
||||||
}
|
}
|
||||||
|
static void aligned_block_fetch(int64_t *dst, int64_t *src, long size) {
|
||||||
static void aligned_block_fill(int64_t *__restrict dst_, int64_t *__restrict src, int size) {
|
(void)src;
|
||||||
volatile int64_t *dst = dst_;
|
__asm__ __volatile__(
|
||||||
int64_t data = *src;
|
"1:\n"
|
||||||
while ((size -= 64) >= 0) {
|
"ldp x3, x4, [%[p], #0x00]\n"
|
||||||
*dst++ = data;
|
"ldp x5, x6, [%[p], #0x10]\n"
|
||||||
*dst++ = data;
|
"ldp x7, x8, [%[p], #0x20]\n"
|
||||||
*dst++ = data;
|
"ldp x9, x10, [%[p], #0x30]\n"
|
||||||
*dst++ = data;
|
"ldp x11, x12, [%[p], #0x40]\n"
|
||||||
*dst++ = data;
|
"ldp x13, x14, [%[p], #0x50]\n"
|
||||||
*dst++ = data;
|
"ldp x15, x16, [%[p], #0x60]\n"
|
||||||
*dst++ = data;
|
"ldp x17, x18, [%[p], #0x70]\n"
|
||||||
*dst++ = data;
|
"add %[p], %[p], #0x80\n"
|
||||||
}
|
"subs %[n], %[n], #0x80\n"
|
||||||
|
"b.gt 1b\n"
|
||||||
|
: [p] "+r"(dst), [n] "+r"(size)
|
||||||
|
:
|
||||||
|
: BW_GPR_CLOBBERS, "cc", "memory");
|
||||||
|
}
|
||||||
|
static void aligned_block_fill(int64_t *dst, int64_t *src, long size) {
|
||||||
|
(void)src;
|
||||||
|
__asm__ __volatile__(
|
||||||
|
"1:\n"
|
||||||
|
"stp x3, x4, [%[d], #0x00]\n"
|
||||||
|
"stp x5, x6, [%[d], #0x10]\n"
|
||||||
|
"stp x7, x8, [%[d], #0x20]\n"
|
||||||
|
"stp x9, x10, [%[d], #0x30]\n"
|
||||||
|
"stp x11, x12, [%[d], #0x40]\n"
|
||||||
|
"stp x13, x14, [%[d], #0x50]\n"
|
||||||
|
"stp x15, x16, [%[d], #0x60]\n"
|
||||||
|
"stp x17, x18, [%[d], #0x70]\n"
|
||||||
|
"add %[d], %[d], #0x80\n"
|
||||||
|
"subs %[n], %[n], #0x80\n"
|
||||||
|
"b.gt 1b\n"
|
||||||
|
: [d] "+r"(dst), [n] "+r"(size)
|
||||||
|
:
|
||||||
|
: BW_GPR_CLOBBERS, "cc", "memory");
|
||||||
}
|
}
|
||||||
|
|
||||||
static double gettime(void) {
|
static double gettime(void) {
|
||||||
@@ -136,9 +199,12 @@ static double gettime(void) {
|
|||||||
return (double)((int64_t)tv.tv_sec * 1000000 + tv.tv_usec) / 1000000.;
|
return (double)((int64_t)tv.tv_sec * 1000000 + tv.tv_usec) / 1000000.;
|
||||||
}
|
}
|
||||||
|
|
||||||
static double bandwidth_bench_helper(int threads, int64_t *dstbuf, int64_t *srcbuf, int size, void (*f)(int64_t *, int64_t *, int)) {
|
static double bandwidth_bench_helper(bw_pool_t *pool, int threads, int64_t *dstbuf, int64_t *srcbuf, long size, bench_kernel_fn f) {
|
||||||
int i, loopcount, innerloopcount, n;
|
int i, loopcount, innerloopcount, n;
|
||||||
double t, t1, t2, speed, maxspeed, s, s0, s1, s2;
|
double t, speed, maxspeed, s, s0, s1, s2;
|
||||||
|
|
||||||
|
if (!pool || !pool->ok)
|
||||||
|
return 0.;
|
||||||
|
|
||||||
s = s0 = s1 = s2 = 0.;
|
s = s0 = s1 = s2 = 0.;
|
||||||
maxspeed = 0.;
|
maxspeed = 0.;
|
||||||
@@ -148,21 +214,8 @@ static double bandwidth_bench_helper(int threads, int64_t *dstbuf, int64_t *srcb
|
|||||||
t = 0.;
|
t = 0.;
|
||||||
do {
|
do {
|
||||||
loopcount += innerloopcount;
|
loopcount += innerloopcount;
|
||||||
for (i = 0; i < innerloopcount; i++) {
|
for (i = 0; i < innerloopcount; i++)
|
||||||
parallel_init(threads);
|
t += bw_run_once(pool, f, dstbuf, srcbuf, size);
|
||||||
for (int pt = 0; pt < threads; pt++) {
|
|
||||||
(worker_data + pt)->func = f;
|
|
||||||
(worker_data + pt)->arg1 = dstbuf + size * pt / sizeof(int64_t);
|
|
||||||
(worker_data + pt)->arg2 = srcbuf + size * pt / sizeof(int64_t);
|
|
||||||
(worker_data + pt)->arg3 = size;
|
|
||||||
}
|
|
||||||
t1 = gettime();
|
|
||||||
parallel_run();
|
|
||||||
for (int pt = 0; pt < threads; pt++)
|
|
||||||
pthread_join(p_worker[pt], NULL);
|
|
||||||
t2 = gettime();
|
|
||||||
t += t2 - t1;
|
|
||||||
}
|
|
||||||
innerloopcount *= 2;
|
innerloopcount *= 2;
|
||||||
} while (t < 0.5);
|
} while (t < 0.5);
|
||||||
|
|
||||||
@@ -306,13 +359,16 @@ void bench_run_full(bench_results_t *out, bench_progress_fn progress, void *user
|
|||||||
|
|
||||||
int64_t *srcbuf, *dstbuf;
|
int64_t *srcbuf, *dstbuf;
|
||||||
void *poolbuf = alloc_nonaliased_buffers((void **)&srcbuf, size * threads, (void **)&dstbuf, size * threads, NULL, 0);
|
void *poolbuf = alloc_nonaliased_buffers((void **)&srcbuf, size * threads, (void **)&dstbuf, size * threads, NULL, 0);
|
||||||
|
bw_pool_t bw;
|
||||||
|
bw_pool_init(&bw, threads);
|
||||||
|
|
||||||
STEP("CPU copy", 0.40f);
|
STEP("CPU copy", 0.40f);
|
||||||
out->cpu_copy = bandwidth_bench_helper(threads, dstbuf, srcbuf, size, aligned_block_copy);
|
out->cpu_copy = bandwidth_bench_helper(&bw, threads, dstbuf, srcbuf, size, aligned_block_copy);
|
||||||
STEP("CPU read", 0.55f);
|
STEP("CPU read", 0.55f);
|
||||||
out->cpu_read = bandwidth_bench_helper(threads, dstbuf, srcbuf, size, aligned_block_fetch);
|
out->cpu_read = bandwidth_bench_helper(&bw, threads, dstbuf, srcbuf, size, aligned_block_fetch);
|
||||||
STEP("CPU write", 0.70f);
|
STEP("CPU write", 0.70f);
|
||||||
out->cpu_write = bandwidth_bench_helper(threads, dstbuf, srcbuf, size, aligned_block_fill);
|
out->cpu_write = bandwidth_bench_helper(&bw, threads, dstbuf, srcbuf, size, aligned_block_fill);
|
||||||
|
bw_pool_free(&bw);
|
||||||
free(poolbuf);
|
free(poolbuf);
|
||||||
|
|
||||||
STEP("Latency", 0.85f);
|
STEP("Latency", 0.85f);
|
||||||
@@ -325,9 +381,10 @@ void bench_run_full(bench_results_t *out, bench_progress_fn progress, void *user
|
|||||||
struct bench_ctx {
|
struct bench_ctx {
|
||||||
int phase;
|
int phase;
|
||||||
bool is_4gb;
|
bool is_4gb;
|
||||||
void *pool;
|
void *buf;
|
||||||
int64_t *src;
|
int64_t *src;
|
||||||
int64_t *dst;
|
int64_t *dst;
|
||||||
|
bw_pool_t bw;
|
||||||
};
|
};
|
||||||
|
|
||||||
bench_ctx *bench_begin(void) {
|
bench_ctx *bench_begin(void) {
|
||||||
@@ -335,7 +392,8 @@ bench_ctx *bench_begin(void) {
|
|||||||
if (!c)
|
if (!c)
|
||||||
return NULL;
|
return NULL;
|
||||||
c->is_4gb = (appletGetAppletType() == AppletType_Application);
|
c->is_4gb = (appletGetAppletType() == AppletType_Application);
|
||||||
c->pool = alloc_nonaliased_buffers((void **)&c->src, SIZE * 3, (void **)&c->dst, SIZE * 3, NULL, 0);
|
c->buf = alloc_nonaliased_buffers((void **)&c->src, SIZE * 3, (void **)&c->dst, SIZE * 3, NULL, 0);
|
||||||
|
bw_pool_init(&c->bw, 3);
|
||||||
return c;
|
return c;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -349,24 +407,25 @@ bool bench_step(bench_ctx *c, bench_results_t *out, const char **label, float *f
|
|||||||
*frac = 0.25f;
|
*frac = 0.25f;
|
||||||
break;
|
break;
|
||||||
case 1:
|
case 1:
|
||||||
out->cpu_copy = bandwidth_bench_helper(threads, c->dst, c->src, size, aligned_block_copy);
|
out->cpu_copy = bandwidth_bench_helper(&c->bw, threads, c->dst, c->src, size, aligned_block_copy);
|
||||||
*label = "CPU copy";
|
*label = "CPU copy";
|
||||||
*frac = 0.45f;
|
*frac = 0.45f;
|
||||||
break;
|
break;
|
||||||
case 2:
|
case 2:
|
||||||
out->cpu_read = bandwidth_bench_helper(threads, c->dst, c->src, size, aligned_block_fetch);
|
out->cpu_read = bandwidth_bench_helper(&c->bw, threads, c->dst, c->src, size, aligned_block_fetch);
|
||||||
*label = "CPU read";
|
*label = "CPU read";
|
||||||
*frac = 0.60f;
|
*frac = 0.60f;
|
||||||
break;
|
break;
|
||||||
case 3:
|
case 3:
|
||||||
out->cpu_write = bandwidth_bench_helper(threads, c->dst, c->src, size, aligned_block_fill);
|
out->cpu_write = bandwidth_bench_helper(&c->bw, threads, c->dst, c->src, size, aligned_block_fill);
|
||||||
*label = "CPU write";
|
*label = "CPU write";
|
||||||
*frac = 0.75f;
|
*frac = 0.75f;
|
||||||
break;
|
break;
|
||||||
case 4:
|
case 4:
|
||||||
if (c->pool) {
|
bw_pool_free(&c->bw);
|
||||||
free(c->pool);
|
if (c->buf) {
|
||||||
c->pool = NULL;
|
free(c->buf);
|
||||||
|
c->buf = NULL;
|
||||||
}
|
}
|
||||||
latency_bench(&out->l2_ns, &out->ram_ns);
|
latency_bench(&out->l2_ns, &out->ram_ns);
|
||||||
*label = "Latency";
|
*label = "Latency";
|
||||||
@@ -384,7 +443,8 @@ bool bench_step(bench_ctx *c, bench_results_t *out, const char **label, float *f
|
|||||||
void bench_end(bench_ctx *c) {
|
void bench_end(bench_ctx *c) {
|
||||||
if (!c)
|
if (!c)
|
||||||
return;
|
return;
|
||||||
if (c->pool)
|
bw_pool_free(&c->bw);
|
||||||
free(c->pool);
|
if (c->buf)
|
||||||
|
free(c->buf);
|
||||||
free(c);
|
free(c);
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user