GPU computing with CUDA

Lecture 3: CUDA Memories

 Hardware Implementation of CUDA
Memories
Grid

! Each thread can:

Block (0, 0) Block (1, 0)
! Read/write per-thread
registers Shared Memory Shared Memory

! Read/write per-thread
local memory Registers Registers Registers Registers

! Read/write per-block
shared memory Thread (0, 0) Thread (1, 0) Thread (0, 0) Thread (1, 0)

! Read/write per-grid
global memory Host Global Memory
! Read/only per-grid
constant memory Constant Memory
CUDA Variable Type Qualifiers
Variable declaration Memory Scope Lifetime
int var; register thread thread
int array_var[10]; local thread thread
__shared__ int shared_var; shared block block
__device__ int global_var; global grid application
__constant__ int constant_var; constant grid application

! automatic scalar variables without qualifier reside

in a register
! compiler will spill to thread local memory
! automatic array variables without qualifier reside
in thread-local memory
CUDA Variable Type Performance
Variable declaration Memory Penalty
int var; register 1x
int array_var[10]; local 100x
__shared__ int shared_var; shared 1x
__device__ int global_var; global 100x
__constant__ int constant_var; constant 1x

! scalar variables reside in fast, on-chip registers

! shared variables reside in fast, on-chip memories
! thread-local arrays & global variables reside in
uncached off-chip memory
! constant variables reside in cached off-chip memory
CUDA Variable Type Scale
Variable declaration Instances Visibility
int var; 100,000s 1
int array_var[10]; 100,000s 1
__shared__ int shared_var; 100s 100s
__device__ int global_var; 1 100,000s
__constant__ int constant_var; 1 100,000s

! 100Ks per-thread variables, R/W by 1 thread

! 100s shared variables, each R/W by 100s of threads
! 1 global variable is R/W by 100Ks threads
! 1 constant variable is readable by 100Ks threads
Where to declare variables?

Can host access it?

Yes No

Outside of any In the kernel

function

__constant__ int constant_var; int var;

__device__ int global_var; int array_var[10];
__shared__ int shared_var;
 Example – thread-local variables
// motivate per-thread variables with
// Ten Nearest Neighbors application
__global__ void ten_nn(float2 *result, float2 *ps, float2 *qs,
size_t num_qs)
{
// p goes in a register
float2 p = ps[threadIdx.x];

// per-thread heap goes in off-chip memory

float2 heap[10];

// read through num_qs points, maintaining

// the nearest 10 qs to p in the heap
...
// write out the contents of heap to result
...
}
 Example – shared variables
// motivate shared variables with
// Adjacent Difference application:
// compute result[i] = input[i] – input[i-1]
__global__ void adj_diff_naive(int *result, int *input)
{
// compute this thread s global index
unsigned int i = blockDim.x * blockIdx.x + threadIdx.x;

if(i > 0)
{
// each thread loads two elements from global memory
int x_i = input[i];
int x_i_minus_one = input[i-1];

result[i] = x_i – x_i_minus_one;

}
}
 Example – shared variables
// motivate shared variables with
// Adjacent Difference application:
// compute result[i] = input[i] – input[i-1]
__global__ void adj_diff_naive(int *result, int *input)
{
// compute this thread s global index
unsigned int i = blockDim.x * blockIdx.x + threadIdx.x;

if(i > 0)
{
// what are the bandwidth requirements of this kernel?
int x_i = input[i];
int x_i_minus_one = input[i-1];
Two loads

result[i] = x_i – x_i_minus_one;

}
}
 Example – shared variables
// motivate shared variables with
// Adjacent Difference application:
// compute result[i] = input[i] – input[i-1]
__global__ void adj_diff_naive(int *result, int *input)
{
// compute this thread s global index
unsigned int i = blockDim.x * blockIdx.x + threadIdx.x;

if(i > 0)
{
// How many times does this kernel load input[i]?
int x_i = input[i]; // once by thread i
int x_i_minus_one = input[i-1]; // again by thread i+1

result[i] = x_i – x_i_minus_one;

}
}
 Example – shared variables
// motivate shared variables with
// Adjacent Difference application:
// compute result[i] = input[i] – input[i-1]
__global__ void adj_diff_naive(int *result, int *input)
{
// compute this thread s global index
unsigned int i = blockDim.x * blockIdx.x + threadIdx.x;

if(i > 0)
{
// Idea: eliminate redundancy by sharing data
int x_i = input[i];
int x_i_minus_one = input[i-1];

result[i] = x_i – x_i_minus_one;

}
}
 Example – shared variables
// optimized version of adjacent difference
__global__ void adj_diff(int *result, int *input)
{
// shorthand for threadIdx.x
int tx = threadIdx.x;
// allocate a __shared__ array, one element per thread
__shared__ int s_data[BLOCK_SIZE];
// each thread reads one element to s_data
unsigned int i = blockDim.x * blockIdx.x + tx;
s_data[tx] = input[i];

// avoid race condition: ensure all loads

// complete before continuing
__syncthreads();
...
}
 Example – shared variables
// optimized version of adjacent difference
__global__ void adj_diff(int *result, int *input)
{
...
if(tx > 0)
result[i] = s_data[tx] – s_data[tx–1];
else if(i > 0)
{
// handle thread block boundary
result[i] = s_data[tx] – input[i-1];
}
}
 Example – shared variables
// when the size of the array isn t known at compile time...
__global__ void adj_diff(int *result, int *input)
{
// use extern to indicate a __shared__ array will be
// allocated dynamically at kernel launch time
extern __shared__ int s_data[];
...
}

// pass the size of the per-block array, in bytes, as the third

// argument to the triple chevrons
adj_diff<<<num_blocks, block_size, block_size * sizeof(int)>>>(r,i);
About Pointers

! Yes, you can use them!

! You can point at any memory space per se:
__device__ int my_global_variable;
__constant__ int my_constant_variable = 13;

__global__ void foo(void)

{
__shared__ int my_shared_variable;

int *ptr_to_global = &my_global_variable;

const int *ptr_to_constant = &my_constant_variable;
int *ptr_to_shared = &my_shared_variable;
...
*ptr_to_global = *ptr_to_shared;
}
About Pointers
! The address obtained by taking the address of a
__device__, __shared__ or __constant__ variable
can only be used in device code.

! The address of a __device__ or __constant__

variable obtained through cudaGetSymbolAddress()
can only be used in host code.
Don t confuse the compiler!
__device__ int my_global_variable;
__global__ void foo(int *input)
{
__shared__ int my_shared_variable;

int *ptr = 0;
if(input[threadIdx.x] % 2)
ptr = &my_global_variable;
else
ptr = &my_shared_variable;
// where does ptr point?
}

Warning: Cannot tell what pointer points to, assuming global

memory space
Advice

! Prefer dereferencing pointers in simple, regular

access patterns
! Avoid propagating pointers
! Avoid pointers to pointers
! The GPU would rather not pointer chase
! Linked lists will not perform well
! Pay attention to compiler warning messages
! Warning: Cannot tell what pointer points to,
assuming global memory space
! Crash waiting to happen
A Common Programming Strategy

! Global memory resides in device memory (DRAM)

! Much slower access than shared memory
! Tile data to take advantage of fast shared memory:
! Generalize from adjacent_difference
example
! Divide and conquer
 A Common Programming Strategy

! Partition data into subsets that fit into shared memory

 A Common Programming Strategy

! Handle each data subset with one thread block

 A Common Programming Strategy

! Load the subset from global memory to shared

memory, using multiple threads to exploit memory-
level parallelism
A Common Programming Strategy

! Perform the computation on the subset from shared

memory
A Common Programming Strategy

! Copy the result from shared memory back to global

memory
A Common Programming Strategy

! Carefully partition data according to access patterns

! Read-only è __constant__ memory (fast)
! R/W & shared within block è __shared__ memory
(fast)
! R/W within each thread è registers (fast)
! Indexed R/W within each thread è local memory
(slow)
! R/W inputs/results è cudaMalloc ed global
memory (slow)
Communication Through Memory

! Question:

__global__ void race(void)

{
__shared__ int my_shared_variable;
my_shared_variable = threadIdx.x;

// what is the value of

// my_shared_variable?
}
Communication Through Memory

! This is a race condition

! The result is undefined
! The order in which threads access the variable is
undefined without explicit coordination
! Use barriers (e.g., __syncthreads) or atomic
operations (e.g., atomicAdd) to enforce well-defined
semantics
Communication Through Memory
! Use __syncthreads to ensure data is ready for
access

__global__ void share_data(int *input)

{
__shared__ int data[BLOCK_SIZE];
data[threadIdx.x] = input[threadIdx.x];
__syncthreads();
// the state of the entire data array
// is now well-defined for all threads
// in this block
}
Communication Through Memory
! Use atomic operations to ensure exclusive access to
a variable

// assume *result is initialized to 0

__global__ void sum(int *input, int *result)
{
atomicAdd(result, input[threadIdx.x]);

// after this kernel exits, the value of

// *result will be the sum of the input
}
Resource Contention

! Atomic operations aren t cheap!

! They imply serialized access to a variable

__global__ void sum(int *input, int *result)

{
atomicAdd(result, input[threadIdx.x]);
}
...
// how many threads will contend
// for exclusive access to result?
sum<<<B,N/B>>>(input,result);
Hierarchical Atomics
Σ

Σ0
Σ1
Σι

! Divide & Conquer

! Per-thread atomicAdd to a __shared__ partial sum
! Per-block atomicAdd to the total sum
Hierarchical Atomics
__global__ void sum(int *input, int *result)
{
__shared__ int partial_sum;

// thread 0 is responsible for

// initializing partial_sum
if(threadIdx.x == 0)
partial_sum = 0;
__syncthreads();

...
}
Hierarchical Atomics
__global__ void sum(int *input, int *result)
{
...
// each thread updates the partial sum
atomicAdd(&partial_sum,
input[threadIdx.x]);
__syncthreads();

// thread 0 updates the total sum

if(threadIdx.x == 0)
atomicAdd(result, partial_sum);
}
Advice

! Use barriers such as __syncthreads to wait until

__shared__ data is ready
! Prefer barriers to atomics when data access patterns
are regular or predictable
! Prefer atomics to barriers when data access patterns
are sparse or unpredictable
! Atomics to __shared__ variables are much faster
than atomics to global variables
! Don t synchronize or serialize unnecessarily
Matrix Multiplication Example

! Generalize adjacent_difference
example
! AB = A * B
! Each element ABij
! = dot(row(A,i),col(B,j))
! Parallelization strategy
! Thread à ABij
! 2D kernel
First Implementation
__global__ void mat_mul(float *a, float *b,
float *ab, int width)
{
// calculate the row & col index of the element
int row = blockIdx.y*blockDim.y + threadIdx.y;
int col = blockIdx.x*blockDim.x + threadIdx.x;

float result = 0;

// do dot product between row of a and col of b

for(int k = 0; k < width; ++k)
result += a[row*width+k] * b[k*width+col];

ab[row*width+col] = result;
}
How will this perform?

How many loads per term of dot 2 floats (a & b) =

product? 8 Bytes
How many floating point operations? 2 (multiply & addition)
Global memory access to flop ratio 8 Bytes / 2 ops =
(GMAC) 4 B/op
What is the peak fp performance of 1.35 TFLOPS
GeForce GTX 480?
Lower bound on bandwidth required to GMAC * Peak FLOPS = 4 * 1.350 =
reach peak fp performance 5.4 TB/s
What is the actual memory bandwidth 177 GB/s
of GeForce GTX 480?
Then what is an upper bound on Actual BW / GMAC = 177 / 4 =
performance of our implementation? 44 GFLOPS
Idea: Use __shared__ memory to
reuse global data

! Each input element is

read by width threads
! Load each element into
__shared__ memory
and have several threads
use the local version to
reduce the memory
bandwidth

width
Tiled Multiply
TILE_WIDTH

! Partition kernel loop

into phases
! Load a tile of both
matrices into
__shared__ each
phase
! Each phase, each
thread computes a
partial result
Better Implementation
__global__ void mat_mul(float *a, float *b,
float *ab, int width)
{
// shorthand
int tx = threadIdx.x, ty = threadIdx.y;
int bx = blockIdx.x, by = blockIdx.y;
// allocate tiles in __shared__ memory
__shared__ float s_a[TILE_WIDTH][TILE_WIDTH];
__shared__ float s_b[TILE_WIDTH][TILE_WIDTH];
// calculate the row & col index
int row = by*blockDim.y + ty;
int col = bx*blockDim.x + tx;

float result = 0;
 Better Implementation
// loop over the tiles of the input in phases
for(int p = 0; p < width/TILE_WIDTH; ++p)
{
// collaboratively load tiles into __shared__
s_a[ty][tx] = a[row*width + (p*TILE_WIDTH + tx)];
s_b[ty][tx] = b[(m*TILE_WIDTH + ty)*width + col];
__syncthreads();

// dot product between row of s_a and col of s_b

for(int k = 0; k < TILE_WIDTH; ++k)
result += s_a[ty][k] * s_b[k][tx];
__syncthreads();
}

ab[row*width+col] = result;
}
Use of Barriers in mat_mul

! Two barriers per phase:

! __syncthreads after all data is loaded into __shared__
memory
! __syncthreads after all data is read from __shared__
memory
! Note that second __syncthreads in phase p guards the
load in phase p+1

! Use barriers to guard data

! Guard against using uninitialized data
! Guard against bashing live data
First Order Size Considerations

! Each thread block should have many threads

! TILE_WIDTH = 16 à 16*16 = 256 threads

! There should be many thread blocks

! 1024*1024 matrices à 64*64 = 4096 thread blocks
! TILE_WIDTH = 16 à gives each SM 4 blocks, 1024 threads
! Full occupancy

! Each thread block performs 2 * 256 = 512 x 4B loads

for 256 * (2 * 16) = 8,192 fp ops (0.25 B/op)
! Compare to 4B/op
TILE_SIZE Effects
Memory Resources as Limit to
Parallelism
Resource Per GTX480 SM Full Occupancy on
GTX480
Registers 32768 <= 32768 / 1024 threads
= 32 per thread
__shared__ Memory 48KB <= 48KB / 8 blocks
= 6KB per block

! Effective use of different memory resources reduces

the number of accesses to global memory
! These resources are finite!
! The more memory locations each thread requires à
the fewer threads an SM can accommodate
Final Thoughts

! Effective use of CUDA memory hierarchy decreases

bandwidth consumption to increase throughput
! Use __shared__ memory to eliminate redundant
loads from global memory
! Use __syncthreads barriers to protect __shared__ data
! Use atomics if access patterns are sparse or unpredictable
! Optimization comes with a development cost
! Memory resources ultimately limit parallelism