219 lines
8.6 KiB
Python
219 lines
8.6 KiB
Python
import torch
|
|
import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
|
|
|
|
# pylint: disable=protected-access, missing-function-docstring, line-too-long
|
|
|
|
original_torch_bmm = torch.bmm
|
|
|
|
|
|
def torch_bmm(input, mat2, *, out=None):
|
|
if input.dtype != mat2.dtype:
|
|
mat2 = mat2.to(input.dtype)
|
|
|
|
# ARC GPUs can't allocate more than 4GB to a single block, Slice it:
|
|
batch_size_attention, input_tokens, mat2_shape = (
|
|
input.shape[0],
|
|
input.shape[1],
|
|
mat2.shape[2],
|
|
)
|
|
block_multiply = input.element_size()
|
|
slice_block_size = input_tokens * mat2_shape / 1024 / 1024 * block_multiply
|
|
block_size = batch_size_attention * slice_block_size
|
|
|
|
split_slice_size = batch_size_attention
|
|
if block_size > 4:
|
|
do_split = True
|
|
# Find something divisible with the input_tokens
|
|
while (split_slice_size * slice_block_size) > 4:
|
|
split_slice_size = split_slice_size // 2
|
|
if split_slice_size <= 1:
|
|
split_slice_size = 1
|
|
break
|
|
else:
|
|
do_split = False
|
|
|
|
split_2_slice_size = input_tokens
|
|
if split_slice_size * slice_block_size > 4:
|
|
slice_block_size2 = split_slice_size * mat2_shape / 1024 / 1024 * block_multiply
|
|
do_split_2 = True
|
|
# Find something divisible with the input_tokens
|
|
while (split_2_slice_size * slice_block_size2) > 4:
|
|
split_2_slice_size = split_2_slice_size // 2
|
|
if split_2_slice_size <= 1:
|
|
split_2_slice_size = 1
|
|
break
|
|
else:
|
|
do_split_2 = False
|
|
|
|
if do_split:
|
|
hidden_states = torch.zeros(
|
|
input.shape[0],
|
|
input.shape[1],
|
|
mat2.shape[2],
|
|
device=input.device,
|
|
dtype=input.dtype,
|
|
)
|
|
for i in range(batch_size_attention // split_slice_size):
|
|
start_idx = i * split_slice_size
|
|
end_idx = (i + 1) * split_slice_size
|
|
if do_split_2:
|
|
for i2 in range(
|
|
input_tokens // split_2_slice_size
|
|
): # pylint: disable=invalid-name
|
|
start_idx_2 = i2 * split_2_slice_size
|
|
end_idx_2 = (i2 + 1) * split_2_slice_size
|
|
hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = (
|
|
original_torch_bmm(
|
|
input[start_idx:end_idx, start_idx_2:end_idx_2],
|
|
mat2[start_idx:end_idx, start_idx_2:end_idx_2],
|
|
out=out,
|
|
)
|
|
)
|
|
else:
|
|
hidden_states[start_idx:end_idx] = original_torch_bmm(
|
|
input[start_idx:end_idx], mat2[start_idx:end_idx], out=out
|
|
)
|
|
else:
|
|
return original_torch_bmm(input, mat2, out=out)
|
|
return hidden_states
|
|
|
|
|
|
original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention
|
|
|
|
|
|
def scaled_dot_product_attention(
|
|
query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False
|
|
):
|
|
# ARC GPUs can't allocate more than 4GB to a single block, Slice it:
|
|
if len(query.shape) == 3:
|
|
batch_size_attention, query_tokens, shape_four = query.shape
|
|
shape_one = 1
|
|
no_shape_one = True
|
|
else:
|
|
shape_one, batch_size_attention, query_tokens, shape_four = query.shape
|
|
no_shape_one = False
|
|
|
|
block_multiply = query.element_size()
|
|
slice_block_size = (
|
|
shape_one * query_tokens * shape_four / 1024 / 1024 * block_multiply
|
|
)
|
|
block_size = batch_size_attention * slice_block_size
|
|
|
|
split_slice_size = batch_size_attention
|
|
if block_size > 4:
|
|
do_split = True
|
|
# Find something divisible with the shape_one
|
|
while (split_slice_size * slice_block_size) > 4:
|
|
split_slice_size = split_slice_size // 2
|
|
if split_slice_size <= 1:
|
|
split_slice_size = 1
|
|
break
|
|
else:
|
|
do_split = False
|
|
|
|
split_2_slice_size = query_tokens
|
|
if split_slice_size * slice_block_size > 4:
|
|
slice_block_size2 = (
|
|
shape_one * split_slice_size * shape_four / 1024 / 1024 * block_multiply
|
|
)
|
|
do_split_2 = True
|
|
# Find something divisible with the batch_size_attention
|
|
while (split_2_slice_size * slice_block_size2) > 4:
|
|
split_2_slice_size = split_2_slice_size // 2
|
|
if split_2_slice_size <= 1:
|
|
split_2_slice_size = 1
|
|
break
|
|
else:
|
|
do_split_2 = False
|
|
|
|
if do_split:
|
|
hidden_states = torch.zeros(query.shape, device=query.device, dtype=query.dtype)
|
|
for i in range(batch_size_attention // split_slice_size):
|
|
start_idx = i * split_slice_size
|
|
end_idx = (i + 1) * split_slice_size
|
|
if do_split_2:
|
|
for i2 in range(
|
|
query_tokens // split_2_slice_size
|
|
): # pylint: disable=invalid-name
|
|
start_idx_2 = i2 * split_2_slice_size
|
|
end_idx_2 = (i2 + 1) * split_2_slice_size
|
|
if no_shape_one:
|
|
hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = (
|
|
original_scaled_dot_product_attention(
|
|
query[start_idx:end_idx, start_idx_2:end_idx_2],
|
|
key[start_idx:end_idx, start_idx_2:end_idx_2],
|
|
value[start_idx:end_idx, start_idx_2:end_idx_2],
|
|
attn_mask=(
|
|
attn_mask[start_idx:end_idx, start_idx_2:end_idx_2]
|
|
if attn_mask is not None
|
|
else attn_mask
|
|
),
|
|
dropout_p=dropout_p,
|
|
is_causal=is_causal,
|
|
)
|
|
)
|
|
else:
|
|
hidden_states[:, start_idx:end_idx, start_idx_2:end_idx_2] = (
|
|
original_scaled_dot_product_attention(
|
|
query[:, start_idx:end_idx, start_idx_2:end_idx_2],
|
|
key[:, start_idx:end_idx, start_idx_2:end_idx_2],
|
|
value[:, start_idx:end_idx, start_idx_2:end_idx_2],
|
|
attn_mask=(
|
|
attn_mask[
|
|
:, start_idx:end_idx, start_idx_2:end_idx_2
|
|
]
|
|
if attn_mask is not None
|
|
else attn_mask
|
|
),
|
|
dropout_p=dropout_p,
|
|
is_causal=is_causal,
|
|
)
|
|
)
|
|
else:
|
|
if no_shape_one:
|
|
hidden_states[start_idx:end_idx] = (
|
|
original_scaled_dot_product_attention(
|
|
query[start_idx:end_idx],
|
|
key[start_idx:end_idx],
|
|
value[start_idx:end_idx],
|
|
attn_mask=(
|
|
attn_mask[start_idx:end_idx]
|
|
if attn_mask is not None
|
|
else attn_mask
|
|
),
|
|
dropout_p=dropout_p,
|
|
is_causal=is_causal,
|
|
)
|
|
)
|
|
else:
|
|
hidden_states[:, start_idx:end_idx] = (
|
|
original_scaled_dot_product_attention(
|
|
query[:, start_idx:end_idx],
|
|
key[:, start_idx:end_idx],
|
|
value[:, start_idx:end_idx],
|
|
attn_mask=(
|
|
attn_mask[:, start_idx:end_idx]
|
|
if attn_mask is not None
|
|
else attn_mask
|
|
),
|
|
dropout_p=dropout_p,
|
|
is_causal=is_causal,
|
|
)
|
|
)
|
|
else:
|
|
return original_scaled_dot_product_attention(
|
|
query,
|
|
key,
|
|
value,
|
|
attn_mask=attn_mask,
|
|
dropout_p=dropout_p,
|
|
is_causal=is_causal,
|
|
)
|
|
return hidden_states
|
|
|
|
|
|
def attention_init():
|
|
# ARC GPUs can't allocate more than 4GB to a single block:
|
|
torch.bmm = torch_bmm
|
|
torch.nn.functional.scaled_dot_product_attention = scaled_dot_product_attention
|