Skip to content

use size_t for dds and ktx surface size to avoid overflow #626

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
solidpixel merged 8 commits into from
Jun 2, 2026
Merged

use size_t for dds and ktx surface size to avoid overflow #626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5a70942
use size_t for dds and ktx surface size to avoid overflow
sahvx655-wq Jun 2, 2026
6b52841
Merge remote-tracking branch 'origin/main' into pr/sahvx655-wq/626
solidpixel Jun 2, 2026
a9a3c62
Refactor to use common library code
solidpixel Jun 2, 2026
7622815
Make components unsigned
solidpixel Jun 2, 2026
3b7ce46
Revert error message fix - will do in separate PR
solidpixel Jun 2, 2026
9f79021
Handle output plane allocation overflow too
solidpixel Jun 2, 2026
7ae82f8
Minor comment fix
solidpixel Jun 2, 2026
3708105
Minor cleanup
solidpixel Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Source/astcenccli_image.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,6 @@ astcenc_image* astc_img_from_unorm8x4_array(
return img;
}

// initialize a flattened array of float values from an ASTC codec image
// The returned array is allocated with new[] and must be deleted with delete[].
/* See header for documentation. */
float* floatx4_array_from_astc_img(
const astcenc_image* img,
Expand Down
141 changes: 90 additions & 51 deletions Source/astcenccli_image_load_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -979,7 +979,7 @@ static astcenc_image* load_ktx_uncompressed_image(

// Cartesian product of gl_type=(UNSIGNED_BYTE, UNSIGNED_SHORT, HALF_FLOAT, FLOAT) x gl_format=(RED, RG, RGB, RGBA, BGR, BGRA)

int components;
unsigned int components;
switch (hdr.gl_format)
{
case GL_RED:
Expand Down Expand Up @@ -1013,8 +1013,8 @@ static astcenc_image* load_ktx_uncompressed_image(
}

// Although these are set up later, use default initializer to remove warnings
int bitness = 8; // Internal precision after conversion
int bytes_per_component = 1; // Bytes per component in the KTX file
unsigned int bitness = 8; // Internal precision after conversion
unsigned int bytes_per_component = 1; // Bytes per component in the KTX file
scanline_transfer copy_method = R8_TO_RGBA8;

switch (hdr.gl_type)
Expand Down Expand Up @@ -1194,14 +1194,25 @@ static astcenc_image* load_ktx_uncompressed_image(
specified_bytes_of_surface = reverse_bytes_u32(specified_bytes_of_surface);
}

// read the surface
uint32_t xstride = bytes_per_component * components * dim_x;
uint32_t ystride = xstride * dim_y;
uint32_t computed_bytes_of_surface = dim_z * ystride;
if (computed_bytes_of_surface != specified_bytes_of_surface)
// Compute surface size, checking for overflow caused by bad user-defined sizes
// These values are trusted and cannot overflow
size_t bytes_per_pixel = bytes_per_component * components;

bool overflow { false };

size_t bytes_per_row = astc::mul_safe(bytes_per_pixel, dim_x, overflow);
size_t bytes_per_plane = astc::mul_safe(bytes_per_row, dim_y, overflow);
size_t bytes_per_image = astc::mul_safe(bytes_per_plane, dim_z, overflow);

// Also verify that our output plane allocation does not overflow because
// this always uses 4 components which can be more than the input file used
size_t plane_texels = astc::mul_safe(dim_x, dim_y, overflow);
astc::mul_safe(plane_texels, 4, overflow);

if (overflow || bytes_per_image != specified_bytes_of_surface)
{
fclose(f);
printf("%s: KTX file inconsistency: computed surface size is %u bytes, but specified size is %u bytes\n", filename, computed_bytes_of_surface, specified_bytes_of_surface);
printf("%s: KTX file inconsistency: computed surface size is %zu bytes, but specified size is %u bytes\n", filename, bytes_per_image, specified_bytes_of_surface);
return nullptr;
Comment thread
solidpixel marked this conversation as resolved.
}

Expand Down Expand Up @@ -1252,7 +1263,7 @@ static astcenc_image* load_ktx_uncompressed_image(
dst = static_cast<void*>(&data16[4 * dim_x * ydst]);
}

uint8_t *src = buf + (z * ystride) + (y * xstride);
uint8_t *src = buf + (z * bytes_per_plane) + (y * bytes_per_row);
copy_scanline(dst, src, dim_x, copy_method);
}
}
Expand Down Expand Up @@ -1860,11 +1871,11 @@ static astcenc_image* load_dds_uncompressed_image(
unsigned int dim_z = (hdr.flags & 0x800000) ? hdr.depth : 1;

// The bitcount that we will use internally in the codec
int bitness = 0;
unsigned int bitness = 0;

// The bytes per component in the DDS file itself
int bytes_per_component = 0;
int components = 0;
unsigned int bytes_per_component = 0;
unsigned int components = 0;
scanline_transfer copy_method = R8_TO_RGBA8;

// figure out the format actually used in the DDS file.
Expand All @@ -1877,41 +1888,41 @@ static astcenc_image* load_dds_uncompressed_image(
#define DXGI_FORMAT_R16G16B16A16_UNORM 11
#define DXGI_FORMAT_R32G32_FLOAT 16
#define DXGI_FORMAT_R8G8B8A8_UNORM 28
#define DXGI_FORMAT_R16G16_FLOAT 34
#define DXGI_FORMAT_R16G16_UNORM 35
#define DXGI_FORMAT_R32_FLOAT 41
#define DXGI_FORMAT_R8G8_UNORM 49
#define DXGI_FORMAT_R16_FLOAT 54
#define DXGI_FORMAT_R16_UNORM 56
#define DXGI_FORMAT_R8_UNORM 61
#define DXGI_FORMAT_B8G8R8A8_UNORM 86
#define DXGI_FORMAT_B8G8R8X8_UNORM 87
#define DXGI_FORMAT_R16G16_FLOAT 34
#define DXGI_FORMAT_R16G16_UNORM 35
#define DXGI_FORMAT_R32_FLOAT 41
#define DXGI_FORMAT_R8G8_UNORM 49
#define DXGI_FORMAT_R16_FLOAT 54
#define DXGI_FORMAT_R16_UNORM 56
#define DXGI_FORMAT_R8_UNORM 61
#define DXGI_FORMAT_B8G8R8A8_UNORM 86
#define DXGI_FORMAT_B8G8R8X8_UNORM 87

struct dxgi_params
{
int bitness;
int bytes_per_component;
int components;
unsigned int bitness;
unsigned int bytes_per_component;
unsigned int components;
scanline_transfer copy_method;
uint32_t dxgi_format_number;
};

static const dxgi_params format_params[] {
{16, 4, 4, RGBA32F_TO_RGBA16F, DXGI_FORMAT_R32G32B32A32_FLOAT},
{16, 4, 3, RGB32F_TO_RGBA16F, DXGI_FORMAT_R32G32B32_FLOAT},
{16, 2, 4, RGBA16F_TO_RGBA16F, DXGI_FORMAT_R16G16B16A16_FLOAT},
{16, 2, 4, RGBA16_TO_RGBA16F, DXGI_FORMAT_R16G16B16A16_UNORM},
{16, 4, 2, RG32F_TO_RGBA16F, DXGI_FORMAT_R32G32_FLOAT},
{8, 1, 4, RGBA8_TO_RGBA8, DXGI_FORMAT_R8G8B8A8_UNORM},
{16, 2, 2, RG16F_TO_RGBA16F, DXGI_FORMAT_R16G16_FLOAT},
{16, 2, 2, RG16_TO_RGBA16F, DXGI_FORMAT_R16G16_UNORM},
{16, 4, 1, R32F_TO_RGBA16F, DXGI_FORMAT_R32_FLOAT},
{8, 1, 2, RG8_TO_RGBA8, DXGI_FORMAT_R8G8_UNORM},
{16, 2, 1, R16F_TO_RGBA16F, DXGI_FORMAT_R16_FLOAT},
{16, 2, 1, R16_TO_RGBA16F, DXGI_FORMAT_R16_UNORM},
{8, 1, 1, R8_TO_RGBA8, DXGI_FORMAT_R8_UNORM},
{8, 1, 4, BGRA8_TO_RGBA8, DXGI_FORMAT_B8G8R8A8_UNORM},
{8, 1, 4, BGRX8_TO_RGBA8, DXGI_FORMAT_B8G8R8X8_UNORM},
{ 16, 4, 4, RGBA32F_TO_RGBA16F, DXGI_FORMAT_R32G32B32A32_FLOAT },
{ 16, 4, 3, RGB32F_TO_RGBA16F, DXGI_FORMAT_R32G32B32_FLOAT },
{ 16, 2, 4, RGBA16F_TO_RGBA16F, DXGI_FORMAT_R16G16B16A16_FLOAT },
{ 16, 2, 4, RGBA16_TO_RGBA16F, DXGI_FORMAT_R16G16B16A16_UNORM },
{ 16, 4, 2, RG32F_TO_RGBA16F, DXGI_FORMAT_R32G32_FLOAT },
{ 8, 1, 4, RGBA8_TO_RGBA8, DXGI_FORMAT_R8G8B8A8_UNORM },
{ 16, 2, 2, RG16F_TO_RGBA16F, DXGI_FORMAT_R16G16_FLOAT },
{ 16, 2, 2, RG16_TO_RGBA16F, DXGI_FORMAT_R16G16_UNORM },
{ 16, 4, 1, R32F_TO_RGBA16F, DXGI_FORMAT_R32_FLOAT },
{ 8, 1, 2, RG8_TO_RGBA8, DXGI_FORMAT_R8G8_UNORM },
{ 16, 2, 1, R16F_TO_RGBA16F, DXGI_FORMAT_R16_FLOAT },
{ 16, 2, 1, R16_TO_RGBA16F, DXGI_FORMAT_R16_UNORM },
{ 8, 1, 1, R8_TO_RGBA8, DXGI_FORMAT_R8_UNORM },
{ 8, 1, 4, BGRA8_TO_RGBA8, DXGI_FORMAT_B8G8R8A8_UNORM },
{ 8, 1, 4, BGRX8_TO_RGBA8, DXGI_FORMAT_B8G8R8X8_UNORM },
};

int dxgi_modes_supported = sizeof(format_params) / sizeof(format_params[0]);
Expand Down Expand Up @@ -2027,22 +2038,50 @@ static astcenc_image* load_dds_uncompressed_image(
bitness = bytes_per_component * 8;
}

// then, load the actual file.
uint32_t xstride = bytes_per_component * components * dim_x;
uint32_t ystride = xstride * dim_y;
uint32_t bytes_of_surface = ystride * dim_z;
// Compute surface size, checking for overflow caused by bad user-defined sizes
// These values are trusted and cannot overflow
size_t bytes_per_pixel = bytes_per_component * components;

// These values are not and can overflow
bool overflow { false };
size_t bytes_per_row = astc::mul_safe(bytes_per_pixel, dim_x, overflow);
size_t bytes_per_plane = astc::mul_safe(bytes_per_row, dim_y, overflow);
size_t bytes_per_image = astc::mul_safe(bytes_per_plane, dim_z, overflow);

uint8_t *buf = new uint8_t[bytes_of_surface];
size_t bytes_read = fread(buf, 1, bytes_of_surface, f);
// Also verify that our output plane allocation does not overflow because
// this always uses 4 components which can be more than the input file used
size_t plane_texels = astc::mul_safe(dim_x, dim_y, overflow);
astc::mul_safe(plane_texels, 4, overflow);

if (overflow)
{
printf("DDS file %s has invalid dimensions\n", filename);
fclose(f);
return nullptr;
}

uint8_t* buf { nullptr };
try
{
buf = new uint8_t[bytes_per_image];
}
catch (...)
{
printf("Failed to allocate memory for file %s\n", filename);
fclose(f);
return nullptr;
}

size_t bytes_read = fread(buf, 1, bytes_per_image, f);
fclose(f);
if (bytes_read != bytes_of_surface)
if (bytes_read != bytes_per_image)
{
delete[] buf;
printf("Failed to read file %s\n", filename);
delete[] buf;
return nullptr;
}

// then transfer data from the surface to our own image-data-structure.
// Transfer data from the surface to our own image data structure
astcenc_image *astc_img = alloc_image(bitness, dim_x, dim_y, dim_z);

for (unsigned int z = 0; z < dim_z; z++)
Expand All @@ -2065,7 +2104,7 @@ static astcenc_image* load_dds_uncompressed_image(
dst = static_cast<void*>(&data16[4 * dim_x * ydst]);
}

uint8_t *src = buf + (z * ystride) + (y * xstride);
uint8_t *src = buf + (z * bytes_per_plane) + (y * bytes_per_row);
copy_scanline(dst, src, dim_x, copy_method);
}
}
Expand Down Expand Up @@ -2559,7 +2598,7 @@ int load_cimage(
{
buffer = new uint8_t[data_size];
}
catch(...)
catch (...)
{
print_error("ERROR: Image memory allocation failed '%s'\n", filename);
return 1;
Expand Down
Loading