Skip to content

encode: codec profile and level fixes #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dabrain34 merged 7 commits into from
Aug 31, 2026
Merged

encode: codec profile and level fixes #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a1c8ba0
VkEncoderConfigH264: improve level limits table and related code
srinathkr-nv Nov 26, 2025
8f269be
encode: separate level/tier determination into a distinct initializat…
srinathkr-nv Jul 19, 2026
e5ce080
VkEncoderConfig: rename InitVideoProfile to MakeVideoProfile
srinathkr-nv Jul 19, 2026
92f15d8
encode: add/move codec profile auto-selection into InitVideoProfileCa…
srinathkr-nv Jul 19, 2026
a27639c
EncoderConfigH264: add a profile-probing loop in InitVideoProfileCapa…
srinathkr-nv Jul 7, 2026
07c7452
EncoderConfigH264: simplify transform_8x8_mode_flag handling
srinathkr-nv Jul 8, 2026
20ee9f7
EncoderConfigH265: add MaxLumaSr check to IsSuitableLevel
srinathkr-nv Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 7 additions & 17 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -822,24 +822,14 @@ VkResult EncoderConfig::CreateCodecConfig(int argc, const char *argv[],
return VK_ERROR_INITIALIZATION_FAILED;
}

void EncoderConfig::InitVideoProfile()
VkVideoCoreProfile EncoderConfig::MakeVideoProfile(uint32_t codecProfile)
{
if (encodeBitDepthLuma == 0) {
encodeBitDepthLuma = input.bpp;
}

if (encodeBitDepthChroma == 0) {
encodeBitDepthChroma = encodeBitDepthLuma;
}

// update the video profile
videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile(
codec, encodeChromaSubsampling,
GetComponentBitDepthFlagBits(encodeBitDepthLuma),
GetComponentBitDepthFlagBits(encodeBitDepthChroma),
(videoProfileIdc != (uint32_t)-1) ? videoProfileIdc :
GetDefaultVideoProfileIdc(),
tuningMode);
return VkVideoCoreProfile::CreateEncodeProfile(
codec, encodeChromaSubsampling,
GetComponentBitDepthFlagBits(encodeBitDepthLuma),
GetComponentBitDepthFlagBits(encodeBitDepthChroma),
codecProfile,
tuningMode);
}

bool EncoderConfig::InitRateControl()
Expand Down
16 changes: 11 additions & 5 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,7 +695,6 @@ struct EncoderConfig : public VkVideoRefCountBase {
bool noDeviceFallback;
VkVideoCodecOperationFlagBitsKHR codec;
bool useDpbArray;
uint32_t videoProfileIdc;
uint32_t numInputImages;
EncoderInputImageParameters input;
uint8_t encodeBitDepthLuma;
Expand Down Expand Up @@ -803,7 +802,6 @@ struct EncoderConfig : public VkVideoRefCountBase {
, noDeviceFallback(false)
, codec(VK_VIDEO_CODEC_OPERATION_NONE_KHR)
, useDpbArray(false)
, videoProfileIdc((uint32_t)-1)
, numInputImages(DEFAULT_NUM_INPUT_IMAGES)
, input()
, encodeBitDepthLuma(0)
Expand Down Expand Up @@ -925,7 +923,7 @@ struct EncoderConfig : public VkVideoRefCountBase {
// Factory Function
static VkResult CreateCodecConfig(int argc, const char *argv[], VkSharedBaseObj<EncoderConfig>& encoderConfig);

void InitVideoProfile();
VkVideoCoreProfile MakeVideoProfile(uint32_t codecProfile);

int ParseArguments(int argc, const char *argv[]);

Expand Down Expand Up @@ -965,13 +963,21 @@ struct EncoderConfig : public VkVideoRefCountBase {
return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR;
}

if (encodeBitDepthLuma == 0) {
encodeBitDepthLuma = input.bpp;
}

if (encodeBitDepthChroma == 0) {
encodeBitDepthChroma = encodeBitDepthLuma;
}

return VK_SUCCESS;
}

// These functions should be overwritten from the codec-specific classes
virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) { return VK_ERROR_INITIALIZATION_FAILED; };
virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) { return VK_ERROR_INITIALIZATION_FAILED; };

virtual uint32_t GetDefaultVideoProfileIdc() { return 0; };
virtual bool DetermineLevelTier() = 0;

virtual int8_t InitDpbCount() { return 16; };

Expand Down
29 changes: 23 additions & 6 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,28 @@ bool EncoderConfigAV1::InitSequenceHeader(StdVideoAV1SequenceHeader *seqHdr,
return true;
}

VkResult EncoderConfigAV1::InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx)
VkResult EncoderConfigAV1::InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx)
{
// If profile hasn't been specified, determine it based on bit depth and chroma subsampling
if (profile == STD_VIDEO_AV1_PROFILE_INVALID) {
// PROFESSIONAL is required for 12-bit or 422
if ((encodeBitDepthLuma > 10) ||
(encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_422_BIT_KHR)) {
profile = STD_VIDEO_AV1_PROFILE_PROFESSIONAL;
}
// HIGH is required for 444 chroma
else if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) {
profile = STD_VIDEO_AV1_PROFILE_HIGH;
}
// MAIN supports 8-bit and 10-bit with 420
else {
profile = STD_VIDEO_AV1_PROFILE_MAIN;
}
}

assert(profile != STD_VIDEO_AV1_PROFILE_INVALID);
videoCoreProfile = MakeVideoProfile(static_cast<uint32_t>(profile));

const bool feedback2Requested = enablePictureFeedback || enablePixelCountFeedback ||
enableSkippedPixelCountFeedback || enablePerPartitionFeedback;
videoEncodeFeedback2Capabilities = VkVideoEncodeFeedback2CapabilitiesKHR
Expand Down Expand Up @@ -376,17 +396,14 @@ bool EncoderConfigAV1::DetermineLevelTier()
}
}
if (lvl > STD_VIDEO_AV1_LEVEL_7_3) {
level = STD_VIDEO_AV1_LEVEL_7_3;
tier = 0;
return false;
}

return true;
}

bool EncoderConfigAV1::InitRateControl()
{
DetermineLevelTier();

// Level and tier are already initialized by DetermineLevelTier()
// use level max values for now. Limit it to 120Mbits/sec
uint32_t levelBitrate = std::min(GetLevelBitrate(level, tier), 120000000u);

Expand Down
10 changes: 4 additions & 6 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,7 @@ struct EncoderConfigAV1 : public EncoderConfig {
return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR;
}

virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override;

virtual uint32_t GetDefaultVideoProfileIdc() override { return STD_VIDEO_AV1_PROFILE_MAIN; }
virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override;

virtual int8_t InitDpbCount() override;

Expand All @@ -138,7 +136,7 @@ struct EncoderConfigAV1 : public EncoderConfig {

bool ValidateLevel(uint32_t lLevel, uint32_t lTier);

bool DetermineLevelTier();
virtual bool DetermineLevelTier() override;

uint32_t GetLevelMaxBitrate(uint32_t lLevel, uint32_t lTier) {
if (lLevel < STD_VIDEO_AV1_LEVEL_4_0) {
Expand Down Expand Up @@ -185,8 +183,8 @@ struct EncoderConfigAV1 : public EncoderConfig {
return ((encodeWidth * encodeHeight * picSizeProfileFactor) >> 3);
}

StdVideoAV1Profile profile{ STD_VIDEO_AV1_PROFILE_MAIN };
StdVideoAV1Level level{ STD_VIDEO_AV1_LEVEL_5_0 };
StdVideoAV1Profile profile{ STD_VIDEO_AV1_PROFILE_INVALID };
StdVideoAV1Level level{ STD_VIDEO_AV1_LEVEL_INVALID };
uint8_t tier{};
VkVideoEncodeAV1CapabilitiesKHR av1EncodeCapabilities{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_CAPABILITIES_KHR };
VkVideoEncodeAV1QualityLevelPropertiesKHR av1QualityLevelProperties{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_QUALITY_LEVEL_PROPERTIES_KHR };
Expand Down
Loading
Loading