audio: infer stereo RAW AAC layout from channel-pair packets

Co-authored-by: BlackCAT <52720489+psnwd@users.noreply.github.com>

src/libs/ajm/aac_decoder.h

@@ -38,9 +38,9 @@ static constexpr uint32_t AJM_DEC_M4AAC_MAX_SAMPLING_FREQ_INDEX = 11;
 
 class AjmAacDecoder final: public AjmDecoder {
 public:
-	AjmAacDecoder(uint32_t channels, uint32_t sample_rate, AjmSampleEncoding encoding,
+	AjmAacDecoder(uint32_t max_channels, uint32_t sample_rate, AjmSampleEncoding encoding,
 	              uint64_t flags)
-	    : AjmDecoder(channels, sample_rate, encoding), m_flags(flags) {
+	    : AjmDecoder(0, sample_rate, encoding), m_max_channels(max_channels), m_flags(flags) {
 		m_codec = avcodec_find_decoder(AV_CODEC_ID_AAC);
 		if (m_codec == nullptr) {
 			LOGF("AJM AAC: avcodec_find_decoder failed\n");
@@ -94,6 +94,11 @@ public:
 		    ((m_flags & AJM_INSTANCE_FLAG_DEC_M4AAC_ENABLE_NONDELAY_OUTPUT) != 0 ? 0u : 2u);
 		m_total_decoded_samples = 0;
 		m_bitrate               = 0;
+		m_channels              = 0;
+		m_raw_configured        = false;
+		if (m_config_number == AJM_DEC_M4AAC_CONFIG_NUMBER_RAW) {
+			m_sample_rate = SamplingRateFromIndex(m_sampling_freq_index);
+		}
 		if (m_is_initialized) {
 			OpenDecoder();
 		}
@@ -189,26 +194,6 @@ private:
 			return;
 		}
 
-		if (m_config_number == AJM_DEC_M4AAC_CONFIG_NUMBER_RAW) {
-			const auto sample_index =
-			    std::min(m_sampling_freq_index, AJM_DEC_M4AAC_MAX_SAMPLING_FREQ_INDEX);
-			const auto channels = std::clamp<uint32_t>(m_channels, 1u, AJM_DEC_M4AAC_MAX_CHANNELS);
-			constexpr uint32_t audio_object_type = 2;
-
-			auto* extradata = static_cast<uint8_t*>(av_mallocz(2 + AV_INPUT_BUFFER_PADDING_SIZE));
-			if (extradata == nullptr) {
-				Release();
-				return;
-			}
-
-			extradata[0] = static_cast<uint8_t>((audio_object_type << 3u) | (sample_index >> 1u));
-			extradata[1] = static_cast<uint8_t>(((sample_index & 0x1u) << 7u) |
-			                                    (std::min(channels, 7u) << 3u));
-			m_codec_context->extradata      = extradata;
-			m_codec_context->extradata_size = 2;
-			SetFormat(channels, SamplingRateFromIndex(sample_index), m_sample_encoding);
-		}
-
 		if (avcodec_open2(m_codec_context, m_codec, nullptr) < 0) {
 			LOGF("AJM AAC: decoder initialization failed\n");
 			Release();
@@ -222,6 +207,10 @@ private:
 		}
 
 		const auto channels = static_cast<uint32_t>(std::max(frame->ch_layout.nb_channels, 1));
+		if (channels > m_max_channels) {
+			result->result = AJM_RESULT_TOO_MANY_CHANNELS;
+			return false;
+		}
 		const auto bpf      = channels * AjmBytesPerSample(m_sample_encoding);
 		if (bpf == 0) {
 			result->result = AJM_RESULT_INVALID_PARAMETER;
@@ -306,6 +295,25 @@ private:
 			return false;
 		}
 		std::memcpy(packet->data, packet_data, static_cast<size_t>(packet_size));
+		if (m_config_number == AJM_DEC_M4AAC_CONFIG_NUMBER_RAW && !m_raw_configured) {
+			// Only indexed stereo starts with a channel pair; the instance limit is not
+			// the stream's channel count. A leading SCE remains ambiguous.
+			const auto channels = (packet_data[0] >> 5u) == 1u
+			                          ? 2u
+			                          : std::clamp(m_max_channels, 1u, AJM_DEC_M4AAC_MAX_CHANNELS);
+			auto* extradata = av_packet_new_side_data(packet, AV_PKT_DATA_NEW_EXTRADATA, 2);
+			if (extradata == nullptr) {
+				av_packet_free(&packet);
+				result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL;
+				return false;
+			}
+			constexpr uint32_t audio_object_type = 2;
+			extradata[0] = static_cast<uint8_t>((audio_object_type << 3u) |
+			                                    (m_sampling_freq_index >> 1u));
+			extradata[1] = static_cast<uint8_t>(((m_sampling_freq_index & 1u) << 7u) |
+			                                    (std::min(channels, 7u) << 3u));
+			m_raw_configured = true;
+		}
 
 		int rc = avcodec_send_packet(m_codec_context, packet);
 		av_packet_free(&packet);
@@ -353,12 +361,14 @@ private:
 
 	const AVCodec*  m_codec               = nullptr;
 	AVCodecContext* m_codec_context       = nullptr;
+	const uint32_t  m_max_channels;
 	uint64_t        m_flags               = 0;
 	uint32_t        m_config_number       = AJM_DEC_M4AAC_CONFIG_NUMBER_ADTS;
 	uint32_t        m_sampling_freq_index = 3;
 	uint32_t        m_skip_frames         = 2;
 	uint32_t        m_bitrate             = 0;
 	bool            m_is_initialized      = false;
+	bool            m_raw_configured      = false;
 };
 
 } // namespace Libs::Audio::Ajm

src/libs/ajm/decoder.h

@@ -72,6 +72,7 @@ constexpr int32_t AJM_RESULT_INVALID_DATA       = 0x00000002;
 constexpr int32_t AJM_RESULT_INVALID_PARAMETER  = 0x00000004;
 constexpr int32_t AJM_RESULT_PARTIAL_INPUT      = 0x00000008;
 constexpr int32_t AJM_RESULT_NOT_ENOUGH_ROOM    = 0x00000010;
+constexpr int32_t AJM_RESULT_TOO_MANY_CHANNELS  = 0x00000040;
 constexpr int32_t AJM_RESULT_CODEC_ERROR        = 0x40000000;
 constexpr int32_t AJM_RESULT_FATAL              = static_cast<int32_t>(0x80000000u);
 

src/libs/libAudio.cpp

@@ -1052,7 +1052,7 @@ static int32_t KYTY_SYSV_ABI AudiodecDecode(int32_t handle, AudiodecCtrl* ctrl)
 		ctrl->pPcmItem->uiPcmSize = static_cast<uint32_t>(result.output_written);
 		audiodec_fill_info(ctrl, decoder);
 		auto* info = static_cast<AudiodecM4aacInfo*>(ctrl->pBsiInfo);
-		info->iResult = (result.result & AJM_RESULT_CODEC_ERROR) != 0
+		info->iResult = (result.result & (AJM_RESULT_CODEC_ERROR | AJM_RESULT_TOO_MANY_CHANNELS)) != 0
 		                    ? AUDIODEC_M4AAC_RESULT_DECODE_ERROR
 		                    : (result.result & AJM_RESULT_PARTIAL_INPUT) != 0
 		                          ? AUDIODEC_M4AAC_RESULT_INSUFFICIENT_DATA : 0;