#include <AudioInputProcessor.h>

Inheritance diagram for alexaClientSDK::capabilityAgents::aip::AudioInputProcessor:

[legend]

Collaboration diagram for alexaClientSDK::capabilityAgents::aip::AudioInputProcessor:

[legend]

Public Types
using	ObserverInterface = avsCommon::sdkInterfaces::AudioInputProcessorObserverInterface
	Alias to the `AudioInputProcessorObserverInterface` for brevity. More...

using	EncodingFormatRequest = std::map< std::string, std::pair< avsCommon::utils::AudioFormat::Encoding, avsCommon::utils::AudioFormat::Encoding > >

using	EncodingFormatResponse = std::map< std::string, avsCommon::utils::AudioFormat::Encoding >

using	ExpectSpeechTimeoutHandler = avsCommon::sdkInterfaces::ExpectSpeechTimeoutHandlerInterface

Public Types inherited from alexaClientSDK::avsCommon::sdkInterfaces::DialogUXStateObserverInterface
enum	DialogUXState { DialogUXState::IDLE, DialogUXState::LISTENING, DialogUXState::EXPECTING, DialogUXState::THINKING, DialogUXState::SPEAKING, DialogUXState::FINISHED }
	The different dialog specific AVS UX states. More...

Public Types inherited from alexaClientSDK::avsCommon::sdkInterfaces::MessageRequestObserverInterface
enum	Status { Status::PENDING, Status::SUCCESS, Status::SUCCESS_ACCEPTED, Status::SUCCESS_NO_CONTENT, Status::NOT_CONNECTED, Status::NOT_SYNCHRONIZED, Status::TIMEDOUT, Status::PROTOCOL_ERROR, Status::INTERNAL_ERROR, Status::SERVER_INTERNAL_ERROR_V2, Status::REFUSED, Status::CANCELED, Status::THROTTLED, Status::INVALID_AUTH, Status::BAD_REQUEST, Status::SERVER_OTHER_ERROR }

Public Member Functions
void	addObserver (std::shared_ptr< ObserverInterface > observer)

void	removeObserver (std::shared_ptr< ObserverInterface > observer)

std::future< bool >	recognize (AudioProvider audioProvider, Initiator initiator, std::chrono::steady_clock::time_point startOfSpeechTimestamp=std::chrono::steady_clock::now(), avsCommon::avs::AudioInputStream::Index begin=INVALID_INDEX, avsCommon::avs::AudioInputStream::Index keywordEnd=INVALID_INDEX, std::string keyword="", std::shared_ptr< const std::vector< char >> KWDMetadata=nullptr, const std::string &initiatorToken="")

std::future< bool >	stopCapture ()

std::future< void >	resetState ()

bool	setEncodingAudioFormat (avsCommon::utils::AudioFormat::Encoding encoding)

EncodingFormatResponse	requestEncodingAudioFormats (const EncodingFormatRequest &encodings)

EncodingFormatResponse	getEncodingAudioFormats () const

ContextRequesterInterface Functions
void	onContextAvailable (const std::string &jsonContext) override

void	onContextFailure (const avsCommon::sdkInterfaces::ContextRequestError error) override

MessageRequestObserverInterface Functions
void	onResponseStatusReceived (avsCommon::sdkInterfaces::MessageRequestObserverInterface::Status status) override

void	onSendCompleted (avsCommon::sdkInterfaces::MessageRequestObserverInterface::Status status) override

void	onExceptionReceived (const std::string &exceptionMessage) override

CapabilityAgent/DirectiveHandlerInterface Functions
void	handleDirectiveImmediately (std::shared_ptr< avsCommon::avs::AVSDirective > directive) override

void	preHandleDirective (std::shared_ptr< DirectiveInfo > info) override

void	handleDirective (std::shared_ptr< DirectiveInfo > info) override

void	cancelDirective (std::shared_ptr< DirectiveInfo > info) override

void	onDeregistered () override

avsCommon::avs::DirectiveHandlerConfiguration	getConfiguration () const override

ChannelObserverInterface Functions
void	onFocusChanged (avsCommon::avs::FocusState newFocus, avsCommon::avs::MixingBehavior behavior) override

DialogUXStateObserverInterface Functions
void	onDialogUXStateChanged (avsCommon::sdkInterfaces::DialogUXStateObserverInterface::DialogUXState newState) override

CapabilityConfigurationInterface Functions
std::unordered_set< std::shared_ptr< avsCommon::avs::CapabilityConfiguration > >	getCapabilityConfigurations () override

LocaleAssetsObserverInterface Functions
void	onLocaleAssetsChanged () override

InternetConnectionObserverInterface Functions
void	onConnectionStatusChanged (bool connected) override

Public Member Functions inherited from alexaClientSDK::avsCommon::avs::CapabilityAgent
virtual	~CapabilityAgent ()=default

void	preHandleDirective (std::shared_ptr< AVSDirective > directive, std::unique_ptr< sdkInterfaces::DirectiveHandlerResultInterface > result) override final

bool	handleDirective (const std::string &messageId) override final

void	cancelDirective (const std::string &messageId) override final

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::DirectiveHandlerInterface
virtual	~DirectiveHandlerInterface ()=default

virtual void	preHandleDirective (std::shared_ptr< avsCommon::avs::AVSDirective > directive, std::unique_ptr< DirectiveHandlerResultInterface > result)=0

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::ChannelObserverInterface
virtual	~ChannelObserverInterface ()=default

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::StateProviderInterface
virtual	~StateProviderInterface ()=default

virtual void	provideState (const avs::NamespaceAndName &stateProviderName, const ContextRequestToken stateRequestToken)

virtual void	provideState (const avs::CapabilityTag &stateProviderName, const ContextRequestToken stateRequestToken)

virtual bool	canStateBeRetrieved ()

virtual bool	hasReportableStateProperties ()

virtual bool	shouldQueryState ()

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::ContextRequesterInterface
virtual	~ContextRequesterInterface ()=default

virtual void	onContextAvailable (const endpoints::EndpointIdentifier &endpointId, const avs::AVSContext &endpointContext, ContextRequestToken requestToken)

virtual void	onContextFailure (const ContextRequestError error, ContextRequestToken token)

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::CapabilityConfigurationInterface
virtual	~CapabilityConfigurationInterface ()=default

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::LocaleAssetsObserverInterface
virtual	~LocaleAssetsObserverInterface ()=default

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::DialogUXStateObserverInterface
virtual	~DialogUXStateObserverInterface ()=default

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::MessageRequestObserverInterface
virtual	~MessageRequestObserverInterface ()=default

Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::InternetConnectionObserverInterface
virtual	~InternetConnectionObserverInterface ()=default

Public Member Functions inherited from alexaClientSDK::avsCommon::utils::RequiresShutdown
	RequiresShutdown (const std::string &name)

virtual	~RequiresShutdown ()
	Destructor. More...

const std::string &	name () const

void	shutdown ()

bool	isShutdown () const

Static Public Member Functions
static std::shared_ptr< AudioInputProcessor >	create (std::shared_ptr< avsCommon::sdkInterfaces::DirectiveSequencerInterface > directiveSequencer, std::shared_ptr< avsCommon::sdkInterfaces::MessageSenderInterface > messageSender, std::shared_ptr< avsCommon::sdkInterfaces::ContextManagerInterface > contextManager, std::shared_ptr< avsCommon::sdkInterfaces::FocusManagerInterface > focusManager, std::shared_ptr< avsCommon::avs::DialogUXStateAggregator > dialogUXStateAggregator, std::shared_ptr< avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface > exceptionEncounteredSender, std::shared_ptr< avsCommon::sdkInterfaces::UserInactivityMonitorInterface > userInactivityNotifier, std::shared_ptr< avsCommon::sdkInterfaces::SystemSoundPlayerInterface > systemSoundPlayer, const std::shared_ptr< avsCommon::sdkInterfaces::LocaleAssetsManagerInterface > &assetsManager, std::shared_ptr< settings::WakeWordConfirmationSetting > wakeWordConfirmation, std::shared_ptr< settings::SpeechConfirmationSetting > speechConfirmation, const std::shared_ptr< avsCommon::avs::CapabilityChangeNotifierInterface > &capabilityChangeNotifier, std::shared_ptr< settings::WakeWordsSetting > wakeWordsSetting=nullptr, std::shared_ptr< audioEncoderInterfaces::AudioEncoderInterface > audioEncoder=nullptr, AudioProvider defaultAudioProvider=AudioProvider::null(), std::shared_ptr< avsCommon::sdkInterfaces::PowerResourceManagerInterface > powerResourceManager=nullptr, std::shared_ptr< avsCommon::utils::metrics::MetricRecorderInterface > metricRecorder=nullptr, const std::shared_ptr< ExpectSpeechTimeoutHandler > &expectSpeechTimeoutHandler=nullptr)

static settings::SettingEventMetadata	getWakeWordsEventsMetadata ()

static settings::SettingEventMetadata	getWakeWordConfirmationMetadata ()

static settings::SettingEventMetadata	getSpeechConfirmationMetadata ()

Static Public Member Functions inherited from alexaClientSDK::avsCommon::sdkInterfaces::DialogUXStateObserverInterface
static std::string	stateToString (DialogUXState state)

Static Public Attributes
static constexpr const char *	KEYWORD_TEXT_STOP = "STOP"
	A special keyword sent by supported wakeword engines for "Alexa, Stop". More...

static const auto	INVALID_INDEX = std::numeric_limits<avsCommon::avs::AudioInputStream::Index>::max()
	A reserved `Index` value which is considered invalid. More...

Additional Inherited Members
Protected Member Functions inherited from alexaClientSDK::avsCommon::avs::CapabilityAgent
	CapabilityAgent (const std::string &nameSpace, std::shared_ptr< sdkInterfaces::ExceptionEncounteredSenderInterface > exceptionEncounteredSender)

virtual std::shared_ptr< DirectiveInfo >	createDirectiveInfo (std::shared_ptr< AVSDirective > directive, std::unique_ptr< sdkInterfaces::DirectiveHandlerResultInterface > result)

void	removeDirective (const std::string &messageId)

void	sendExceptionEncounteredAndReportFailed (std::shared_ptr< DirectiveInfo > info, const std::string &message, avsCommon::avs::ExceptionErrorType type=avsCommon::avs::ExceptionErrorType::INTERNAL_ERROR)

const std::pair< std::string, std::string >	buildJsonEventString (const std::string &eventName, const std::string &dialogRequestIdString="", const std::string &payload="{}", const std::string &context="") const

Protected Attributes inherited from alexaClientSDK::avsCommon::avs::CapabilityAgent
const std::string	m_namespace
	The namespace of the capability agent. More...

std::shared_ptr< sdkInterfaces::ExceptionEncounteredSenderInterface >	m_exceptionEncounteredSender
	Object to use to send exceptionEncountered messages. More...

Detailed Description

This class implements a SpeechRecognizer capability agent.

See also: https://developer.amazon.com/public/solutions/alexa/alexa-voice-service/reference/speechrecognizer

AudioInputProcessor is a top-level component which should be instantiated in application code, and connected up to other interfaces in the Alexa Client SDK. Interfaces which are used directly by the AudioInputProcessor are passed directly to its create() function. To start sending an audio event, application code should call one of the recognize() functions. To stop sending audio, application code should call the stopCapture() function. Application code can also register objects which implement the ObserverInterface to receive notifications when the AudioInputProcessor state changes.

Member Typedef Documentation

◆ EncodingFormatRequest

using alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::EncodingFormatRequest = std:: map<std::string, std::pair<avsCommon::utils::AudioFormat::Encoding, avsCommon::utils::AudioFormat::Encoding> >

Request to configure AudioInputProcessor to provide multiple audio streams for a single Recognize event. Key in the map is resolve key, which will be used by caller to resolve the unresolved MessageRequest. Each resolve key correlates to a pair of encoding formats. The first format is the preferred format, and the second one is the fallback format which is used if the preferred one is not supported.

◆ EncodingFormatResponse

using alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::EncodingFormatResponse = std::map<std::string, avsCommon::utils::AudioFormat::Encoding>

Response to caller when AIP receives EncodingFormatRequest. Key in the map is resolve key from the request, and corresponding value is the confirmed encoding format that AIP will provide for this resolve key.

◆ ExpectSpeechTimeoutHandler

using alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::ExpectSpeechTimeoutHandler = avsCommon::sdkInterfaces::ExpectSpeechTimeoutHandlerInterface

This function allows applications to tell the AudioInputProcessor that the ExpectSpeech directive's timeout will be handled externally and stops the AudioInputProcessor from starting an internal timer to handle it.

Parameters

timeout	The timeout of the `ExpectSpeech` directive.
expectSpeechTimedOut	An `std::function` that applications may call if the timeout expires. This results in an `ExpectSpeechTimedOut` event being sent to AVS if no `recognize()` call is made prior to the timeout expiring. This function will return a future which is `true` if called in the correct state and an `ExpectSpeechTimeout` Event was sent successfully, or `false` otherwise.

Returns: true if the ExpectSpeech directive's timeout will be handled externally and should not be handled via an internal timer owned by the AudioInputProcessor.

Note: This function will be called after any calls to the AudioInputProcessorObserverInterface's onStateChanged() method to notify of a state change to EXPECTING_SPEECH.; Implementations are not required to be thread-safe.; If the object that this call came from is destroyed, the expectSpeechTimedOut parameter is no longer valid.

◆ ObserverInterface

using alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::ObserverInterface = avsCommon::sdkInterfaces::AudioInputProcessorObserverInterface

Alias to the AudioInputProcessorObserverInterface for brevity.

Member Function Documentation

◆ addObserver()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::addObserver ( std::shared_ptr< ObserverInterface > observer )

Adds an observer to be notified of AudioInputProcessor state changes.

Parameters

observer The observer object to add.

◆ cancelDirective()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::cancelDirective ( std::shared_ptr< DirectiveInfo > info )

overridevirtual

Cancel an ongoing preHandleDirective() or handleDirective() operation for the AVSDirective in . Once this has been called the CapabilityAgent should not expect to receive further calls regarding this directive.

Note: The implementation of this method MUST be thread-safe.; The implementation of this method MUST return quickly. Failure to do so blocks the processing of subsequent AVSDirectives.

Parameters

info	The `DirectiveInfo` instance for the `AVSDirective` to process.

Implements alexaClientSDK::avsCommon::avs::CapabilityAgent.

◆ create()

static std::shared_ptr<AudioInputProcessor> alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::create	(	std::shared_ptr< avsCommon::sdkInterfaces::DirectiveSequencerInterface >	directiveSequencer,
		std::shared_ptr< avsCommon::sdkInterfaces::MessageSenderInterface >	messageSender,
		std::shared_ptr< avsCommon::sdkInterfaces::ContextManagerInterface >	contextManager,
		std::shared_ptr< avsCommon::sdkInterfaces::FocusManagerInterface >	focusManager,
		std::shared_ptr< avsCommon::avs::DialogUXStateAggregator >	dialogUXStateAggregator,
		std::shared_ptr< avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface >	exceptionEncounteredSender,
		std::shared_ptr< avsCommon::sdkInterfaces::UserInactivityMonitorInterface >	userInactivityNotifier,
		std::shared_ptr< avsCommon::sdkInterfaces::SystemSoundPlayerInterface >	systemSoundPlayer,
		const std::shared_ptr< avsCommon::sdkInterfaces::LocaleAssetsManagerInterface > &	assetsManager,
		std::shared_ptr< settings::WakeWordConfirmationSetting >	wakeWordConfirmation,
		std::shared_ptr< settings::SpeechConfirmationSetting >	speechConfirmation,
		const std::shared_ptr< avsCommon::avs::CapabilityChangeNotifierInterface > &	capabilityChangeNotifier,
		std::shared_ptr< settings::WakeWordsSetting >	wakeWordsSetting = `nullptr`,
		std::shared_ptr< audioEncoderInterfaces::AudioEncoderInterface >	audioEncoder = `nullptr`,
		AudioProvider	defaultAudioProvider = `AudioProvider::null()`,
		std::shared_ptr< avsCommon::sdkInterfaces::PowerResourceManagerInterface >	powerResourceManager = `nullptr`,
		std::shared_ptr< avsCommon::utils::metrics::MetricRecorderInterface >	metricRecorder = `nullptr`,
		const std::shared_ptr< ExpectSpeechTimeoutHandler > &	expectSpeechTimeoutHandler = `nullptr`
	)

static

Creates a new AudioInputProcessor instance.

Parameters

directiveSequencer	The Directive Sequencer to register with for receiving directives.
messageSender	The object to use for sending events.
contextManager	The AVS Context manager used to generate system context for events.
focusManager	The channel focus manager used to manage usage of the dialog channel.
dialogUXStateAggregator	The dialog state aggregator which tracks UX states related to dialog.
exceptionEncounteredSender	The object to use for sending AVS Exception messages.
userInactivityNotifier	The object to use for resetting user inactivity.
systemSoundPlayer	The instance of the system sound player.
assetsManager	Responsible for retrieving and changing the wake words and locale.
wakeWordConfirmation	The wake word confirmation setting.
speechConfirmation	The end of speech confirmation setting.
capabilityChangeNotifier	The object with which to notify observers of `AudioInputProcessor` capability configurations change.
wakeWordsSetting	The setting that represents the enabled wake words. This parameter is required if this device supports wake words.
audioEncoder	The Audio Encoder used to encode audio inputs. This parameter is optional and defaults to nullptr, which disable the encoding feature.
defaultAudioProvider	A default `avsCommon::AudioProvider` to use for ExpectSpeech if the previous provider is not readable (`avsCommon::AudioProvider::alwaysReadable`). This parameter is optional and defaults to an invalid `avsCommon::AudioProvider`.
powerResourceManager	Power Resource Manager.
metricRecorder	The metric recorder.
expectSpeechTimeoutHandler	An optional interface that applications may provide to specify external handling of the `ExpectSpeech` directive's timeout. If provided, this function must remain valid for the lifetime of the `AudioInputProcessor`.

Returns: A std::shared_ptr to the new AudioInputProcessor instance.

◆ getCapabilityConfigurations()

std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration> > alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::getCapabilityConfigurations ( )

overridevirtual

Returns the configurations of the capability interfaces being implemented.

Returns: A set of CapabilityConfigurations

Implements alexaClientSDK::avsCommon::sdkInterfaces::CapabilityConfigurationInterface.

◆ getConfiguration()

avsCommon::avs::DirectiveHandlerConfiguration alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::getConfiguration ( ) const

overridevirtual

Returns the configuration of the directive handler.

The configuration consists of multiple directive routing rules and their respective blocking policy. The directives will be matched from the most specific rule (with all fields defined) to the least specific rule (which only matches the directive endpointId).

Returns: The avs::DirectiveHandlerConfiguration of the handler.

Implements alexaClientSDK::avsCommon::sdkInterfaces::DirectiveHandlerInterface.

◆ getEncodingAudioFormats()

EncodingFormatResponse alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::getEncodingAudioFormats ( ) const

Function to get encoding audio formats.

Returns: Encoding formats requested for multiple audio streams.

◆ getSpeechConfirmationMetadata()

static settings::SettingEventMetadata alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::getSpeechConfirmationMetadata ( )

static

Gets the speech confirmation events metadata.

Returns: The speech confirmation event metadata.

◆ getWakeWordConfirmationMetadata()

static settings::SettingEventMetadata alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::getWakeWordConfirmationMetadata ( )

static

Gets the wake word confirmation events metadata.

Returns: The wake word confirmation event metadata.

◆ getWakeWordsEventsMetadata()

static settings::SettingEventMetadata alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::getWakeWordsEventsMetadata ( )

static

Gets the wake words events metadata.

Returns: The wake words events metadata.

◆ handleDirective()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::handleDirective ( std::shared_ptr< DirectiveInfo > info )

overridevirtual

Handle the action specified by the AVSDirective in info. The handling of subsequent directives with the same DialogRequestId may be blocked until the DirectiveHandler calls the setSucceeded() method of the DirectiveHandlingResult present in info. If handling of this directive fails setFailed() should be called to indicate a failure.

Note: The implementation of this method MUST be thread-safe.; The implementation of this method MUST return quickly. Failure to do so blocks the processing of subsequent AVSDirectives.

Parameters

info	The `DirectiveInfo` instance for the `AVSDirective` to process.

Implements alexaClientSDK::avsCommon::avs::CapabilityAgent.

◆ handleDirectiveImmediately()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::handleDirectiveImmediately ( std::shared_ptr< avsCommon::avs::AVSDirective > directive )

overridevirtual

Handle the action specified AVSDirective. Once this has been called the DirectiveHandler should not expect to receive further calls regarding this directive.

Note: The implementation of this method MUST be thread-safe.; The implementation of this method MUST return quickly. Failure to do so blocks the processing of subsequent AVSDirectives.; If this operation fails, an ExceptionEncountered message should be sent to AVS.

Parameters

directive The directive to handle.

Implements alexaClientSDK::avsCommon::sdkInterfaces::DirectiveHandlerInterface.

◆ onConnectionStatusChanged()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onConnectionStatusChanged ( bool connected )

overridevirtual

Take necessary actions as a result of an internet connection change.

Parameters

connected Whether or not we are currently connected to the internet.

Implements alexaClientSDK::avsCommon::sdkInterfaces::InternetConnectionObserverInterface.

◆ onContextAvailable()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onContextAvailable ( const std::string & jsonContext )

overridevirtual

This is called by the ContextManager once the context is ready and available.

Deprecated:: This method is being deprecated since it cannot specify the source endpoint.

Note: The ContextRequester should perform minimum processing and return quickly. Otherwise it will block the processing of updating the of other ContextProviders.

Parameters

jsonContext Context information.Context provided is of the format {"context"[{...}, {...}]}

Reimplemented from alexaClientSDK::avsCommon::sdkInterfaces::ContextRequesterInterface.

◆ onContextFailure()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onContextFailure ( const avsCommon::sdkInterfaces::ContextRequestError error )

overridevirtual

The contextManager calls this if it is unable to process a getContext request successfully.

Deprecated:: This method is being deprecated since it cannot specify the optional token.

Note: The ContextRequester should perform minimum processing and return quickly. Otherwise it will block the processing of updating the of other ContextProviders.

Parameters

error The reason why the getContext request failed.

Reimplemented from alexaClientSDK::avsCommon::sdkInterfaces::ContextRequesterInterface.

◆ onDeregistered()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onDeregistered ( )

overridevirtual

Notification that this handler has been de-registered and will not receive any more calls.

Reimplemented from alexaClientSDK::avsCommon::avs::CapabilityAgent.

◆ onDialogUXStateChanged()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onDialogUXStateChanged ( avsCommon::sdkInterfaces::DialogUXStateObserverInterface::DialogUXState newState )

overridevirtual

This function is called whenever the AVS UX dialog state of the system changes. This function will block processing of other state changes, so any implementation of this should return quickly.

Parameters

newState The new dialog specific AVS UX state.

Implements alexaClientSDK::avsCommon::sdkInterfaces::DialogUXStateObserverInterface.

◆ onExceptionReceived()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onExceptionReceived ( const std::string & exceptionMessage )

overridevirtual

Implements alexaClientSDK::avsCommon::sdkInterfaces::MessageRequestObserverInterface.

◆ onFocusChanged()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onFocusChanged	(	avsCommon::avs::FocusState	newFocus,
		avsCommon::avs::MixingBehavior	behavior
	)

overridevirtual

Used to notify the observer of the Channel of focus changes. Once called, the client should make a user observable change only and return immediately. Any additional work that needs to be done should be done on a separate thread or after returning. "User observable change" here refers to events that the end user of the product can visibly see or hear. For example, Alexa speech or music playing would be examples of user observable changes. Other work, such as database storing, logging, or communicating via network should be done on a different thread. Not doing so could result in delays for other clients trying to access the Channel.

Parameters

newFocus	The new Focus of the channel.
behavior	The mixingBehavior for the ChannelObserver to take as per the interrupt model

Note: when newFocus is FocusState::FOREGROUND, the MixingBehavior shall be guaranteed to be PRIMARY when newFocus is FocusState::NONE, the MixingBehavior shall be guaranteed to be MUST_STOP

Reimplemented from alexaClientSDK::avsCommon::avs::CapabilityAgent.

◆ onLocaleAssetsChanged()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onLocaleAssetsChanged ( )

overridevirtual

Used to notify the observer of locale assets(wakewords, locales) update.

Implements alexaClientSDK::avsCommon::sdkInterfaces::LocaleAssetsObserverInterface.

◆ onResponseStatusReceived()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onResponseStatusReceived ( avsCommon::sdkInterfaces::MessageRequestObserverInterface::Status status )

overridevirtual

Called when the Response code is received.

Parameters

status The status of the response that was received.

Reimplemented from alexaClientSDK::avsCommon::sdkInterfaces::MessageRequestObserverInterface.

◆ onSendCompleted()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::onSendCompleted ( avsCommon::sdkInterfaces::MessageRequestObserverInterface::Status status )

overridevirtual

Implements alexaClientSDK::avsCommon::sdkInterfaces::MessageRequestObserverInterface.

◆ preHandleDirective()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::preHandleDirective ( std::shared_ptr< DirectiveInfo > info )

overridevirtual

Notification that a directive has arrived. This notification gives the DirectiveHandler a chance to prepare for handling of an AVSDirective. If an error occurs during the pre-Handling phase and that error should cancel the handling of subsequent AVSDirectives with the same DialogRequestId, the DirectiveHandler should call the setFailed method on the result instance passed in to this call.

Note: The implementation of this method MUST be thread-safe.; The implementation of this method MUST return quickly. Failure to do so blocks the processing of subsequent AVSDirectives.

Parameters

info	The `DirectiveInfo` instance for the `AVSDirective` to process.

Implements alexaClientSDK::avsCommon::avs::CapabilityAgent.

◆ recognize()

std::future<bool> alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::recognize	(	AudioProvider	audioProvider,
		Initiator	initiator,
		std::chrono::steady_clock::time_point	startOfSpeechTimestamp = `std::chrono::steady_clock::now()`,
		avsCommon::avs::AudioInputStream::Index	begin = `INVALID_INDEX`,
		avsCommon::avs::AudioInputStream::Index	keywordEnd = `INVALID_INDEX`,
		std::string	keyword = `""`,
		std::shared_ptr< const std::vector< char >>	KWDMetadata = `nullptr`,
		const std::string &	initiatorToken = `""`
	)

This function asks the AudioInputProcessor to send a Recognize Event to AVS and start streaming from audioProvider, which transitions it to the RECOGNIZING state. This function can be called in any state except BUSY, however the flags in AudioProvider will dictate whether the call is allowed to override an ongoing Recognize Event. If the flags do not allow an override, no event will be sent, no state change will occur, and the function will fail.

A special case is that the function will also fail if the keyword passed in is equal to KEYWORD_TEXT_STOP. This check is case insensitive.

Note: This function will not pass the audio stream to MessageSenderInterface to start streaming if the the start index or any subsequent data has already expired from the buffer. In addition, it is assumed that MessageSenderInterface will stop streaming immediately if it detects an overrun, and notify AIP of this condition (through a MessageRequest callback). These requirements ensure that the begin and keywordEnd indices will remain accurate for the actual audio samples sent to AVS.

False-wakeword detection in the cloud will be used when all of the following requirements are met:

initiator == Initiator::WAKEWORD
begin != INVALID_INDEX
end != INVALID_INDEX
!keyword.empty()

If all of the above requirements are met, audio steaming will start between 0 and 500ms before begin, and the cloud will perform additional verification of the wakeword audio before proceeding to recognize the subsequent audio.

Attention: User perceived latency metrics will only be accurate if the startOfSpeechTimestamp is correct. Some keyword detectors determine start of speech at different times, and in some cases exclude the wakeword. This leads to a later timestamp and excludes the wakeword duration from the user perceived latency calculation, thus underestimating the latency. Verify that the startOfSpeechTimestamp is including the wakeword duration if the audio stream is initiated by wakeword detection. (Tap-To-Talk remains unaffected)

Parameters

audioProvider	The `AudioProvider` to stream audio from.
initiator	The type of interface that initiated this recognize event.
startOfSpeechTimestamp	Moment in time when user started talking to Alexa. This parameter is optional and it is used to measure user perceived latency. The startOfSpeechTimestamp must include the wakeword duration if the audio stream is initiated by a wakeword, otherwise the latency calculation will not be correct.
begin	The `Index` in `audioProvider.stream` where audio streaming should begin. This parameter is optional, and defaults to `INVALID_INDEX`. When this parameter is not specified, `recognize()` will stream audio starting at the time of the `recognize()` call. If the `initiator` is `WAKEWORD`, and this and `keywordEnd` are specified, streaming will begin between 0 and 500ms prior to the `Index` specified by this parameter to attempt false wakeword validation.
keywordEnd	The `Index` in `audioProvider.stream` where the wakeword ends. This parameter is optional, and defaults to `INVALID_INDEX`. This parameter is ignored if initiator is not `WAKEWORD`.
keyword	The text of the keyword which was recognized. This parameter is optional, and defaults to an empty string. This parameter is ignored if initiator is not `WAKEWORD`. The only value currently accepted by AVS for keyword is "ALEXA". See https://developer.amazon.com/public/solutions/alexa/alexa-voice-service/reference/context#recognizerstate
KWDMetadata	Wake word engine metadata.
initiatorToken	An optional opaque string associated with the interaction.

Returns: A future which is true if the Recognize Event was started successfully, else false.

◆ removeObserver()

void alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::removeObserver ( std::shared_ptr< ObserverInterface > observer )

Removes an observer from the set of observers to be notified of AudioInputProcessor state changes.

Note: This is a synchronous call which can not be made by an observer callback. Attempting to call removeObserver() from ObserverInterface::onStateChanged() will result in a deadlock.

Parameters

observer The observer object to remove.

◆ requestEncodingAudioFormats()

EncodingFormatResponse alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::requestEncodingAudioFormats ( const EncodingFormatRequest & encodings )

Function to request multiple audio streams from AudioInputProcessor in a single Recognize event. This is an alternative API to setEncodingAudioFormat().

Parameters

encodings A map of resolveKey to a pair of encoding formats. Each resolveKey stands for an audio stream. The first encoding format is the requested format, and the second one is backup format if the requested one isn't supported by AudioInputProcessor.

Returns: A map of resolveKey to result encoding format. If the requested format is supported, the result will be the requested one. If not, AudioInputProcessor will check backup format. If neither of them is supported, the corresponding resolve key will be removed from result.

◆ resetState()

std::future<void> alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::resetState ( )

This function forces the AudioInputProcessor back to the IDLE state. This function can be called in any state, and will end any Event which is currently in progress.

Returns: A future which indicates when the AudioInputProcessor is back to the IDLE state.

◆ setEncodingAudioFormat()

bool alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::setEncodingAudioFormat ( avsCommon::utils::AudioFormat::Encoding encoding )

Set encoding for the audio format. The new encoding will be used for future utterances. Any audio stream already in progress will not be affected. This is an alternative API to requestEncodingAudioFormats(), but will configure AudioInputProcessor to only produce one audio stream.

Parameters

encoding The encoding format to use.

Returns: on success, false on failure to set the encoding.

◆ stopCapture()

std::future<bool> alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::stopCapture ( )

This function asks the AudioInputProcessor to stop streaming audio and end an ongoing Recognize Event, which transitions it to the BUSY state. This function can only be called in the RECOGNIZING state; calling it while in another state will fail.

Returns: A future which is true if called in the correct state and a Recognize Event's audio streaming was stopped successfully, else false.

Member Data Documentation

◆ INVALID_INDEX

const auto alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::INVALID_INDEX = std::numeric_limits<avsCommon::avs::AudioInputStream::Index>::max()

static

A reserved Index value which is considered invalid.

◆ KEYWORD_TEXT_STOP

constexpr const char* alexaClientSDK::capabilityAgents::aip::AudioInputProcessor::KEYWORD_TEXT_STOP = "STOP"

static

A special keyword sent by supported wakeword engines for "Alexa, Stop".

The documentation for this class was generated from the following file:

/workplace/avs-device-sdk/CapabilityAgents/AIP/include/AIP/AudioInputProcessor.h

Public Types

Public Member Functions

Static Public Member Functions

Static Public Attributes

Additional Inherited Members

Detailed Description

Member Typedef Documentation

◆ EncodingFormatRequest

◆ EncodingFormatResponse

◆ ExpectSpeechTimeoutHandler

◆ ObserverInterface

Member Function Documentation

◆ addObserver()

◆ cancelDirective()

◆ create()

◆ getCapabilityConfigurations()

◆ getConfiguration()

◆ getEncodingAudioFormats()

◆ getSpeechConfirmationMetadata()

◆ getWakeWordConfirmationMetadata()

◆ getWakeWordsEventsMetadata()

◆ handleDirective()

◆ handleDirectiveImmediately()

◆ onConnectionStatusChanged()

◆ onContextAvailable()

◆ onContextFailure()

◆ onDeregistered()

◆ onDialogUXStateChanged()

◆ onExceptionReceived()

◆ onFocusChanged()

◆ onLocaleAssetsChanged()

◆ onResponseStatusReceived()

◆ onSendCompleted()

◆ preHandleDirective()

◆ recognize()

◆ removeObserver()

◆ requestEncodingAudioFormats()

◆ resetState()

◆ setEncodingAudioFormat()

◆ stopCapture()

Member Data Documentation

◆ INVALID_INDEX

◆ KEYWORD_TEXT_STOP