AlexaClientSDK  1.20.1
A cross-platform, modular SDK for interacting with the Alexa Voice Service
AudioInputProcessor.h
Go to the documentation of this file.
1 /*
2  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License").
5  * You may not use this file except in compliance with the License.
6  * A copy of the License is located at
7  *
8  * http://aws.amazon.com/apache2.0/
9  *
10  * or in the "license" file accompanying this file. This file is distributed
11  * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12  * express or implied. See the License for the specific language governing
13  * permissions and limitations under the License.
14  */
15 
16 #ifndef ALEXA_CLIENT_SDK_CAPABILITYAGENTS_AIP_INCLUDE_AIP_AUDIOINPUTPROCESSOR_H_
17 #define ALEXA_CLIENT_SDK_CAPABILITYAGENTS_AIP_INCLUDE_AIP_AUDIOINPUTPROCESSOR_H_
18 
19 #include <chrono>
20 #include <map>
21 #include <memory>
22 #include <unordered_set>
23 #include <vector>
24 
53 
54 #include "AudioProvider.h"
55 #include "Initiator.h"
56 
57 namespace alexaClientSDK {
58 namespace capabilityAgents {
59 namespace aip {
60 
80  , public std::enable_shared_from_this<AudioInputProcessor> {
81 public:
84 
86  static constexpr const char* KEYWORD_TEXT_STOP = "STOP";
87 
89  static const auto INVALID_INDEX = std::numeric_limits<avsCommon::avs::AudioInputStream::Index>::max();
90 
116  static std::shared_ptr<AudioInputProcessor> create(
117  std::shared_ptr<avsCommon::sdkInterfaces::DirectiveSequencerInterface> directiveSequencer,
118  std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
119  std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
120  std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
121  std::shared_ptr<avsCommon::avs::DialogUXStateAggregator> dialogUXStateAggregator,
122  std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionEncounteredSender,
123  std::shared_ptr<avsCommon::sdkInterfaces::UserInactivityMonitorInterface> userInactivityNotifier,
124  std::shared_ptr<avsCommon::sdkInterfaces::SystemSoundPlayerInterface> systemSoundPlayer,
125  const std::shared_ptr<avsCommon::sdkInterfaces::LocaleAssetsManagerInterface>& assetsManager,
126  std::shared_ptr<settings::WakeWordConfirmationSetting> wakeWordConfirmation,
127  std::shared_ptr<settings::SpeechConfirmationSetting> speechConfirmation,
128  std::shared_ptr<settings::WakeWordsSetting> wakeWordsSetting = nullptr,
129  std::shared_ptr<speechencoder::SpeechEncoder> speechEncoder = nullptr,
130  AudioProvider defaultAudioProvider = AudioProvider::null(),
131  std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr,
132  std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder = nullptr);
133 
139  void addObserver(std::shared_ptr<ObserverInterface> observer);
140 
149  void removeObserver(std::shared_ptr<ObserverInterface> observer);
150 
197  std::future<bool> recognize(
198  AudioProvider audioProvider,
199  Initiator initiator,
200  std::chrono::steady_clock::time_point startOfSpeechTimestamp = std::chrono::steady_clock::now(),
203  std::string keyword = "",
204  std::shared_ptr<const std::vector<char>> KWDMetadata = nullptr,
205  const std::string& initiatorToken = "");
206 
215  std::future<bool> stopCapture();
216 
223  std::future<void> resetState();
224 
227  void onContextAvailable(const std::string& jsonContext) override;
230 
234  void onExceptionReceived(const std::string& exceptionMessage) override;
236 
239  void handleDirectiveImmediately(std::shared_ptr<avsCommon::avs::AVSDirective> directive) override;
240  void preHandleDirective(std::shared_ptr<DirectiveInfo> info) override;
241  void handleDirective(std::shared_ptr<DirectiveInfo> info) override;
242  void cancelDirective(std::shared_ptr<DirectiveInfo> info) override;
243  void onDeregistered() override;
246 
251 
257 
260  std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> getCapabilityConfigurations() override;
262 
265  void onConnectionStatusChanged(bool connected) override;
267 
274 
281 
288 
289 private:
318  std::shared_ptr<avsCommon::sdkInterfaces::DirectiveSequencerInterface> directiveSequencer,
319  std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
320  std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
321  std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
322  std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionEncounteredSender,
323  std::shared_ptr<avsCommon::sdkInterfaces::UserInactivityMonitorInterface> userInactivityMonitor,
324  std::shared_ptr<avsCommon::sdkInterfaces::SystemSoundPlayerInterface> systemSoundPlayer,
325  std::shared_ptr<speechencoder::SpeechEncoder> speechEncoder,
326  AudioProvider defaultAudioProvider,
327  std::shared_ptr<settings::WakeWordConfirmationSetting> wakeWordConfirmation,
328  std::shared_ptr<settings::SpeechConfirmationSetting> speechConfirmation,
329  std::shared_ptr<settings::WakeWordsSetting> wakeWordsSetting,
330  std::shared_ptr<avsCommon::avs::CapabilityConfiguration> capabilitiesConfiguration,
331  std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager,
332  std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder);
333 
336  void doShutdown() override;
338 
346  std::future<bool> expectSpeechTimedOut();
347 
353  void handleStopCaptureDirective(std::shared_ptr<DirectiveInfo> info);
354 
360  void handleExpectSpeechDirective(std::shared_ptr<DirectiveInfo> info);
361 
367  void handleSetEndOfSpeechOffsetDirective(std::shared_ptr<DirectiveInfo> info);
368 
376  void handleDirectiveFailure(
377  const std::string& errorMessage,
378  std::shared_ptr<DirectiveInfo> info,
380 
388 
409  bool executeRecognize(
410  AudioProvider provider,
411  Initiator initiator,
412  std::chrono::steady_clock::time_point startOfSpeechTimestamp,
415  const std::string& keyword,
416  std::shared_ptr<const std::vector<char>> KWDMetadata,
417  const std::string& initiatorToken);
418 
445  bool executeRecognize(
446  AudioProvider provider,
447  const std::string& initiatorJson,
448  std::chrono::steady_clock::time_point startOfSpeechTimestamp = std::chrono::steady_clock::now(),
451  const std::string& keyword = "",
452  std::shared_ptr<const std::vector<char>> KWDMetadata = nullptr,
453  bool initiatedByWakeword = false,
454  bool falseWakewordDetection = false);
455 
465  void executeOnContextAvailable(const std::string jsonContext);
466 
473  void executeOnContextFailure(const avsCommon::sdkInterfaces::ContextRequestError error);
474 
485  void executeOnFocusChanged(avsCommon::avs::FocusState newFocus);
486 
501  bool executeStopCapture(bool stopImmediately = false, std::shared_ptr<DirectiveInfo> info = nullptr);
502 
507  void executeResetState();
508 
522  bool executeExpectSpeech(std::chrono::milliseconds timeout, std::shared_ptr<DirectiveInfo> info);
523 
529  bool executeExpectSpeechTimedOut();
530 
537  void executeOnDialogUXStateChanged(
539 
543  void executeDisconnected();
544 
551  void setState(ObserverInterface::State state);
552 
558  void removeDirective(std::shared_ptr<DirectiveInfo> info);
559 
561  void sendRequestNow();
562 
564 
571  bool handleSetWakeWordConfirmation(std::shared_ptr<DirectiveInfo> info);
572 
579  bool handleSetSpeechConfirmation(std::shared_ptr<DirectiveInfo> info);
580 
587  bool handleSetWakeWords(std::shared_ptr<DirectiveInfo> info);
588 
593  void managePowerResource(ObserverInterface::State newState);
594 
596  std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> m_metricRecorder;
597 
599  std::shared_ptr<avsCommon::sdkInterfaces::DirectiveSequencerInterface> m_directiveSequencer;
600 
602  std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> m_messageSender;
603 
605  std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> m_contextManager;
606 
608  std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> m_focusManager;
609 
611  std::shared_ptr<avsCommon::sdkInterfaces::UserInactivityMonitorInterface> m_userInactivityMonitor;
612 
614  avsCommon::utils::timing::Timer m_expectingSpeechTimer;
615 
617  std::shared_ptr<speechencoder::SpeechEncoder> m_encoder;
618 
625 
628  std::unordered_set<std::shared_ptr<ObserverInterface>> m_observers;
629 
634  AudioProvider m_defaultAudioProvider;
635 
640  AudioProvider m_lastAudioProvider;
641 
647  std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> m_reader;
648 
654  std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> m_KWDMetadataReader;
655 
661  std::string m_recognizePayload;
662 
669  std::shared_ptr<avsCommon::avs::MessageRequest> m_recognizeRequest;
670 
672  std::shared_ptr<avsCommon::avs::MessageRequest> m_recognizeRequestSent;
673 
675  ObserverInterface::State m_state;
676 
678  avsCommon::avs::FocusState m_focusState;
679 
684  bool m_preparingToSend;
685 
690  std::function<void()> m_deferredStopCapture;
691 
693  bool m_initialDialogUXStateReceived;
694 
699  bool m_localStopCapturePerformed;
700 
702  std::shared_ptr<avsCommon::sdkInterfaces::SystemSoundPlayerInterface> m_systemSoundPlayer;
703 
713  std::unique_ptr<std::string> m_precedingExpectSpeechInitiator;
715 
717  std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> m_capabilityConfigurations;
718 
720  std::shared_ptr<settings::WakeWordConfirmationSetting> m_wakeWordConfirmation;
721 
723  std::shared_ptr<settings::SpeechConfirmationSetting> m_speechConfirmation;
724 
726  std::shared_ptr<settings::WakeWordsSetting> m_wakeWordsSetting;
727 
729  std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> m_powerResourceManager;
730 
732  std::chrono::steady_clock::time_point m_stopCaptureReceivedTime;
733 
741 
746  std::string m_preCachedDialogRequestId;
747 };
748 
749 } // namespace aip
750 } // namespace capabilityAgents
751 } // namespace alexaClientSDK
752 
753 #endif // ALEXA_CLIENT_SDK_CAPABILITYAGENTS_AIP_INCLUDE_AIP_AUDIOINPUTPROCESSOR_H_
void onConnectionStatusChanged(bool connected) override
Definition: AudioInputProcessor.cpp:1521
Definition: CapabilityAgent.h:47
void preHandleDirective(std::shared_ptr< DirectiveInfo > info) override
Definition: AudioInputProcessor.cpp:490
void addObserver(std::shared_ptr< ObserverInterface > observer)
Definition: AudioInputProcessor.cpp:404
MixingBehavior
Definition: MixingBehavior.h:25
Definition: SettingEventMetadata.h:27
static const auto INVALID_INDEX
A reserved Index value which is considered invalid.
Definition: AudioInputProcessor.h:89
void onSendCompleted(avsCommon::sdkInterfaces::MessageRequestObserverInterface::Status status) override
Definition: AudioInputProcessor.cpp:1352
static std::shared_ptr< AudioInputProcessor > create(std::shared_ptr< avsCommon::sdkInterfaces::DirectiveSequencerInterface > directiveSequencer, std::shared_ptr< avsCommon::sdkInterfaces::MessageSenderInterface > messageSender, std::shared_ptr< avsCommon::sdkInterfaces::ContextManagerInterface > contextManager, std::shared_ptr< avsCommon::sdkInterfaces::FocusManagerInterface > focusManager, std::shared_ptr< avsCommon::avs::DialogUXStateAggregator > dialogUXStateAggregator, std::shared_ptr< avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface > exceptionEncounteredSender, std::shared_ptr< avsCommon::sdkInterfaces::UserInactivityMonitorInterface > userInactivityNotifier, std::shared_ptr< avsCommon::sdkInterfaces::SystemSoundPlayerInterface > systemSoundPlayer, const std::shared_ptr< avsCommon::sdkInterfaces::LocaleAssetsManagerInterface > &assetsManager, std::shared_ptr< settings::WakeWordConfirmationSetting > wakeWordConfirmation, std::shared_ptr< settings::SpeechConfirmationSetting > speechConfirmation, std::shared_ptr< settings::WakeWordsSetting > wakeWordsSetting=nullptr, std::shared_ptr< speechencoder::SpeechEncoder > speechEncoder=nullptr, AudioProvider defaultAudioProvider=AudioProvider::null(), std::shared_ptr< avsCommon::sdkInterfaces::PowerResourceManagerInterface > powerResourceManager=nullptr, std::shared_ptr< avsCommon::utils::metrics::MetricRecorderInterface > metricRecorder=nullptr)
Definition: AudioInputProcessor.cpp:308
void onDialogUXStateChanged(avsCommon::sdkInterfaces::DialogUXStateObserverInterface::DialogUXState newState) override
Definition: AudioInputProcessor.cpp:553
static const AudioProvider & null()
Definition: AudioProvider.h:104
ContextRequestError
Definition: ContextRequesterInterface.h:40
ExceptionErrorType
Definition: ExceptionErrorType.h:28
void onContextAvailable(const std::string &jsonContext) override
Definition: AudioInputProcessor.cpp:478
FocusState
Definition: FocusState.h:29
void onExceptionReceived(const std::string &exceptionMessage) override
Definition: AudioInputProcessor.cpp:1347
std::future< bool > stopCapture()
Definition: AudioInputProcessor.cpp:470
static settings::SettingEventMetadata getSpeechConfirmationMetadata()
Definition: AudioInputProcessor.cpp:1395
DialogUXState
The different dialog specific AVS UX states.
Definition: DialogUXStateObserverInterface.h:32
void onContextFailure(const avsCommon::sdkInterfaces::ContextRequestError error) override
Definition: AudioInputProcessor.cpp:482
State
The different states the AudioInputProcessor can be in.
Definition: AudioInputProcessorObserverInterface.h:30
void handleDirectiveImmediately(std::shared_ptr< avsCommon::avs::AVSDirective > directive) override
Definition: AudioInputProcessor.cpp:486
Initiator
Definition: Initiator.h:28
std::unordered_map< directiveRoutingRule::DirectiveRoutingRule, BlockingPolicy > DirectiveHandlerConfiguration
Definition: DirectiveHandlerConfiguration.h:32
static settings::SettingEventMetadata getWakeWordsEventsMetadata()
Definition: AudioInputProcessor.cpp:644
std::unordered_set< std::shared_ptr< avsCommon::avs::CapabilityConfiguration > > getCapabilityConfigurations() override
Definition: AudioInputProcessor.cpp:1368
void removeObserver(std::shared_ptr< ObserverInterface > observer)
Definition: AudioInputProcessor.cpp:412
Index
Index used for setting access.
Definition: StateReportGeneratorTest.cpp:41
Whether or not curl logs should be emitted.
Definition: AVSConnectionManager.h:37
static constexpr const char * KEYWORD_TEXT_STOP
A special keyword sent by supported wakeword engines for "Alexa, Stop".
Definition: AudioInputProcessor.h:86
A state observer for an AudioInputProcessor.
Definition: AudioInputProcessorObserverInterface.h:27
std::future< void > resetState()
Definition: AudioInputProcessor.cpp:474
avsCommon::avs::DirectiveHandlerConfiguration getConfiguration() const override
Definition: AudioInputProcessor.cpp:393
Definition: InternetConnectionObserverInterface.h:26
std::future< bool > recognize(AudioProvider audioProvider, Initiator initiator, std::chrono::steady_clock::time_point startOfSpeechTimestamp=std::chrono::steady_clock::now(), avsCommon::avs::AudioInputStream::Index begin=INVALID_INDEX, avsCommon::avs::AudioInputStream::Index keywordEnd=INVALID_INDEX, std::string keyword="", std::shared_ptr< const std::vector< char >> KWDMetadata=nullptr, const std::string &initiatorToken="")
Definition: AudioInputProcessor.cpp:420
void onDeregistered() override
Definition: AudioInputProcessor.cpp:544
Status
Definition: MessageRequestObserverInterface.h:33
void cancelDirective(std::shared_ptr< DirectiveInfo > info) override
Definition: AudioInputProcessor.cpp:540
static settings::SettingEventMetadata getWakeWordConfirmationMetadata()
Definition: AudioInputProcessor.cpp:1388
A dialog specific UX state observer.
Definition: DialogUXStateObserverInterface.h:29
void onFocusChanged(avsCommon::avs::FocusState newFocus, avsCommon::avs::MixingBehavior behavior) override
Definition: AudioInputProcessor.cpp:548
void handleDirective(std::shared_ptr< DirectiveInfo > info) override
Definition: AudioInputProcessor.cpp:493

AlexaClientSDK 1.20.1 - Copyright 2016-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0