AudioInputProcessor.h

Go to the documentation of this file.

/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *     http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

#ifndef ALEXA_CLIENT_SDK_CAPABILITYAGENTS_AIP_INCLUDE_AIP_AUDIOINPUTPROCESSOR_H_
#define ALEXA_CLIENT_SDK_CAPABILITYAGENTS_AIP_INCLUDE_AIP_AUDIOINPUTPROCESSOR_H_

#include <chrono>
#include <map>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include <acsdk/AudioEncoderInterfaces/AudioEncoderInterface.h>
#include <AVSCommon/AVS/Attachment/InProcessAttachmentReader.h>
#include <AVSCommon/AVS/CapabilityAgent.h>
#include <AVSCommon/AVS/CapabilityConfiguration.h>
#include <AVSCommon/AVS/CapabilityChangeNotifierInterface.h>
#include <AVSCommon/AVS/DialogUXStateAggregator.h>
#include <AVSCommon/AVS/DirectiveHandlerConfiguration.h>
#include <AVSCommon/AVS/ExceptionErrorType.h>
#include <AVSCommon/AVS/EditableMessageRequest.h>
#include <AVSCommon/AVS/MessageRequest.h>
#include <AVSCommon/SDKInterfaces/AudioInputProcessorObserverInterface.h>
#include <AVSCommon/SDKInterfaces/CapabilityConfigurationInterface.h>
#include <AVSCommon/SDKInterfaces/ChannelObserverInterface.h>
#include <AVSCommon/SDKInterfaces/ContextManagerInterface.h>
#include <AVSCommon/SDKInterfaces/DialogUXStateObserverInterface.h>
#include <AVSCommon/SDKInterfaces/DirectiveSequencerInterface.h>
#include <AVSCommon/SDKInterfaces/ExceptionEncounteredSenderInterface.h>
#include <AVSCommon/SDKInterfaces/ExpectSpeechTimeoutHandlerInterface.h>
#include <AVSCommon/SDKInterfaces/FocusManagerInterface.h>
#include <AVSCommon/SDKInterfaces/InternetConnectionObserverInterface.h>
#include <AVSCommon/SDKInterfaces/LocaleAssetsManagerInterface.h>
#include <AVSCommon/SDKInterfaces/LocaleAssetsObserverInterface.h>
#include <AVSCommon/SDKInterfaces/MessageRequestObserverInterface.h>
#include <AVSCommon/SDKInterfaces/MessageSenderInterface.h>
#include <AVSCommon/SDKInterfaces/PowerResourceManagerInterface.h>
#include <AVSCommon/SDKInterfaces/SystemSoundPlayerInterface.h>
#include <AVSCommon/SDKInterfaces/UserInactivityMonitorInterface.h>
#include <AVSCommon/Utils/AudioFormat.h>
#include <AVSCommon/Utils/Metrics/DataPointDurationBuilder.h>
#include <AVSCommon/Utils/Metrics/MetricRecorderInterface.h>
#include <AVSCommon/Utils/RequiresShutdown.h>
#include <AVSCommon/Utils/Threading/Executor.h>
#include <AVSCommon/Utils/Timing/Timer.h>
#include <Settings/DeviceSettingsManager.h>
#include <Settings/SettingEventMetadata.h>

#include "AudioProvider.h"
#include "Initiator.h"

namespace alexaClientSDK {
namespace capabilityAgents {
namespace aip {

class AudioInputProcessor
        : public avsCommon::avs::CapabilityAgent
        , public avsCommon::sdkInterfaces::CapabilityConfigurationInterface
        , public avsCommon::sdkInterfaces::LocaleAssetsObserverInterface
        , public avsCommon::sdkInterfaces::DialogUXStateObserverInterface
        , public avsCommon::sdkInterfaces::MessageRequestObserverInterface
        , public avsCommon::sdkInterfaces::InternetConnectionObserverInterface
        , public avsCommon::utils::RequiresShutdown
        , public std::enable_shared_from_this<AudioInputProcessor> {
public:
    using ObserverInterface = avsCommon::sdkInterfaces::AudioInputProcessorObserverInterface;

    using EncodingFormatRequest = std::
        map<std::string, std::pair<avsCommon::utils::AudioFormat::Encoding, avsCommon::utils::AudioFormat::Encoding>>;

    using EncodingFormatResponse = std::map<std::string, avsCommon::utils::AudioFormat::Encoding>;

    using ExpectSpeechTimeoutHandler = avsCommon::sdkInterfaces::ExpectSpeechTimeoutHandlerInterface;

    static constexpr const char* KEYWORD_TEXT_STOP = "STOP";

    static const auto INVALID_INDEX = std::numeric_limits<avsCommon::avs::AudioInputStream::Index>::max();

    static std::shared_ptr<AudioInputProcessor> create(
        std::shared_ptr<avsCommon::sdkInterfaces::DirectiveSequencerInterface> directiveSequencer,
        std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
        std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
        std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
        std::shared_ptr<avsCommon::avs::DialogUXStateAggregator> dialogUXStateAggregator,
        std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionEncounteredSender,
        std::shared_ptr<avsCommon::sdkInterfaces::UserInactivityMonitorInterface> userInactivityNotifier,
        std::shared_ptr<avsCommon::sdkInterfaces::SystemSoundPlayerInterface> systemSoundPlayer,
        const std::shared_ptr<avsCommon::sdkInterfaces::LocaleAssetsManagerInterface>& assetsManager,
        std::shared_ptr<settings::WakeWordConfirmationSetting> wakeWordConfirmation,
        std::shared_ptr<settings::SpeechConfirmationSetting> speechConfirmation,
        const std::shared_ptr<avsCommon::avs::CapabilityChangeNotifierInterface>& capabilityChangeNotifier,
        std::shared_ptr<settings::WakeWordsSetting> wakeWordsSetting = nullptr,
        std::shared_ptr<audioEncoderInterfaces::AudioEncoderInterface> audioEncoder = nullptr,
        AudioProvider defaultAudioProvider = AudioProvider::null(),
        std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr,
        std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder = nullptr,
        const std::shared_ptr<ExpectSpeechTimeoutHandler>& expectSpeechTimeoutHandler = nullptr);

    void addObserver(std::shared_ptr<ObserverInterface> observer);

    void removeObserver(std::shared_ptr<ObserverInterface> observer);

    std::future<bool> recognize(
        AudioProvider audioProvider,
        Initiator initiator,
        std::chrono::steady_clock::time_point startOfSpeechTimestamp = std::chrono::steady_clock::now(),
        avsCommon::avs::AudioInputStream::Index begin = INVALID_INDEX,
        avsCommon::avs::AudioInputStream::Index keywordEnd = INVALID_INDEX,
        std::string keyword = "",
        std::shared_ptr<const std::vector<char>> KWDMetadata = nullptr,
        const std::string& initiatorToken = "");

    std::future<bool> stopCapture();

    std::future<void> resetState();

    void onContextAvailable(const std::string& jsonContext) override;
    void onContextFailure(const avsCommon::sdkInterfaces::ContextRequestError error) override;

    void onResponseStatusReceived(avsCommon::sdkInterfaces::MessageRequestObserverInterface::Status status) override;
    void onSendCompleted(avsCommon::sdkInterfaces::MessageRequestObserverInterface::Status status) override;
    void onExceptionReceived(const std::string& exceptionMessage) override;

    void handleDirectiveImmediately(std::shared_ptr<avsCommon::avs::AVSDirective> directive) override;
    void preHandleDirective(std::shared_ptr<DirectiveInfo> info) override;
    void handleDirective(std::shared_ptr<DirectiveInfo> info) override;
    void cancelDirective(std::shared_ptr<DirectiveInfo> info) override;
    void onDeregistered() override;
    avsCommon::avs::DirectiveHandlerConfiguration getConfiguration() const override;

    void onFocusChanged(avsCommon::avs::FocusState newFocus, avsCommon::avs::MixingBehavior behavior) override;

    void onDialogUXStateChanged(
        avsCommon::sdkInterfaces::DialogUXStateObserverInterface::DialogUXState newState) override;

    std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> getCapabilityConfigurations() override;

    void onLocaleAssetsChanged() override;

    void onConnectionStatusChanged(bool connected) override;

    static settings::SettingEventMetadata getWakeWordsEventsMetadata();

    static settings::SettingEventMetadata getWakeWordConfirmationMetadata();

    static settings::SettingEventMetadata getSpeechConfirmationMetadata();

    bool setEncodingAudioFormat(avsCommon::utils::AudioFormat::Encoding encoding);

    EncodingFormatResponse requestEncodingAudioFormats(const EncodingFormatRequest& encodings);

    EncodingFormatResponse getEncodingAudioFormats() const;

private:
    bool initialize();

    AudioInputProcessor(
        std::shared_ptr<avsCommon::sdkInterfaces::DirectiveSequencerInterface> directiveSequencer,
        std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
        std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
        std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
        std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionEncounteredSender,
        std::shared_ptr<avsCommon::sdkInterfaces::UserInactivityMonitorInterface> userInactivityMonitor,
        std::shared_ptr<avsCommon::sdkInterfaces::SystemSoundPlayerInterface> systemSoundPlayer,
        const std::shared_ptr<avsCommon::sdkInterfaces::LocaleAssetsManagerInterface>& assetsManager,
        std::shared_ptr<audioEncoderInterfaces::AudioEncoderInterface> audioEncoder,
        AudioProvider defaultAudioProvider,
        std::shared_ptr<settings::WakeWordConfirmationSetting> wakeWordConfirmation,
        std::shared_ptr<settings::SpeechConfirmationSetting> speechConfirmation,
        const std::shared_ptr<avsCommon::avs::CapabilityChangeNotifierInterface>& capabilityChangeNotifier,
        std::shared_ptr<settings::WakeWordsSetting> wakeWordsSetting,
        std::shared_ptr<avsCommon::avs::CapabilityConfiguration> capabilitiesConfiguration,
        std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager,
        std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder,
        const std::shared_ptr<ExpectSpeechTimeoutHandler>& expectSpeechTimeoutHandler);

    void doShutdown() override;

    std::future<bool> expectSpeechTimedOut();

    void handleStopCaptureDirective(std::shared_ptr<DirectiveInfo> info);

    void handleExpectSpeechDirective(std::shared_ptr<DirectiveInfo> info);

    void handleSetEndOfSpeechOffsetDirective(std::shared_ptr<DirectiveInfo> info);

    void handleDirectiveFailure(
        const std::string& errorMessage,
        std::shared_ptr<DirectiveInfo> info,
        avsCommon::avs::ExceptionErrorType errorType);


    bool executeRecognize(
        AudioProvider provider,
        Initiator initiator,
        std::chrono::steady_clock::time_point startOfSpeechTimestamp,
        avsCommon::avs::AudioInputStream::Index begin,
        avsCommon::avs::AudioInputStream::Index keywordEnd,
        const std::string& keyword,
        std::shared_ptr<const std::vector<char>> KWDMetadata,
        const std::string& initiatorToken);

    bool executeRecognize(
        AudioProvider provider,
        const std::string& initiatorJson,
        std::chrono::steady_clock::time_point startOfSpeechTimestamp = std::chrono::steady_clock::now(),
        avsCommon::avs::AudioInputStream::Index begin = INVALID_INDEX,
        avsCommon::avs::AudioInputStream::Index end = INVALID_INDEX,
        const std::string& keyword = "",
        std::shared_ptr<const std::vector<char>> KWDMetadata = nullptr,
        bool initiatedByWakeword = false,
        bool falseWakewordDetection = false,
        const std::string& initiatorString = "");

    void executeOnContextAvailable(const std::string& jsonContext);

    void executeOnContextFailure(const avsCommon::sdkInterfaces::ContextRequestError error);

    void executeOnFocusChanged(avsCommon::avs::FocusState newFocus);

    bool executeStopCapture(bool stopImmediately = false, std::shared_ptr<DirectiveInfo> info = nullptr);

    void executeResetState();

    bool executeExpectSpeech(std::chrono::milliseconds timeout, std::shared_ptr<DirectiveInfo> info);

    bool executeExpectSpeechTimedOut();

    void executeOnDialogUXStateChanged(
        avsCommon::sdkInterfaces::DialogUXStateObserverInterface::DialogUXState newState);

    void executeDisconnected();

    void executeOnLocaleAssetsChanged();

    void setState(ObserverInterface::State state);

    void removeDirective(std::shared_ptr<DirectiveInfo> info);

    void sendRequestNow();


    bool handleSetWakeWordConfirmation(std::shared_ptr<DirectiveInfo> info);

    bool handleSetSpeechConfirmation(std::shared_ptr<DirectiveInfo> info);

    bool handleSetWakeWords(std::shared_ptr<DirectiveInfo> info);

    void managePowerResource(ObserverInterface::State newState);

    avsCommon::avs::MessageRequest::MessageRequestResolveFunction getMessageRequestResolverLocked() const;

    void closeAttachmentReaders(
        avsCommon::avs::attachment::AttachmentReader::ClosePoint closePoint =
            avsCommon::avs::attachment::AttachmentReader::ClosePoint::AFTER_DRAINING_CURRENT_BUFFER);

    bool isEncodingFormatSupported(avsCommon::utils::AudioFormat::Encoding encodingFormat) const;

    bool isUsingEncoderLocked() const;

    bool multiStreamsRequestedLocked() const;

    std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> m_metricRecorder;

    std::shared_ptr<avsCommon::sdkInterfaces::DirectiveSequencerInterface> m_directiveSequencer;

    std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> m_messageSender;

    std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> m_contextManager;

    std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> m_focusManager;

    std::shared_ptr<avsCommon::sdkInterfaces::UserInactivityMonitorInterface> m_userInactivityMonitor;

    avsCommon::utils::timing::Timer m_expectingSpeechTimer;

    const std::shared_ptr<audioEncoderInterfaces::AudioEncoderInterface> m_encoder;


    std::unordered_set<std::shared_ptr<ObserverInterface>> m_observers;

    AudioProvider m_defaultAudioProvider;

    AudioProvider m_lastAudioProvider;

    std::unordered_map<std::string, std::vector<std::shared_ptr<avsCommon::avs::MessageRequest::NamedReader>>>
        m_attachmentReaders;

    std::string m_recognizePayload;

    std::shared_ptr<avsCommon::avs::MessageRequest> m_recognizeRequest;

    std::shared_ptr<avsCommon::avs::MessageRequest> m_recognizeRequestSent;

    ObserverInterface::State m_state;

    avsCommon::avs::FocusState m_focusState;

    bool m_preparingToSend;

    std::function<void()> m_deferredStopCapture;

    bool m_initialDialogUXStateReceived;

    bool m_localStopCapturePerformed;

    bool m_streamIsClosedInRecognizingState;

    std::shared_ptr<avsCommon::sdkInterfaces::SystemSoundPlayerInterface> m_systemSoundPlayer;

    std::shared_ptr<avsCommon::sdkInterfaces::LocaleAssetsManagerInterface> m_assetsManager;

    std::unique_ptr<std::string> m_precedingExpectSpeechInitiator;

    std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> m_capabilityConfigurations;

    std::shared_ptr<settings::WakeWordConfirmationSetting> m_wakeWordConfirmation;

    std::shared_ptr<settings::SpeechConfirmationSetting> m_speechConfirmation;

    std::shared_ptr<avsCommon::avs::CapabilityChangeNotifierInterface> m_capabilityChangeNotifier;

    std::shared_ptr<settings::WakeWordsSetting> m_wakeWordsSetting;

    std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> m_powerResourceManager;

    std::chrono::steady_clock::time_point m_stopCaptureReceivedTime;

    std::shared_ptr<ExpectSpeechTimeoutHandler> m_expectSpeechTimeoutHandler;

    std::string m_preCachedDialogRequestId;

    std::chrono::milliseconds m_timeSinceLastResumeMS;

    std::chrono::milliseconds m_timeSinceLastPartialMS;

    avsCommon::sdkInterfaces::PowerResourceManagerInterface::PartialStateBitSet m_resourceFlags;

    bool m_usingEncoder;

    std::mutex m_mutex;

    avsCommon::avs::MessageRequest::MessageRequestResolveFunction m_messageRequestResolver;

    EncodingFormatResponse m_encodingAudioFormats;

    mutable std::mutex m_encodingFormatMutex;

    unsigned int m_audioBytesForMetricThreshold;

    std::string m_uploadMetricName;

    avsCommon::utils::metrics::DataPointDurationBuilder m_fetchContextTimeMetricData;

    std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface::PowerResourceId> m_powerResourceId;

    avsCommon::utils::threading::Executor m_executor;
};

}  // namespace aip
}  // namespace capabilityAgents
}  // namespace alexaClientSDK

#endif  // ALEXA_CLIENT_SDK_CAPABILITYAGENTS_AIP_INCLUDE_AIP_AUDIOINPUTPROCESSOR_H_