SpeechSynthesizer.h

Go to the documentation of this file.

/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *     http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

#ifndef ALEXA_CLIENT_SDK_CAPABILITYAGENTS_SPEECHSYNTHESIZER_INCLUDE_SPEECHSYNTHESIZER_SPEECHSYNTHESIZER_H_
#define ALEXA_CLIENT_SDK_CAPABILITYAGENTS_SPEECHSYNTHESIZER_INCLUDE_SPEECHSYNTHESIZER_SPEECHSYNTHESIZER_H_

#include <memory>
#include <mutex>
#include <string>
#include <unordered_set>
#include <deque>

#include <acsdkApplicationAudioPipelineFactoryInterfaces/ApplicationAudioPipelineFactoryInterface.h>
#include <AVSCommon/AVS/AVSDirective.h>
#include <AVSCommon/AVS/PlayBehavior.h>
#include <AVSCommon/AVS/CapabilityAgent.h>
#include <AVSCommon/AVS/CapabilityConfiguration.h>
#include <AVSCommon/SDKInterfaces/CapabilityConfigurationInterface.h>
#include <AVSCommon/AVS/DialogUXStateAggregator.h>
#include <AVSCommon/SDKInterfaces/SpeechSynthesizerObserverInterface.h>
#include <AVSCommon/SDKInterfaces/ExceptionEncounteredSenderInterface.h>
#include <AVSCommon/SDKInterfaces/ContextManagerInterface.h>
#include <AVSCommon/SDKInterfaces/DirectiveSequencerInterface.h>
#include <AVSCommon/SDKInterfaces/FocusManagerInterface.h>
#include <AVSCommon/SDKInterfaces/DialogUXStateObserverInterface.h>
#include <AVSCommon/SDKInterfaces/MessageSenderInterface.h>
#include <AVSCommon/SDKInterfaces/PowerResourceManagerInterface.h>
#include <AVSCommon/Utils/AudioAnalyzer/AudioAnalyzerState.h>
#include <AVSCommon/Utils/MediaPlayer/MediaPlayerInterface.h>
#include <AVSCommon/Utils/MediaPlayer/MediaPlayerObserverInterface.h>
#include <AVSCommon/Utils/Metrics/MetricEventBuilder.h>
#include <AVSCommon/Utils/Metrics/MetricRecorderInterface.h>
#include <AVSCommon/Utils/RequiresShutdown.h>
#include <AVSCommon/Utils/Threading/Executor.h>
#include <Captions/CaptionData.h>
#include <Captions/CaptionManagerInterface.h>

namespace alexaClientSDK {
namespace capabilityAgents {
namespace speechSynthesizer {

static const constexpr char* SPEAK_MEDIA_PLAYER_NAME = "SpeakMediaPlayer";

class SpeechSynthesizer
        : public avsCommon::avs::CapabilityAgent
        , public avsCommon::sdkInterfaces::DialogUXStateObserverInterface
        , public avsCommon::sdkInterfaces::CapabilityConfigurationInterface
        , public avsCommon::utils::mediaPlayer::MediaPlayerObserverInterface
        , public avsCommon::utils::RequiresShutdown
        , public std::enable_shared_from_this<SpeechSynthesizer> {
public:
    using SpeechSynthesizerObserverInterface = avsCommon::sdkInterfaces::SpeechSynthesizerObserverInterface;

    static std::shared_ptr<SpeechSynthesizer> createSpeechSynthesizer(
        std::shared_ptr<acsdkApplicationAudioPipelineFactoryInterfaces::ApplicationAudioPipelineFactoryInterface>
            audioPipelineFactory,
        std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
        std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
        std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
        std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionSender,
        std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder,
        std::shared_ptr<avsCommon::avs::DialogUXStateAggregator> dialogUXStateAggregator,
        std::shared_ptr<captions::CaptionManagerInterface> captionManager = nullptr,
        std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr);

    static std::shared_ptr<SpeechSynthesizer> create(
        std::shared_ptr<avsCommon::utils::mediaPlayer::MediaPlayerInterface> mediaPlayer,
        std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
        std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
        std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
        std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionSender,
        std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder,
        std::shared_ptr<avsCommon::avs::DialogUXStateAggregator> dialogUXStateAggregator,
        std::shared_ptr<captions::CaptionManagerInterface> captionManager = nullptr,
        std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr);

    void onDialogUXStateChanged(DialogUXState newState) override;

    avsCommon::avs::DirectiveHandlerConfiguration getConfiguration() const override;

    void addObserver(std::shared_ptr<SpeechSynthesizerObserverInterface> observer);

    void removeObserver(std::shared_ptr<SpeechSynthesizerObserverInterface> observer);

    void onDeregistered() override;

    void handleDirectiveImmediately(std::shared_ptr<avsCommon::avs::AVSDirective> directive) override;

    void preHandleDirective(std::shared_ptr<DirectiveInfo> info) override;

    void handleDirective(std::shared_ptr<DirectiveInfo> info) override;

    void cancelDirective(std::shared_ptr<DirectiveInfo> info) override;

    void onFocusChanged(avsCommon::avs::FocusState newFocus, avsCommon::avs::MixingBehavior behavior) override;

    void provideState(const avsCommon::avs::NamespaceAndName& stateProviderName, const unsigned int stateRequestToken)
        override;

    void onContextAvailable(const std::string& jsonContext) override;

    void onContextFailure(const avsCommon::sdkInterfaces::ContextRequestError error) override;

    void onFirstByteRead(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState& state) override;
    void onPlaybackStarted(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState& state) override;
    void onPlaybackFinished(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState& state) override;
    void onPlaybackError(
        SourceId id,
        const avsCommon::utils::mediaPlayer::ErrorType& type,
        std::string error,
        const avsCommon::utils::mediaPlayer::MediaPlayerState& state) override;
    void onPlaybackStopped(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState& state) override;
    void onBufferUnderrun(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState& state) override;

    std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> getCapabilityConfigurations() override;

private:
    struct SpeakDirectiveInfo {
        SpeakDirectiveInfo(std::shared_ptr<DirectiveInfo> directiveInfo);

        void clear();

        std::shared_ptr<avsCommon::avs::AVSDirective> directive;

        std::shared_ptr<avsCommon::sdkInterfaces::DirectiveHandlerResultInterface> result;

        std::string token;

        std::unique_ptr<avsCommon::avs::attachment::AttachmentReader> attachmentReader;

        bool sendPlaybackStartedMessage;

        bool sendPlaybackFinishedMessage;

        bool sendCompletedMessage;

        bool isSetFailedCalled;

        bool isPlaybackInitiated;

        bool isHandled;

        bool isCancelInitiated;

        avsCommon::avs::PlayBehavior playBehavior;

        captions::CaptionData captionData;

        std::vector<avsCommon::utils::audioAnalyzer::AudioAnalyzerState> analyzersData;
    };

    SpeechSynthesizer(
        std::shared_ptr<avsCommon::utils::mediaPlayer::MediaPlayerInterface> mediaPlayer,
        std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
        std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
        std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
        std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder,
        std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionSender,
        std::shared_ptr<captions::CaptionManagerInterface> captionManager = nullptr,
        std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr);

    void doShutdown() override;

    void init();

    void executeHandleImmediately(std::shared_ptr<DirectiveInfo> info);

    void executePreHandleAfterValidation(std::shared_ptr<SpeakDirectiveInfo> speakInfo);

    void executeHandleAfterValidation(std::shared_ptr<SpeakDirectiveInfo> speakInfo);

    void executePreHandle(std::shared_ptr<DirectiveInfo> info);

    void executeHandle(std::shared_ptr<DirectiveInfo> info);

    void executeCancel(std::shared_ptr<DirectiveInfo> info);

    void executeCancel(std::shared_ptr<SpeakDirectiveInfo> speakInfo, bool internalCancel);

    void executeStateChange(SpeechSynthesizerObserverInterface::SpeechSynthesizerState newState);

    void executeProvideStateLocked(const unsigned int& stateRequestToken);

    void executePlaybackStarted();

    void executePlaybackFinished();

    void executePlaybackStopped(SourceId id);

    void executePlaybackError(const avsCommon::utils::mediaPlayer::ErrorType& type, std::string error);

    void submitMetric(avsCommon::utils::metrics::MetricEventBuilder& metricEventBuilder);

    void submitInstanceEntryMetric(
        const std::string& segmentId,
        const std::string& name,
        const std::map<std::string, std::string>& metadata = {});

    void executeOnDialogUXStateChanged(
        avsCommon::sdkInterfaces::DialogUXStateObserverInterface::DialogUXState newState);

    std::string buildState(std::string& token, int64_t offsetInMilliseconds) const;

    void sendEvent(const std::string& eventName, const std::string& payload) const;

    static std::string buildPayload(std::string& token);

    static std::string buildPayload(std::string& token, int64_t offsetInMilliseconds);

    void startPlaying();

    void stopPlaying();

    void setCurrentStateLocked(SpeechSynthesizerObserverInterface::SpeechSynthesizerState newState);

    void setDesiredState(SpeechSynthesizerObserverInterface::SpeechSynthesizerState desiredState);

    void resetCurrentInfo();

    void setHandlingCompleted();

    void setHandlingFailed(const std::string& description);

    void sendExceptionEncounteredAndReportFailed(
        std::shared_ptr<SpeakDirectiveInfo> info,
        avsCommon::avs::ExceptionErrorType type,
        const std::string& message);

    void sendExceptionEncounteredAndReportMissingProperty(
        std::shared_ptr<SpeakDirectiveInfo> info,
        const std::string& missingProperty);

    void sendExceptionEncounteredAndReportUnexpectedPropertyType(
        std::shared_ptr<SpeakDirectiveInfo> info,
        const std::string& unexpectedProperty);

    void releaseForegroundFocus();

    std::shared_ptr<SpeakDirectiveInfo> validateInfo(
        const std::string& caller,
        std::shared_ptr<DirectiveInfo> info,
        bool checkResult = true);

    std::shared_ptr<SpeakDirectiveInfo> getSpeakDirectiveInfo(const std::string& messageId);

    bool setSpeakDirectiveInfo(
        const std::string& messageId,
        std::shared_ptr<SpeechSynthesizer::SpeakDirectiveInfo> speakDirectiveInfo);

    void addToDirectiveQueue(std::shared_ptr<SpeakDirectiveInfo> speakInfo);

    void removeSpeakDirectiveInfo(const std::string& messageId);

    void resetMediaSourceId();

    void clearPendingDirectivesLocked();

    void managePowerResource(SpeechSynthesizerObserverInterface::SpeechSynthesizerState newState);

    avsCommon::utils::mediaPlayer::MediaPlayerInterface::SourceId m_mediaSourceId;

    int64_t m_offsetInMilliseconds;

    std::shared_ptr<avsCommon::utils::mediaPlayer::MediaPlayerInterface> m_speechPlayer;

    std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> m_metricRecorder;

    std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> m_messageSender;

    std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> m_focusManager;

    std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> m_contextManager;

    std::shared_ptr<captions::CaptionManagerInterface> m_captionManager;

    std::unordered_set<std::shared_ptr<SpeechSynthesizerObserverInterface>> m_observers;

    SpeechSynthesizerObserverInterface::SpeechSynthesizerState m_currentState;

    SpeechSynthesizerObserverInterface::SpeechSynthesizerState m_desiredState;

    avsCommon::avs::FocusState m_currentFocus;

    std::shared_ptr<SpeakDirectiveInfo> m_currentInfo;

    std::mutex m_mutex;

    std::condition_variable m_waitOnStateChange;

    std::unordered_map<std::string, std::shared_ptr<SpeakDirectiveInfo>> m_speakDirectiveInfoMap;

    std::mutex m_speakDirectiveInfoMutex;

    std::deque<std::shared_ptr<SpeakDirectiveInfo>> m_speakInfoQueue;

    bool m_isShuttingDown;

    std::mutex m_speakInfoQueueMutex;

    bool m_initialDialogUXStateReceived;

    std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> m_capabilityConfigurations;

    std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface::PowerResourceId> m_powerResourceId;

    std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> m_powerResourceManager;

    avsCommon::utils::threading::Executor m_executor;
};

}  // namespace speechSynthesizer
}  // namespace capabilityAgents
}  // namespace alexaClientSDK

#endif  // ALEXA_CLIENT_SDK_CAPABILITYAGENTS_SPEECHSYNTHESIZER_INCLUDE_SPEECHSYNTHESIZER_SPEECHSYNTHESIZER_H_