AlexaClientSDK  1.25.0
A cross-platform, modular SDK for interacting with the Alexa Voice Service
SpeechSynthesizer.h
Go to the documentation of this file.
1 /*
2  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License").
5  * You may not use this file except in compliance with the License.
6  * A copy of the License is located at
7  *
8  * http://aws.amazon.com/apache2.0/
9  *
10  * or in the "license" file accompanying this file. This file is distributed
11  * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12  * express or implied. See the License for the specific language governing
13  * permissions and limitations under the License.
14  */
15 
16 #ifndef ALEXA_CLIENT_SDK_CAPABILITYAGENTS_SPEECHSYNTHESIZER_INCLUDE_SPEECHSYNTHESIZER_SPEECHSYNTHESIZER_H_
17 #define ALEXA_CLIENT_SDK_CAPABILITYAGENTS_SPEECHSYNTHESIZER_INCLUDE_SPEECHSYNTHESIZER_SPEECHSYNTHESIZER_H_
18 
19 #include <memory>
20 #include <mutex>
21 #include <string>
22 #include <unordered_set>
23 #include <deque>
24 
47 #include <Captions/CaptionData.h>
49 
50 namespace alexaClientSDK {
51 namespace capabilityAgents {
52 namespace speechSynthesizer {
53 
55 static const constexpr char* SPEAK_MEDIA_PLAYER_NAME = "SpeakMediaPlayer";
56 
67  , public std::enable_shared_from_this<SpeechSynthesizer> {
68 public:
71 
89  static std::shared_ptr<SpeechSynthesizer> createSpeechSynthesizer(
90  std::shared_ptr<acsdkApplicationAudioPipelineFactoryInterfaces::ApplicationAudioPipelineFactoryInterface>
91  audioPipelineFactory,
92  std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
93  std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
94  std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
95  std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionSender,
96  std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder,
97  std::shared_ptr<avsCommon::avs::DialogUXStateAggregator> dialogUXStateAggregator,
98  std::shared_ptr<captions::CaptionManagerInterface> captionManager = nullptr,
99  std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr);
100 
117  static std::shared_ptr<SpeechSynthesizer> create(
118  std::shared_ptr<avsCommon::utils::mediaPlayer::MediaPlayerInterface> mediaPlayer,
119  std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
120  std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
121  std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
122  std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionSender,
123  std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder,
124  std::shared_ptr<avsCommon::avs::DialogUXStateAggregator> dialogUXStateAggregator,
125  std::shared_ptr<captions::CaptionManagerInterface> captionManager = nullptr,
126  std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr);
127 
128  void onDialogUXStateChanged(DialogUXState newState) override;
129 
131 
137  void addObserver(std::shared_ptr<SpeechSynthesizerObserverInterface> observer);
138 
147  void removeObserver(std::shared_ptr<SpeechSynthesizerObserverInterface> observer);
148 
149  void onDeregistered() override;
150 
151  void handleDirectiveImmediately(std::shared_ptr<avsCommon::avs::AVSDirective> directive) override;
152 
153  void preHandleDirective(std::shared_ptr<DirectiveInfo> info) override;
154 
155  void handleDirective(std::shared_ptr<DirectiveInfo> info) override;
156 
157  void cancelDirective(std::shared_ptr<DirectiveInfo> info) override;
158 
163 
164  void provideState(const avsCommon::avs::NamespaceAndName& stateProviderName, const unsigned int stateRequestToken)
165  override;
166 
167  void onContextAvailable(const std::string& jsonContext) override;
168 
170 
176  void onPlaybackError(
177  SourceId id,
179  std::string error,
184 
187  std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> getCapabilityConfigurations() override;
189 
190 private:
194  struct SpeakDirectiveInfo {
200  SpeakDirectiveInfo(std::shared_ptr<DirectiveInfo> directiveInfo);
201 
203  void clear();
204 
206  std::shared_ptr<avsCommon::avs::AVSDirective> directive;
207 
209  std::shared_ptr<avsCommon::sdkInterfaces::DirectiveHandlerResultInterface> result;
210 
212  std::string token;
213 
215  std::unique_ptr<avsCommon::avs::attachment::AttachmentReader> attachmentReader;
216 
218  bool sendPlaybackStartedMessage;
219 
221  bool sendPlaybackFinishedMessage;
222 
224  bool sendCompletedMessage;
225 
227  bool isSetFailedCalled;
228 
230  bool isPlaybackInitiated;
231 
233  bool isHandled;
234 
236  bool isCancelInitiated;
237 
239  avsCommon::avs::PlayBehavior playBehavior;
240 
242  captions::CaptionData captionData;
243 
245  std::vector<avsCommon::utils::audioAnalyzer::AudioAnalyzerState> analyzersData;
246  };
247 
263  std::shared_ptr<avsCommon::utils::mediaPlayer::MediaPlayerInterface> mediaPlayer,
264  std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> messageSender,
265  std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> focusManager,
266  std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> contextManager,
267  std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> metricRecorder,
268  std::shared_ptr<avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface> exceptionSender,
269  std::shared_ptr<captions::CaptionManagerInterface> captionManager = nullptr,
270  std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> powerResourceManager = nullptr);
271 
272  void doShutdown() override;
273 
279  void init();
280 
287  void executeHandleImmediately(std::shared_ptr<DirectiveInfo> info);
288 
297  void executePreHandleAfterValidation(std::shared_ptr<SpeakDirectiveInfo> speakInfo);
298 
307  void executeHandleAfterValidation(std::shared_ptr<SpeakDirectiveInfo> speakInfo);
308 
315  void executePreHandle(std::shared_ptr<DirectiveInfo> info);
316 
323  void executeHandle(std::shared_ptr<DirectiveInfo> info);
324 
330  void executeCancel(std::shared_ptr<DirectiveInfo> info);
331 
337  void executeCancel(std::shared_ptr<SpeakDirectiveInfo> speakInfo, bool internalCancel);
338 
346  void executeStateChange(SpeechSynthesizerObserverInterface::SpeechSynthesizerState newState);
347 
354  void executeProvideStateLocked(const unsigned int& stateRequestToken);
355 
359  void executePlaybackStarted();
360 
364  void executePlaybackFinished();
365 
371  void executePlaybackStopped(SourceId id);
372 
378  void executePlaybackError(const avsCommon::utils::mediaPlayer::ErrorType& type, std::string error);
379 
384  void submitMetric(avsCommon::utils::metrics::MetricEventBuilder& metricEventBuilder);
385 
392  void executeOnDialogUXStateChanged(
394 
402  std::string buildState(std::string& token, int64_t offsetInMilliseconds) const;
403 
409  void sendEvent(const std::string& eventName, const std::string& payload) const;
410 
417  static std::string buildPayload(std::string& token);
418 
426  static std::string buildPayload(std::string& token, int64_t offsetInMilliseconds);
427 
431  void startPlaying();
432 
436  void stopPlaying();
437 
447  void setCurrentStateLocked(SpeechSynthesizerObserverInterface::SpeechSynthesizerState newState);
448 
455  void setDesiredState(SpeechSynthesizerObserverInterface::SpeechSynthesizerState desiredState);
456 
461  void resetCurrentInfo();
462 
466  void setHandlingCompleted();
467 
473  void setHandlingFailed(const std::string& description);
474 
482  void sendExceptionEncounteredAndReportFailed(
483  std::shared_ptr<SpeakDirectiveInfo> info,
485  const std::string& message);
486 
493  void sendExceptionEncounteredAndReportMissingProperty(
494  std::shared_ptr<SpeakDirectiveInfo> info,
495  const std::string& missingProperty);
496 
503  void sendExceptionEncounteredAndReportUnexpectedPropertyType(
504  std::shared_ptr<SpeakDirectiveInfo> info,
505  const std::string& unexpectedProperty);
506 
510  void releaseForegroundFocus();
511 
520  std::shared_ptr<SpeakDirectiveInfo> validateInfo(
521  const std::string& caller,
522  std::shared_ptr<DirectiveInfo> info,
523  bool checkResult = true);
524 
531  std::shared_ptr<SpeakDirectiveInfo> getSpeakDirectiveInfo(const std::string& messageId);
532 
540  bool setSpeakDirectiveInfo(
541  const std::string& messageId,
542  std::shared_ptr<SpeechSynthesizer::SpeakDirectiveInfo> speakDirectiveInfo);
543 
549  void addToDirectiveQueue(std::shared_ptr<SpeakDirectiveInfo> speakInfo);
550 
556  void removeSpeakDirectiveInfo(const std::string& messageId);
557 
562  void resetMediaSourceId();
563 
569  void clearPendingDirectivesLocked();
570 
575  void managePowerResource(SpeechSynthesizerObserverInterface::SpeechSynthesizerState newState);
576 
583 
585  int64_t m_offsetInMilliseconds;
586 
588  std::shared_ptr<avsCommon::utils::mediaPlayer::MediaPlayerInterface> m_speechPlayer;
589 
591  std::shared_ptr<avsCommon::utils::metrics::MetricRecorderInterface> m_metricRecorder;
592 
594  std::shared_ptr<avsCommon::sdkInterfaces::MessageSenderInterface> m_messageSender;
595 
597  std::shared_ptr<avsCommon::sdkInterfaces::FocusManagerInterface> m_focusManager;
598 
600  std::shared_ptr<avsCommon::sdkInterfaces::ContextManagerInterface> m_contextManager;
601 
603  std::shared_ptr<captions::CaptionManagerInterface> m_captionManager;
604 
606  std::unordered_set<std::shared_ptr<SpeechSynthesizerObserverInterface>> m_observers;
607 
613 
619 
621  avsCommon::avs::FocusState m_currentFocus;
622 
625  std::shared_ptr<SpeakDirectiveInfo> m_currentInfo;
626 
628  std::mutex m_mutex;
629 
631  std::condition_variable m_waitOnStateChange;
632 
634  std::unordered_map<std::string, std::shared_ptr<SpeakDirectiveInfo>> m_speakDirectiveInfoMap;
635 
640  std::mutex m_speakDirectiveInfoMutex;
641 
644  std::deque<std::shared_ptr<SpeakDirectiveInfo>> m_speakInfoQueue;
645 
648  bool m_isShuttingDown;
649 
651  std::mutex m_speakInfoQueueMutex;
652 
654  bool m_initialDialogUXStateReceived;
655 
657  std::unordered_set<std::shared_ptr<avsCommon::avs::CapabilityConfiguration>> m_capabilityConfigurations;
658 
660  std::shared_ptr<avsCommon::sdkInterfaces::PowerResourceManagerInterface> m_powerResourceManager;
661 
669 };
670 
671 } // namespace speechSynthesizer
672 } // namespace capabilityAgents
673 } // namespace alexaClientSDK
674 
675 #endif // ALEXA_CLIENT_SDK_CAPABILITYAGENTS_SPEECHSYNTHESIZER_INCLUDE_SPEECHSYNTHESIZER_SPEECHSYNTHESIZER_H_
Definition: CapabilityAgent.h:47
void onContextAvailable(const std::string &jsonContext) override
Definition: SpeechSynthesizer.cpp:377
MixingBehavior
Definition: MixingBehavior.h:25
Definition: NamespaceAndName.h:32
void onContextFailure(const avsCommon::sdkInterfaces::ContextRequestError error) override
Definition: SpeechSynthesizer.cpp:382
PlayBehavior
Used to determine how a client must handle queueing and playback of a stream.
Definition: PlayBehavior.h:29
void onPlaybackError(SourceId id, const avsCommon::utils::mediaPlayer::ErrorType &type, std::string error, const avsCommon::utils::mediaPlayer::MediaPlayerState &state) override
Definition: SpeechSynthesizer.cpp:442
static std::shared_ptr< SpeechSynthesizer > createSpeechSynthesizer(std::shared_ptr< acsdkApplicationAudioPipelineFactoryInterfaces::ApplicationAudioPipelineFactoryInterface > audioPipelineFactory, std::shared_ptr< avsCommon::sdkInterfaces::MessageSenderInterface > messageSender, std::shared_ptr< avsCommon::sdkInterfaces::FocusManagerInterface > focusManager, std::shared_ptr< avsCommon::sdkInterfaces::ContextManagerInterface > contextManager, std::shared_ptr< avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface > exceptionSender, std::shared_ptr< avsCommon::utils::metrics::MetricRecorderInterface > metricRecorder, std::shared_ptr< avsCommon::avs::DialogUXStateAggregator > dialogUXStateAggregator, std::shared_ptr< captions::CaptionManagerInterface > captionManager=nullptr, std::shared_ptr< avsCommon::sdkInterfaces::PowerResourceManagerInterface > powerResourceManager=nullptr)
Definition: SpeechSynthesizer.cpp:167
uint64_t SourceId
A type that identifies which source is currently being operated on. This must be unique across all in...
Definition: MediaPlayerInterface.h:69
void addObserver(std::shared_ptr< SpeechSynthesizerObserverInterface > observer)
Definition: SpeechSynthesizer.cpp:253
ContextRequestError
Definition: ContextRequesterInterface.h:40
void onDialogUXStateChanged(DialogUXState newState) override
Definition: SpeechSynthesizer.cpp:1376
ExceptionErrorType
Definition: ExceptionErrorType.h:28
void onBufferUnderrun(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState &state) override
Definition: SpeechSynthesizer.cpp:457
avsCommon::avs::DirectiveHandlerConfiguration getConfiguration() const override
Definition: SpeechSynthesizer.cpp:247
void onFirstByteRead(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState &state) override
Definition: SpeechSynthesizer.cpp:387
FocusState
Definition: FocusState.h:29
DialogUXState
The different dialog specific AVS UX states.
Definition: DialogUXStateObserverInterface.h:32
Definition: CaptionData.h:30
static const constexpr char * SPEAK_MEDIA_PLAYER_NAME
String to identify the speak media player to render audio.
Definition: SpeechSynthesizer.h:55
std::unordered_set< std::shared_ptr< avsCommon::avs::CapabilityConfiguration > > getCapabilityConfigurations() override
Definition: SpeechSynthesizer.cpp:1398
std::unordered_map< directiveRoutingRule::DirectiveRoutingRule, BlockingPolicy > DirectiveHandlerConfiguration
Definition: DirectiveHandlerConfiguration.h:32
void onFocusChanged(avsCommon::avs::FocusState newFocus, avsCommon::avs::MixingBehavior behavior) override
Definition: SpeechSynthesizer.cpp:306
MediaPlayerInterface::SourceId SourceId
A type that identifies which source is currently being operated on.
Definition: MediaPlayerObserverInterface.h:49
void provideState(const avsCommon::avs::NamespaceAndName &stateProviderName, const unsigned int stateRequestToken) override
Definition: SpeechSynthesizer.cpp:367
Whether or not curl logs should be emitted.
Definition: AVSConnectionManager.h:36
void onPlaybackStopped(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState &state) override
Definition: SpeechSynthesizer.cpp:451
void cancelDirective(std::shared_ptr< DirectiveInfo > info) override
Definition: SpeechSynthesizer.cpp:297
void onPlaybackFinished(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState &state) override
Definition: SpeechSynthesizer.cpp:418
void preHandleDirective(std::shared_ptr< DirectiveInfo > info) override
Definition: SpeechSynthesizer.cpp:274
void removeObserver(std::shared_ptr< SpeechSynthesizerObserverInterface > observer)
Definition: SpeechSynthesizer.cpp:258
void onDeregistered() override
Definition: SpeechSynthesizer.cpp:263
ErrorType
Identifies the specific type of error in a PlaybackFailed event.
Definition: ErrorTypes.h:27
Definition: SpeechSynthesizerObserverInterface.h:32
SpeechSynthesizerState
Definition: SpeechSynthesizerObserverInterface.h:37
void onPlaybackStarted(SourceId id, const avsCommon::utils::mediaPlayer::MediaPlayerState &state) override
Definition: SpeechSynthesizer.cpp:394
void handleDirective(std::shared_ptr< DirectiveInfo > info) override
Definition: SpeechSynthesizer.cpp:284
void handleDirectiveImmediately(std::shared_ptr< avsCommon::avs::AVSDirective > directive) override
Definition: SpeechSynthesizer.cpp:268
A dialog specific UX state observer.
Definition: DialogUXStateObserverInterface.h:29
static std::shared_ptr< SpeechSynthesizer > create(std::shared_ptr< avsCommon::utils::mediaPlayer::MediaPlayerInterface > mediaPlayer, std::shared_ptr< avsCommon::sdkInterfaces::MessageSenderInterface > messageSender, std::shared_ptr< avsCommon::sdkInterfaces::FocusManagerInterface > focusManager, std::shared_ptr< avsCommon::sdkInterfaces::ContextManagerInterface > contextManager, std::shared_ptr< avsCommon::sdkInterfaces::ExceptionEncounteredSenderInterface > exceptionSender, std::shared_ptr< avsCommon::utils::metrics::MetricRecorderInterface > metricRecorder, std::shared_ptr< avsCommon::avs::DialogUXStateAggregator > dialogUXStateAggregator, std::shared_ptr< captions::CaptionManagerInterface > captionManager=nullptr, std::shared_ptr< avsCommon::sdkInterfaces::PowerResourceManagerInterface > powerResourceManager=nullptr)
Definition: SpeechSynthesizer.cpp:201

AlexaClientSDK 1.25.0 - Copyright 2016-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0