Skip to content

Commit 7e9f67c

Browse files
tluybentluyben
committed
added the browser stt api
1 parent 1d0c334 commit 7e9f67c

File tree

5 files changed

+173
-9
lines changed

5 files changed

+173
-9
lines changed

‎app/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"dependencies": {
55
"@auth0/auth0-spa-js": "^2.0.4",
66
"@emotion/css": "^11.10.6",
7+
"@emotion/react": "^11.10.6",
78
"@emotion/styled": "^11.10.6",
89
"@mantine/core": "^5.10.5",
910
"@mantine/hooks": "^5.10.5",

‎app/src/components/input.tsx

+36-8
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
import styled from '@emotion/styled';
22
import { Button, ActionIcon, Textarea, Loader } from '@mantine/core';
33
import { useMediaQuery } from '@mantine/hooks';
4-
import { useCallback, useMemo } from 'react';
4+
import { useCallback, useMemo, useState } from 'react';
55
import { FormattedMessage, useIntl } from 'react-intl';
66
import { useLocation } from 'react-router-dom';
77
import { useAppContext } from '../context';
88
import { useAppDispatch, useAppSelector } from '../store';
99
import { selectMessage, setMessage } from '../store/message';
1010
import { selectTemperature } from '../store/parameters';
1111
import { openSystemPromptPanel, openTemperaturePanel } from '../store/settings-ui';
12+
import { speechRecognition } from '../speech-recognition-types.d'
1213

1314
const Container = styled.div`
1415
background: #292933;
@@ -37,9 +38,9 @@ export interface MessageInputProps {
3738
export default function MessageInput(props: MessageInputProps) {
3839
const temperature = useAppSelector(selectTemperature);
3940
const message = useAppSelector(selectMessage);
40-
41+
const [recording, setRecording] = useState(false);
4142
const hasVerticalSpace = useMediaQuery('(min-height: 1000px)');
42-
43+
4344
const context = useAppContext();
4445
const dispatch = useAppDispatch();
4546
const intl = useIntl();
@@ -58,6 +59,26 @@ export default function MessageInput(props: MessageInputProps) {
5859
}
5960
}, [context, message, dispatch]);
6061

62+
const onSpeechStart = () => {
63+
console.log("onSpeechStart", recording)
64+
if (!recording) {
65+
setRecording(true);
66+
speechRecognition.continuous = true;
67+
speechRecognition.interimResults = true;
68+
69+
speechRecognition.onresult = (event) => {
70+
const transcript = event.results[event.results.length - 1][0].transcript;
71+
dispatch(setMessage(transcript));
72+
};
73+
74+
speechRecognition.start();
75+
} else {
76+
setRecording(false);
77+
speechRecognition.stop();
78+
}
79+
}
80+
81+
6182
const onKeyDown = useCallback((e: React.KeyboardEvent<HTMLTextAreaElement>) => {
6283
if (e.key === 'Enter' && e.shiftKey === false && !props.disabled) {
6384
e.preventDefault();
@@ -66,6 +87,7 @@ export default function MessageInput(props: MessageInputProps) {
6687
}, [onSubmit, props.disabled]);
6788

6889
const rightSection = useMemo(() => {
90+
6991
return (
7092
<div style={{
7193
opacity: '0.8',
@@ -84,14 +106,20 @@ export default function MessageInput(props: MessageInputProps) {
84106
<Loader size="xs" style={{ padding: '0 0.8rem 0 0.5rem' }} />
85107
</>)}
86108
{!context.generating && (
87-
<ActionIcon size="xl"
88-
onClick={onSubmit}>
89-
<i className="fa fa-paper-plane" style={{ fontSize: '90%' }} />
90-
</ActionIcon>
109+
<>
110+
<ActionIcon size="xl"
111+
onClick={onSubmit}>
112+
<i className="fa fa-paper-plane" style={{ fontSize: '90%' }} />
113+
</ActionIcon>
114+
<ActionIcon size="xl"
115+
onClick={onSpeechStart}>
116+
<i className="fa fa-microphone" style={{ fontSize: '90%', color: recording ? 'red' : 'inherit' }} />
117+
</ActionIcon>
118+
</>
91119
)}
92120
</div>
93121
);
94-
}, [onSubmit, props.disabled, context.generating]);
122+
}, [recording, onSubmit, props.disabled, context.generating]);
95123

96124
const disabled = context.generating;
97125

‎app/src/index.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ async function bootstrapApplication() {
7272

7373
root.render(
7474
<React.StrictMode>
75-
<IntlProvider locale={navigator.language} messages={messages}>
75+
<IntlProvider locale={navigator.language} defaultLocale="en-GB" messages={messages}>
7676
<MantineProvider theme={{ colorScheme: "dark" }}>
7777
<Provider store={store}>
7878
<PersistGate loading={null} persistor={persistor}>

‎app/src/speech-recognition-types.d.ts

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
declare global {
2+
interface Window {
3+
SpeechRecognition: SpeechRecognition
4+
}
5+
interface SpeechGrammar {
6+
src: string
7+
weight: number
8+
}
9+
10+
const SpeechGrammar: {
11+
prototype: SpeechGrammar
12+
new(): SpeechGrammar
13+
}
14+
15+
interface SpeechGrammarList {
16+
readonly length: number
17+
addFromString(string: string, weight?: number): void
18+
addFromURI(src: string, weight?: number): void
19+
item(index: number): SpeechGrammar
20+
[index: number]: SpeechGrammar
21+
}
22+
23+
const SpeechGrammarList: {
24+
prototype: SpeechGrammarList
25+
new(): SpeechGrammarList
26+
}
27+
28+
interface SpeechRecognitionEventMap {
29+
audioend: Event
30+
audiostart: Event
31+
end: Event
32+
error: SpeechRecognitionError
33+
nomatch: SpeechRecognitionEvent
34+
result: SpeechRecognitionEvent
35+
soundend: Event
36+
soundstart: Event
37+
speechend: Event
38+
speechstart: Event
39+
start: Event
40+
}
41+
42+
interface SpeechRecognition {
43+
continuous: boolean
44+
grammars: SpeechGrammarList
45+
interimResults: boolean
46+
lang: string
47+
maxAlternatives: number
48+
onaudioend: ((this: SpeechRecognition, ev: Event) => any) | null
49+
onaudiostart: ((this: SpeechRecognition, ev: Event) => any) | null
50+
onend: ((this: SpeechRecognition, ev: Event) => any) | null
51+
onerror:
52+
| ((this: SpeechRecognition, ev: SpeechRecognitionError) => any)
53+
| null
54+
onnomatch:
55+
| ((this: SpeechRecognition, ev: SpeechRecognitionEvent) => any)
56+
| null
57+
onresult:
58+
| ((this: SpeechRecognition, ev: SpeechRecognitionEvent) => any)
59+
| null
60+
onsoundend: ((this: SpeechRecognition, ev: Event) => any) | null
61+
onsoundstart: ((this: SpeechRecognition, ev: Event) => any) | null
62+
onspeechend: ((this: SpeechRecognition, ev: Event) => any) | null
63+
onspeechstart: ((this: SpeechRecognition, ev: Event) => any) | null
64+
onstart: ((this: SpeechRecognition, ev: Event) => any) | null
65+
serviceURI: string
66+
abort(): void
67+
start(): void
68+
stop(): void
69+
addEventListener<K extends keyof SpeechRecognitionEventMap>(
70+
type: K,
71+
listener: (
72+
this: SpeechRecognition,
73+
ev: SpeechRecognitionEventMap[K]
74+
) => any,
75+
options?: boolean | AddEventListenerOptions
76+
): void
77+
addEventListener(
78+
type: string,
79+
listener: EventListenerOrEventListenerObject,
80+
options?: boolean | AddEventListenerOptions
81+
): void
82+
removeEventListener<K extends keyof SpeechRecognitionEventMap>(
83+
type: K,
84+
listener: (
85+
this: SpeechRecognition,
86+
ev: SpeechRecognitionEventMap[K]
87+
) => any,
88+
options?: boolean | EventListenerOptions
89+
): void
90+
removeEventListener(
91+
type: string,
92+
listener: EventListenerOrEventListenerObject,
93+
options?: boolean | EventListenerOptions
94+
): void
95+
}
96+
97+
const SpeechRecognition: {
98+
prototype: SpeechRecognition
99+
new(): SpeechRecognition
100+
}
101+
102+
interface SpeechRecognitionError extends Event {
103+
// readonly error: SpeechRecognitionErrorCode;
104+
readonly message: string
105+
}
106+
107+
const SpeechRecognitionError: {
108+
prototype: SpeechRecognitionError
109+
new(): SpeechRecognitionError
110+
}
111+
112+
interface SpeechRecognitionEvent extends Event {
113+
readonly emma: Document | null
114+
readonly interpretation: any
115+
readonly resultIndex: number
116+
readonly results: SpeechRecognitionResultList
117+
}
118+
119+
const SpeechRecognitionEvent: {
120+
prototype: SpeechRecognitionEvent
121+
new(): SpeechRecognitionEvent
122+
}
123+
}
124+
125+
let speechRecognition: SpeechRecognition
126+
127+
if (window.SpeechRecognition) {
128+
speechRecognition = new SpeechRecognition()
129+
} else {
130+
speechRecognition = new webkitSpeechRecognition()
131+
}
132+
133+
export { speechRecognition }

‎app/src/store/index.ts

+2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ const persistMessageConfig = {
2525
storage,
2626
}
2727

28+
29+
2830
const store = configureStore({
2931
reducer: {
3032
// auth: authReducer,

0 commit comments

Comments
 (0)