Skip to content

Commit

Permalink
feat: Traversing UI (#631)
Browse files Browse the repository at this point in the history
  • Loading branch information
@m13v
m13v authored Nov 6, 2024
1 parent b55a00b commit 72bd255
Show file tree
Hide file tree
Showing 17 changed files with 1,082 additions and 368 deletions.
106 changes: 69 additions & 37 deletions screenpipe-app-tauri/components/recording-settings.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ import { initOpenTelemetry } from "@/lib/opentelemetry";
import { Language } from "@/lib/language";
import { Command as ShellCommand } from "@tauri-apps/plugin-shell";
import { CliCommandDialog } from "./cli-command-dialog";
import { ToastAction } from "@/components/ui/toast";

interface AudioDevice {
name: string;
Expand Down Expand Up @@ -559,6 +560,42 @@ export function RecordingSettings({
await invoke("show_timeline");
};

const handleUiMonitoringToggle = async (checked: boolean) => {
try {
if (checked) {
// Check accessibility permissions first
const hasPermission = await invoke('check_accessibility_permissions');
if (!hasPermission) {
toast({
title: "accessibility permission required",
description: "please grant accessibility permission in system preferences",
action: (
<ToastAction altText="open preferences" onClick={() => invoke('open_accessibility_preferences')}>
open preferences
</ToastAction>
),
variant: "destructive",
});
return;
}
}

// Just update the local setting - the update button will handle the restart
setLocalSettings({
...localSettings,
enableUiMonitoring: checked
});

} catch (error) {
console.error("failed to toggle ui monitoring:", error);
toast({
title: "error checking accessibility permissions",
description: "please try again or check the logs",
variant: "destructive",
});
}
};

return (
<>
<div className="relative">
Expand Down Expand Up @@ -1389,43 +1426,38 @@ export function RecordingSettings({
)}
</div>
</div>
<div className="flex items-center space-x-2">
<Switch
id="ui-monitoring-toggle"
checked={localSettings.enableUiMonitoring}
onCheckedChange={(checked) =>
setLocalSettings({
...localSettings,
enableUiMonitoring: checked,
})
}
/>
<Label
htmlFor="ui-monitoring-toggle"
className="flex items-center space-x-2"
>
<span>enable UI monitoring</span>
<Badge variant="outline" className="ml-2">macOS only</Badge>
<TooltipProvider>
<Tooltip>
<TooltipTrigger>
<HelpCircle className="h-4 w-4 cursor-default" />
</TooltipTrigger>
<TooltipContent side="right">
<p>
enables monitoring of UI elements and their interactions.
<br />
this allows for better context in search results
<br />
and more accurate activity tracking.
<br />
(macOS only)
</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</Label>
</div>
{isMacOS && (
<div className="flex items-center space-x-2">
<Switch
id="ui-monitoring-toggle"
checked={localSettings.enableUiMonitoring}
onCheckedChange={handleUiMonitoringToggle}
/>
<Label
htmlFor="ui-monitoring-toggle"
className="flex items-center space-x-2"
>
<span>enable UI monitoring</span>
<Badge variant="outline" className="ml-2">accessibility permissions</Badge>
<TooltipProvider>
<Tooltip>
<TooltipTrigger>
<HelpCircle className="h-4 w-4 cursor-default" />
</TooltipTrigger>
<TooltipContent side="right">
<p>
enables monitoring of UI elements and their interactions.
<br />
this allows for better context in search results
<br />
* requires accessibility permission
</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</Label>
</div>
)}
</CardContent>
</Card>
</div>
Expand Down
68 changes: 44 additions & 24 deletions screenpipe-app-tauri/components/screenpipe-status.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,20 +110,24 @@ const HealthStatus = ({ className }: { className?: string }) => {
status: string,
frameStatus: string,
audioStatus: string,
audioDisabled: boolean
uiStatus: string,
audioDisabled: boolean,
uiMonitoringEnabled: boolean
) => {
if (status === "loading") return "bg-yellow-500";
const isVisionOk = frameStatus === "ok" || frameStatus === "disabled";
const isAudioOk =
audioStatus === "ok" || audioStatus === "disabled" || audioDisabled;
return isVisionOk && isAudioOk ? "bg-green-500" : "bg-red-500";
const isAudioOk = audioStatus === "ok" || audioStatus === "disabled" || audioDisabled;
const isUiOk = uiStatus === "ok" || uiStatus === "disabled" || !uiMonitoringEnabled;
return isVisionOk && isAudioOk && isUiOk ? "bg-green-500" : "bg-red-500";
};

const getStatusMessage = (
status: string,
frameStatus: string,
audioStatus: string,
audioDisabled: boolean
uiStatus: string,
audioDisabled: boolean,
uiMonitoringEnabled: boolean
) => {
if (status === "loading")
return "screenpipe is starting up. this may take a few minutes...";
Expand All @@ -133,6 +137,8 @@ const HealthStatus = ({ className }: { className?: string }) => {
issues.push("screen recording");
if (!audioDisabled && audioStatus !== "ok" && audioStatus !== "disabled")
issues.push("audio recording");
if (uiMonitoringEnabled && uiStatus !== "ok" && uiStatus !== "disabled")
issues.push("ui monitoring");

if (issues.length === 0) return "screenpipe is running smoothly";
return `there might be an issue with ${issues.join(" and ")}`;
Expand Down Expand Up @@ -364,13 +370,17 @@ const HealthStatus = ({ className }: { className?: string }) => {
health.status,
health.frame_status,
health.audio_status,
settings.disableAudio
health.ui_status,
settings.disableAudio,
settings.enableUiMonitoring
);
const statusMessage = getStatusMessage(
health.status,
health.frame_status,
health.audio_status,
settings.disableAudio
health.ui_status,
settings.disableAudio,
settings.enableUiMonitoring
);

return (
Expand Down Expand Up @@ -414,23 +424,33 @@ const HealthStatus = ({ className }: { className?: string }) => {
</div>
</DialogHeader>
<div className="flex-grow overflow-auto">
<p className="text-sm mb-2 font-semibold">{statusMessage}</p>
<div className="text-xs mb-4">
<p>screen recording: {health.frame_status}</p>
<p>
audio recording:{" "}
{settings.disableAudio ? "turned off" : health.audio_status}
</p>
<p>
last screen capture:{" "}
{formatTimestamp(health.last_frame_timestamp)}
</p>
<p>
last audio capture:{" "}
{settings.disableAudio
? "n/a"
: formatTimestamp(health.last_audio_timestamp)}
</p>
<p className="text-sm mb-4 font-semibold">{statusMessage}</p>
<div className="space-y-2 text-xs mb-4">
{/* Screen Recording Status */}
<div className="flex items-center">
<div className={`w-2 h-2 rounded-full ${health.frame_status === "ok" ? "bg-green-500" : "bg-red-500"} mr-2`} />
<span>screen recording</span>
<span className="text-muted-foreground ml-2">status: {health.frame_status},</span>
<span className="text-muted-foreground ml-2">last update: {formatTimestamp(health.last_frame_timestamp)}</span>
</div>

{/* Audio Recording Status */}
<div className="flex items-center">
<div className={`w-2 h-2 rounded-full ${settings.disableAudio ? "bg-gray-400" : health.audio_status === "ok" ? "bg-green-500" : "bg-red-500"} mr-2`} />
<span>audio recording</span>
<span className="text-muted-foreground ml-2">status: {settings.disableAudio ? "turned off" : health.audio_status},</span>
<span className="text-muted-foreground ml-2">last update: {settings.disableAudio ? "n/a" : formatTimestamp(health.last_audio_timestamp)}</span>
</div>

{/* UI Monitoring Status - Only show if enabled */}
{settings.enableUiMonitoring && (
<div className="flex items-center">
<div className={`w-2 h-2 rounded-full ${health.ui_status === "ok" ? "bg-green-500" : "bg-red-500"} mr-2`} />
<span>ui monitoring</span>
<span className="text-muted-foreground ml-2">status: {health.ui_status},</span>
<span className="text-muted-foreground ml-2">last update: {formatTimestamp(health.last_ui_timestamp)}</span>
</div>
)}
</div>

<Separator className="my-12" />
Expand Down
Loading

0 comments on commit 72bd255

Please sign in to comment.