const Applet = imports.ui.applet; const Gio = imports.gi.Gio; const GLib = imports.gi.GLib; const Lang = imports.lang; const Mainloop = imports.mainloop; const Soup = imports.gi.Soup; const St = imports.gi.St; const Settings = imports.ui.settings; const Json = imports.gi.Json; function MyApplet(metadata, orientation, panel_height, instance_id) { this._init(metadata, orientation, panel_height, instance_id); } MyApplet.prototype = { __proto__: Applet.IconApplet.prototype, _init: function(metadata, orientation, panel_height, instance_id) { Applet.IconApplet.prototype._init.call(this, orientation, panel_height, instance_id); this.metadata = metadata; this.settings = new Settings.AppletSettings(this, metadata.uuid, instance_id); this.settings.bindProperty( Settings.BindingDirection.IN, "baseUrl", "baseUrl", this.onSettingsChanged, null ); this.settings.bindProperty( Settings.BindingDirection.IN, "username", "username", this.onSettingsChanged, null ); this.settings.bindProperty( Settings.BindingDirection.IN, "password", "password", this.onSettingsChanged, null ); this._setCustomIcon(metadata.path + "/icon.svg"); this.set_applet_tooltip(_("Voice Assistant")); this._streamSocket = null; this._nodeSocket = null; this._isRecording = false; this._recorder = null; this._player = null; this._audioBuffer = new Uint8Array(0); this._playbackBuffer = []; this._accessToken = null; global.log("[Voice Assistant] Applet initialized"); this._authenticate(); }, _setCustomIcon: function(iconPath) { try { let file = Gio.File.new_for_path(iconPath); let gicon = new Gio.FileIcon({ file: file }); this.set_applet_icon_symbolic_name(iconPath); this._applet_icon.gicon = gicon; } catch (e) { global.logError("[Voice Assistant] Error setting custom icon: " + e.message); // Fallback to the default icon if there's an error this.set_applet_icon_name("microphone-sensitivity-medium"); } }, onSettingsChanged: function() { global.log("[Voice Assistant] Settings changed, re-authenticating"); this._closeExistingSockets(); this._authenticate(); }, _authenticate: function() { let session = new Soup.Session(); let message = Soup.Message.new( 'POST', `https://${this.baseUrl}/auth/login` ); let body = JSON.stringify({ username: this.username, password: this.password }); let bytes = GLib.Bytes.new(body); message.set_request_body_from_bytes('application/json', bytes); session.send_and_read_async(message, GLib.PRIORITY_DEFAULT, null, (session, result) => { try { let bytes = session.send_and_read_finish(result); if (message.get_status() === 200) { let data = JSON.parse(new TextDecoder().decode(bytes.get_data())); this._accessToken = data.access_token; global.log("[Voice Assistant] Authentication successful"); this._initSockets(); } else { global.logError("[Voice Assistant] Authentication failed: " + message.get_status()); } } catch (e) { global.logError("[Voice Assistant] Error during authentication: " + e.message); } }); }, _initSockets: function() { if (!this._accessToken) { global.logError("[Voice Assistant] No access token available. Cannot initialize sockets."); return; } global.log("[Voice Assistant] Initializing WebSockets"); let maxPayloadSize = 10 * 1024 * 1024; // 10 MB in bytes const STREAM_SOCKET_URL = `wss://${this.baseUrl}/node/v1/stream?token=${this._accessToken}`; const NODE_SOCKET_URL = `wss://${this.baseUrl}/node/v1?token=${this._accessToken}`; // Initialize Node WebSocket try { let session = new Soup.Session(); let message = new Soup.Message({ method: 'GET', uri: GLib.Uri.parse(NODE_SOCKET_URL, GLib.UriFlags.NONE) }); session.websocket_connect_async(message, null, null, null, null, (session, result) => { try { this._nodeSocket = session.websocket_connect_finish(result); this._nodeSocket.set_max_incoming_payload_size(maxPayloadSize); this._nodeSocket.connect('message', Lang.bind(this, this._onNodeMessage)); this._nodeSocket.connect('error', Lang.bind(this, this._onSocketError)); global.log("[Voice Assistant] Node WebSocket initialized"); } catch (e) { global.logError("[Voice Assistant] Error finalizing Node WebSocket: " + e.message); } }); // Initialize streaming WebSocket try { let streamSession = new Soup.Session(); let streamMessage = new Soup.Message({ method: 'GET', uri: GLib.Uri.parse(STREAM_SOCKET_URL, GLib.UriFlags.NONE) }); streamSession.websocket_connect_async(streamMessage, null, null, null, null, (streamSession, streamResult) => { try { this._streamSocket = streamSession.websocket_connect_finish(streamResult); this._streamSocket.connect('message', Lang.bind(this, this._onStreamMessage)); this._streamSocket.connect('error', Lang.bind(this, this._onSocketError)); global.log("[Voice Assistant] Stream WebSocket initialized"); } catch (e) { global.logError("[Voice Assistant] Error finalizing stream WebSocket: " + e.message); } }); } catch (e) { global.logError("[Voice Assistant] Error initializing stream WebSocket: " + e.message); } } catch (e) { global.logError("[Voice Assistant] Error initializing Node WebSocket: " + e.message); } }, _closeExistingSockets: function() { if (this._streamSocket) { this._streamSocket.close(Soup.WebsocketCloseCode.NORMAL, null); this._streamSocket = null; } if (this._nodeSocket) { this._nodeSocket.close(Soup.WebsocketCloseCode.NORMAL, null); this._nodeSocket = null; } }, _onSocketError: function(socket, error) { global.logError("[Voice Assistant] WebSocket error: " + error.message); this._closeExistingSockets(); this._authenticate(); }, _onNodeMessage: function(connection, type, message) { try { if (type === Soup.WebsocketDataType.TEXT) { let data = message.get_data(); let jsonData = JSON.parse(data); global.log("[Voice Assistant] Parsed node message: " + jsonData.type); // Handle text messages if needed } else { global.log("[Voice Assistant] Received unknown data type from node: " + type); } } catch (e) { global.logError("[Voice Assistant] Error handling node message: " + e.message); } }, _onStreamMessage: function(connection, type, message) { try { if (type === Soup.WebsocketDataType.BINARY) { global.log("[Voice Assistant] Received binary audio data of length: " + message.get_data().length); this._playbackBuffer.push(message.get_data()); this._playAudio(); } else { global.log("[Voice Assistant] Received unknown data type from stream: " + type); } } catch (e) { global.logError("[Voice Assistant] Error handling stream message: " + e.message); } }, _playAudio: function() { if (this._player) { // If a player is already running, just add the new data to the buffer return; } if (this._playbackBuffer.length === 0) { return; } let audioData = this._playbackBuffer.shift(); // Create a temporary file to store the audio data let [file, stream] = Gio.File.new_tmp("voice-assistant-XXXXXX"); stream.output_stream.write_all(audioData, null); stream.close(null); // Play the audio using GStreamer this._player = new Gio.Subprocess({ argv: [ 'gst-launch-1.0', 'filesrc', 'location=' + file.get_path(), '!', 'decodebin', '!', 'audioconvert', '!', 'audioresample', '!', 'autoaudiosink' ], flags: Gio.SubprocessFlags.NONE }); this._player.init(null); // Clean up when playback is finished this._player.wait_async(null, (source, result) => { try { source.wait_finish(result); } catch (e) { global.logError("[Voice Assistant] Error during audio playback: " + e.message); } finally { this._player = null; file.delete(null); // Play the next audio chunk if available this._playAudio(); } }); }, _startRecording: function() { if (this._isRecording) return; this._isRecording = true; this._setCustomIcon(this.metadata.path + "/icon-active.svg"); global.log("[Voice Assistant] Starting recording"); try { // Initialize GStreamer pipeline for recording this._recorder = new Gio.Subprocess({ argv: [ 'gst-launch-1.0', 'pulsesrc', '!', 'audioconvert', '!', 'audio/x-raw,format=S16LE,channels=1,rate=16000', '!', 'fdsink' ], flags: Gio.SubprocessFlags.STDOUT_PIPE }); this._recorder.init(null); global.log("[Voice Assistant] Recording subprocess initialized"); // Read audio data and send it over WebSocket let stdout = this._recorder.get_stdout_pipe(); this._readAudioData(stdout); } catch (e) { global.logError("[Voice Assistant] Error starting recording: " + e.message); } }, _readAudioData: function(stdout) { stdout.read_bytes_async(4096, GLib.PRIORITY_DEFAULT, null, (source, result) => { try { let bytes = source.read_bytes_finish(result); if (bytes && bytes.get_size() > 0) { // Append new data to the existing buffer let newData = new Uint8Array(bytes.get_data()); let combinedBuffer = new Uint8Array(this._audioBuffer.length + newData.length); combinedBuffer.set(this._audioBuffer); combinedBuffer.set(newData, this._audioBuffer.length); this._audioBuffer = combinedBuffer; // If we have accumulated 4096 or more bytes, send them while (this._audioBuffer.length >= 4096) { let chunkToSend = this._audioBuffer.slice(0, 4096); this._streamSocket.send_binary(chunkToSend); // global.log("[Voice Assistant] Sent 4096 bytes of audio data"); // Keep the remaining data in the buffer this._audioBuffer = this._audioBuffer.slice(4096); } // Continue reading this._readAudioData(stdout); } else { global.log("[Voice Assistant] End of audio stream reached"); // // Send any remaining data in the buffer // if (this._audioBuffer.length > 0) { // this._streamSocket.send_binary(this._audioBuffer); // global.log("[Voice Assistant] Sent final " + this._audioBuffer.length + " bytes of audio data"); // } this._stopRecording(); } } catch (e) { global.logError("[Voice Assistant] Error reading audio data: " + e.message); this._stopRecording(); } }); }, _stopRecording: function() { if (!this._isRecording) return; this._isRecording = false; this._setCustomIcon(this.metadata.path + "/icon.svg"); global.log("[Voice Assistant] Stopping recording"); if (this._recorder) { this._recorder.force_exit(); this._recorder = null; global.log("[Voice Assistant] Recording subprocess terminated"); } // Clear the audio buffer this._audioBuffer = new Uint8Array(0); }, on_applet_clicked: function() { if (this._isRecording) { global.log("[Voice Assistant] Applet clicked: stopping recording"); this._stopRecording(); } else { global.log("[Voice Assistant] Applet clicked: starting recording"); this._startRecording(); } }, on_applet_removed_from_panel: function() { global.log("[Voice Assistant] Applet removed from panel"); this._stopRecording(); this._closeExistingSockets(); if (this._player) { this._player.force_exit(); global.log("[Voice Assistant] Audio player terminated"); } } }; function main(metadata, orientation, panel_height, instance_id) { global.log("[Voice Assistant] Main function called"); return new MyApplet(metadata, orientation, panel_height, instance_id); }