Egor/Vision
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Well Done

Browse Source
This commit is contained in:
Egor
2026-03-20 19:26:33 +02:00
parent f123690cb4
commit ac183f8eb6
9 changed files with 231 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
ModuleWeather/Module.cs
View File

@@ -8,11 +8,8 @@ public class WeatherModule : IModule
public string[] GetCommands() => new[] { "Погода", "Пинг" };
public object Execute(string command, object[] args)
public void Settings(object[] args)
{
switch (command)
{
case "Погода":
// args[0] — это родительское окно из Ядра
var parentWindow = args != null && args.Length > 0 ? args[0] as Window : null;
@@ -28,6 +25,14 @@ public class WeatherModule : IModule
if (parentWindow != null) win.Show(parentWindow);
else win.Show();
}
public object Execute(string command)
{
switch (command)
{
case "Погода":
return "Окно открыто успешно";
case "Пинг":

2
ModuleWeather/ModuleWeather.csproj
View File

@@ -22,7 +22,7 @@
<Message Text="Копирую Module.dll в папку ядра..." Importance="high" />
<Copy SourceFiles="@(ModuleFiles)"
DestinationFolder="E:\Project\Visual\VisionAsist\VisionAsist\bin\Debug\net10.0\Modules\ModuleWeather"
DestinationFolder="/home/egor/RiderProjects/Vision/VisionAsist/bin/Debug/net10.0/Modules/ModuleWeather/"
OverwriteReadOnlyFiles="true" />
</Target>

3
VisionAsist.SDK/IModule.cs
View File

@@ -4,5 +4,6 @@ public interface IModule
{
string Name { get; }
string[] GetCommands();
object Execute(string command, object[] args);
void Settings(object[] args);
object Execute(string command);
}

4
VisionAsist.sln.DotSettings.user
View File

@@ -1,2 +1,4 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APorcupine_002Ecs_002Fl_003AC_0021_003FUsers_003Fmarin_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F395b25d8175b43798becf32fa73896954c00_003Fa6_003F28640b1d_003FPorcupine_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AInitHelpers_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F3bfb329b50ab4b719e770a577377990fdba600_003F92_003F7acfd65c_003FInitHelpers_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APorcupine_002Ecs_002Fl_003AC_0021_003FUsers_003Fmarin_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F395b25d8175b43798becf32fa73896954c00_003Fa6_003F28640b1d_003FPorcupine_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASoundIO_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F2ac0091ca0084789a96acdec8e3a9449d000_003F84_003Fd6e02c94_003FSoundIO_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

16
VisionAsist/Models/Core.cs
View File

@@ -12,7 +12,15 @@ namespace VisionAsist.Models;
public class Core
{
public static Dictionary<string, IModule> _loadedModules = new();
public class Modules
{
public string Name { get; set; }
public IModule Module { get; set; }
public string[] commands { get; set; }
}
public static List<Modules> modulelist = new();
public static TrigerCore triger = new();
public static string TextAsist;
static string Plugin = Path.Combine(AppContext.BaseDirectory, "Modules");
@@ -35,7 +43,7 @@ public class Core
if (type != null)
{
var module = (IModule)Activator.CreateInstance(type)!;
_loadedModules.Add(module.Name, module);
modulelist.Add(new Modules{Name = folderName, Module = module, commands = module.GetCommands()});
foreach (var cmd in module.GetCommands())
{
Console.WriteLine($"- {cmd}");
@@ -53,8 +61,10 @@ public class Core
triger.OnRecognized += word =>
{
Console.WriteLine(word); // печатаем сразу, как распознано
TextAsist = triger.RecognizedText;
Selector.selector(triger.RecognizedText);
};
// Запускаем запись

12
VisionAsist/Models/Selector.cs
View File

@@ -1,6 +1,14 @@
namespace VisionAsist.Models;
using System;
namespace VisionAsist.Models;
public class Selector
{
public static void selector(string text)
{
if (text.Contains("вижен"))
{
Console.WriteLine("dddddd");
}
}
}

211
VisionAsist/Models/TrigerCore.cs
View File

@@ -1,76 +1,207 @@
using System;
using System.IO;
using Vosk;
using System.Text.Json;
using NAudio.Wave;
using Avalonia.Controls;
using System.Threading;
using System.Collections.Concurrent;
using Vosk;
using SoundIOSharp;
using Avalonia.Threading;
namespace VisionAsist.Models;
public class TrigerCore
public class TrigerCore : IDisposable
{
private SoundIO? _soundIO;
private SoundIODevice? _device;
private SoundIOInStream? _inStream;
private Thread? _eventThread;
private Thread? _processingThread;
private bool _isRecording;
private readonly Model _model;
private readonly VoskRecognizer _rec;
private readonly BlockingCollection<byte[]> _audioQueue = new(100);
private WaveInEvent? _waveIn;
private Model? _model;
private VoskRecognizer? _rec;
private readonly object _voskLock = new();
public string RecognizedText { get; private set; } = "";
public event Action<string>? OnRecognized;
// Сохраняем ссылки в полях класса, чтобы GC не удалил их в Debug-режиме
private Action<int, int>? _readCallback;
private Action? _overflowCallback;
private Action? _errorCallback;
public TrigerCore()
{
string VoskPath = Path.Combine(AppContext.BaseDirectory, "models/Vosk/");
if (!Directory.Exists(VoskPath))
throw new DirectoryNotFoundException($"Модель не найдена по пути: {VoskPath}");
string voskPath = Path.Combine(AppContext.BaseDirectory, "models", "Vosk");
if (!Directory.Exists(voskPath))
throw new DirectoryNotFoundException($"Модель Vosk не найдена: {voskPath}");
_model = new Model(VoskPath);
_model = new Model(voskPath);
_rec = new VoskRecognizer(_model, 16000.0f);
}
public void StartRecording()
{
if (_waveIn != null || _rec == null) return;
if (_isRecording) return;
_waveIn = new WaveInEvent { WaveFormat = new WaveFormat(16000, 1) };
_waveIn.DataAvailable += OnDataAvailable;
_waveIn.StartRecording();
_soundIO = new SoundIO();
_soundIO.Connect();
_soundIO.FlushEvents();
int deviceIndex = _soundIO.DefaultInputDeviceIndex;
if (deviceIndex < 0) throw new Exception("Микрофон не найден.");
_device = _soundIO.GetInputDevice(deviceIndex);
_inStream = _device.CreateInStream();
_inStream.Format = SoundIOFormat.S16LE;
_inStream.SampleRate = 16000;
_inStream.Layout = SoundIOChannelLayout.GetDefault(1);
// Привязываем методы к полям
_readCallback = OnDataAvailable;
_overflowCallback = () => Console.WriteLine("Audio Buffer Overflow");
_errorCallback = () => Console.WriteLine("Audio Stream Error occurred");
_inStream.ReadCallback = _readCallback;
_inStream.OverflowCallback = _overflowCallback;
_inStream.ErrorCallback = _errorCallback;
_inStream.Open();
_inStream.Start();
_isRecording = true;
// Поток обработки событий SoundIO
_eventThread = new Thread(() => {
while (_isRecording && _soundIO != null)
{
try { _soundIO.WaitEvents(); } catch { break; }
}
}) { IsBackground = true, Name = "SoundIO_Wait" };
_eventThread.Start();
// Поток обработки Vosk
_processingThread = new Thread(ProcessQueue) {
IsBackground = true,
Name = "Vosk_Worker"
};
_processingThread.Start();
}
private unsafe void OnDataAvailable(int frameCountMin, int frameCountMax)
{
if (!_isRecording || _inStream == null) return;
int frameCount = frameCountMax;
// Работаем с результатом BeginRead как с объектом SoundIOChannelAreas
var areas = _inStream.BeginRead(ref frameCount);
if (frameCount <= 0) return;
try
{
var area = areas.GetArea(0);
if (area.Pointer == IntPtr.Zero) return;
int bytesNeeded = frameCount * 2; // 2 байта на семпл для S16LE
byte[] data = new byte[bytesNeeded];
fixed (byte* pDst = data)
{
short* pSrc = (short*)area.Pointer;
short* pDstShort = (short*)pDst;
int stepInShorts = area.Step / 2;
for (int i = 0; i < frameCount; i++)
{
pDstShort[i] = pSrc[i * stepInShorts];
}
}
// Отправляем в очередь и мгновенно освобождаем аудио-поток
_audioQueue.TryAdd(data);
}
catch (Exception ex)
{
Console.WriteLine("Error copying audio data: " + ex.Message);
}
finally
{
_inStream.EndRead();
}
}
private void ProcessQueue()
{
// GetConsumingEnumerable будет ждать появления данных в очереди
foreach (var data in _audioQueue.GetConsumingEnumerable())
{
if (!_isRecording) break;
try
{
bool isFinal;
lock (_rec)
{
isFinal = _rec.AcceptWaveform(data, data.Length);
}
if (isFinal)
{
ParseAndSend(_rec.Result());
}
}
catch (Exception ex)
{
Console.WriteLine("Vosk processing error: " + ex.Message);
}
}
}
private void ParseAndSend(string json)
{
if (string.IsNullOrWhiteSpace(json)) return;
try
{
using var doc = JsonDocument.Parse(json);
if (doc.RootElement.TryGetProperty("text", out var el))
{
string text = el.GetString() ?? "";
if (!string.IsNullOrWhiteSpace(text))
{
RecognizedText += text + " ";
// Безопасный проброс в UI поток Avalonia
Dispatcher.UIThread.Post(() => OnRecognized?.Invoke(text));
}
}
}
catch { /* JSON Parse Error */ }
}
public void StopRecording()
{
_waveIn?.StopRecording();
_waveIn?.Dispose();
_waveIn = null;
}
public event Action<string>? OnRecognized;
if (!_isRecording) return;
_isRecording = false;
private void OnDataAvailable(object? sender, WaveInEventArgs e)
{
lock (_voskLock)
{
if (_rec != null && _rec.AcceptWaveform(e.Buffer, e.BytesRecorded))
{
var json = _rec.Result();
using var doc = JsonDocument.Parse(json);
var result = doc.RootElement.GetProperty("text").GetString();
_audioQueue.CompleteAdding();
_soundIO?.Wakeup();
if (!string.IsNullOrWhiteSpace(result))
{
RecognizedText += result + " ";
OnRecognized?.Invoke(result); // уведомляем подписчиков
}
}
}
_inStream?.Dispose();
_device?.RemoveReference();
_soundIO?.Dispose();
_inStream = null;
_device = null;
_soundIO = null;
}
protected void Stop()
public void Dispose()
{
StopRecording();
_rec?.Dispose();
_model?.Dispose();
}
}

8
VisionAsist/ViewModels/SettingsViewModel.cs
View File

@@ -23,11 +23,11 @@ public class SettingsViewModel : ViewModelBase
public SettingsViewModel()
{
foreach (string Name in Core._loadedModules.Keys)
foreach (var module in Core.modulelist)
{
AddModule(Name);
AddModule(module.Name);
}
}
@@ -44,10 +44,10 @@ public class SettingsViewModel : ViewModelBase
private void OpenSettings(string moduleName)
{
var module = Core._loadedModules.Values.FirstOrDefault(m => m.Name == moduleName);
var module = Core.modulelist.FirstOrDefault(m => m.Name == moduleName);
if (module != null)
{
module.Execute("ShowWeather", new object[] { this });
module.Module.Settings(new object[] { this });
}

4
VisionAsist/VisionAsist.csproj
View File

@@ -5,6 +5,8 @@
<Nullable>enable</Nullable>
<ApplicationManifest>app.manifest</ApplicationManifest>
<AvaloniaUseCompiledBindingsByDefault>true</AvaloniaUseCompiledBindingsByDefault>
<RunCommand>LD_LIBRARY_PATH=/usr/lib $(RunCommand)</RunCommand>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
@@ -22,6 +24,7 @@
<PrivateAssets Condition="'$(Configuration)' != 'Debug'">All</PrivateAssets>
</PackageReference>
<PackageReference Include="CommunityToolkit.Mvvm" Version="8.4.0" />
<PackageReference Include="libsoundio-sharp-net" Version="1.0.0" />
<PackageReference Include="NAudio" Version="2.3.0" />
<PackageReference Include="Vosk" Version="0.3.38" />
</ItemGroup>
@@ -29,4 +32,5 @@
<ItemGroup>
<ProjectReference Include="..\VisionAsist.SDK\VisionAsist.SDK.csproj" />
</ItemGroup>
</Project>