Unity 讯飞之讯飞星火大模型的简单封装和使用（补充讯飞大模型识图功能）-编程知识

Unity 讯飞之讯飞星火大模型的简单封装和使用（补充讯飞大模型识图功能）

一、简单介绍

二、实现原理

三、注意事项

四、效果预览

五、案例简单实现步骤

六、关键代码

七、案例下载地址

补充、讯飞大模型-图像识别

一、简单介绍

Unity 工具类，自己整理的一些游戏开发可能用到的模块，单独独立使用，方便游戏开发。

这里简单的介绍讯飞大模型的封装和使用，进行聊天。

“讯飞星火认知大模型”是科大讯飞发布的产品，具有7大核心能力，即文本生成、语言理解、知识问答、逻辑推理、数学能力、代码能力、多模态能力。

星火v1.5、v2.0地址： http://spark-api.xf-yun.com/v1/completions

星火v3.0地址： http://spark-api.xf-yun.com/v3/completions

星火v1.5和v2.0使用的是同一个URL，domain参数配置如下：
# domain = "general"       # v1.5版本
#domain = "generalv2"    # v2.0版本
domain = "generalv3"    # v3.0版本
星火认知大模型 WebAPI 接口调用示例接口文档（必看）：星火认知大模型Web API文档 | 讯飞开放平台文档中心
错误码链接：星火认知大模型服务说明 | 讯飞开放平台文档中心

二、实现原理

1、申请星火大模型的 APP_ID 等相关信息

2、通过使用的大模型版本，以及当前的时间，结合申请星火大模型的 APP_ID 等相关信息，生成需要的 URL

3、通过对应的 json 数据格式，websocket 进行建立连接请求

4、这里是流式返回，对应解析数据格式，得到返回的信息

5、返回的关键信息结构，有些类似 gpt 的数据格式，用过的话，使用起来会很快

三、注意事项

1、注意 code 返回码，不同的返回码可以进行不同处理，避免产生意想不到的问题

2、注意 sid 的区分，如果上一次返回没有结束，关闭连接后，重新发起新的访问，可能会同时接收到上一次的未结束的数据流，和当次的数据流；如果不想接收到，注意通过 sid 进行区分；

3、注意在 LLMConfig 配置你的 APP_ID 等相关信息

四、效果预览

五、案例简单实现步骤

1、首先到讯飞官网获取对应的星火大模型的 APP_ID 等相关自己星火大模型的信息

讯飞星火认知大模型-AI大语言模型-星火大模型-科大讯飞

2、打开Unity ，简单搭建一个场景

3、可以参考星火大模型的相关demo，完成对应的讯飞大模型接口封装

这里包括 IIFlyLLMHandler（接口类，可以根据自己需要重新定义接口），BaseIFlyLLMHandler（基类），IFlyLLMHandler（基本的讯飞接口大模型类），AutoReConnectIFlyLLMHandler（待自动重连的讯飞接口大模型类）

4、完成对应接口的封装之后，添加一个简单的测试类 TestIFlyLLMHandler

5、把 TestIFlyLLMHandler 添加到场景中，比对应赋值

6、运行结果如上

六、关键代码

1、TestIFlyLLMHandler

using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;public class TestIFlyLLMHandler : MonoBehaviour
{public Text TagrgetText;public InputField TagrgetInputField;public Button TagrgetButton;List<string> m_StrLst = new List<string>() { "你是谁","最近天气"};IIFlyLLMHandler m_Handler;// Start is called before the first frame updatevoid Start(){m_Handler = new IFlyLLMHandler();m_Handler.Initialized();TagrgetText.text = "";m_Handler.OnMessageReceivedAction = (jsonResponse) => { TagrgetText.text += jsonResponse.payload.choices.text[0].content; };TagrgetButton.onClick.AddListener(() => {TagrgetText.text = "";m_Handler.SendMsg(TagrgetInputField.text);});}// Update is called once per framevoid Update(){if (Input.anyKeyDown) {//m_Handler.SendMsg(m_StrLst[Random.Range(0,m_StrLst.Count)]);}}
}

2、IFlyLLMHandler

using BestHTTP.WebSocket;
using System;
using UnityEngine;/// <summary>
/// 使用讯飞大模型
/// </summary>
public class IFlyLLMHandler : BaseIFlyLLMHandler
{#region Data/// <summary>/// 发送连接服务器失败事件/// </summary>int m_TimerServerAccessFailure = -1;const float INTERVAL_TIME_SEND_SERVER_ACCESS_FAILURE = 4.0f;#endregion#region interface function/// <summary>/// 初始化/// </summary>public override void Initialized(){base.Initialized();}/// <summary>/// 发送数据/// </summary>/// <param name="askContent"></param>public override void SendMsg(string askContent){try{CacheHistory(askContent);Connect();}catch (Exception e){Debug.LogError($"e.ToString() {e.ToString()}\ne.Message {e.Message}");}}#endregion#region Websocket回调/// <summary>/// 建立连接事件/// </summary>/// <param name="ws"></param>protected override void OnWebSocketOpen(WebSocket ws){Debug.Log("WebSocket Connected!");Send();}/// <summary>/// 连接错误信息/// </summary>/// <param name="ws"></param>/// <param name="ex"></param>protected override void OnWebSocketError(WebSocket ws, Exception ex){base.OnWebSocketError(ws, ex);SendServerAccessFailure(1005);}/// <summary>/// 连接关闭事件/// </summary>/// <param name="ws"></param>/// <param name="code"></param>/// <param name="message"></param>protected override void OnWebSocketClosed(WebSocket ws, ushort code, string message){base.OnWebSocketClosed(ws, code, message);SendServerAccessFailure(code);}#endregion#region private function/// <summary>/// 发送服务器连接失败错误/// </summary>/// <param name="code"></param>private void SendServerAccessFailure(UInt16 code){if (code == 1005) // 连接失败错误码{if (m_TimerServerAccessFailure != -1) return;m_TimerServerAccessFailure = Timer.Instance.Post2Scale((index) => {Debug.Log("[BubbleChatUseIFlyLLMHandler] SendServerAccessFailure");m_TimerServerAccessFailure = -1;}, INTERVAL_TIME_SEND_SERVER_ACCESS_FAILURE);}}#endregion
}

3、BaseIFlyLLMHandler

using Newtonsoft.Json;
using System;
using UnityEngine;
using System.Text;
using BestHTTP.WebSocket;
using System.Collections.Generic;/*** 星火认知大模型 WebAPI 接口调用示例 接口文档（必看）：https://www.xfyun.cn/doc/spark/Web.html* 错误码链接：https://www.xfyun.cn/doc/spark/%E6%8E%A5%E5%8F%A3%E8%AF%B4%E6%98%8E.html （code返回错误码时必看）* @author iflytek*//// <summary>
/// 讯飞大模型基类
/// </summary>
public class BaseIFlyLLMHandler : IIFlyLLMHandler
{#region Data/// <summary>/// WebSocket/// </summary>protected WebSocket m_WebSocket;/// <summary>/// 缓存的历史数据的队列/// </summary>protected List<IIFlyLLMHandler.Content> m_CacheHistory = new List<IIFlyLLMHandler.Content>();/// <summary>/// 是否读完/// </summary>protected bool m_ReadEnd = true;/// <summary>/// 是否需要重发/// </summary>protected bool m_NeedSend = false;/// <summary>/// 大模型的配置/// </summary>protected LLMConfig m_Config;/// <summary>/// 记录上一个老的 sid ，因为打断后，如果上一次的没有传输完，下一次连接依然会接收到该 sid 的数据流/// </summary>protected string m_OldSid = "";/// <summary>/// // 用来记录中间的sid ,因为可能被打断(不一定每次都能完全正常流程接收完数据)/// </summary>protected string m_RcdSid = ""; /// <summary>/// 接收到数据的事件/// </summary>public Action<IIFlyLLMHandler.JsonResponse> OnMessageReceivedAction { get; set; }#endregion#region interface function/// <summary>/// 初始化/// </summary>public virtual void Initialized(){m_Config = new LLMConfig();}/// <summary>/// 连接/// </summary>public virtual void Connect(){if (m_WebSocket != null){Close();}string authUrl = GetAuthUrl();string url = authUrl.Replace("http://", "ws://").Replace("https://", "wss://");m_WebSocket = new WebSocket(new Uri(url));m_WebSocket.OnOpen += OnWebSocketOpen;m_WebSocket.OnMessage += OnWebSocketMessage;m_WebSocket.OnError += OnWebSocketError;m_WebSocket.OnClosed += OnWebSocketClosed;m_WebSocket.Open();}/// <summary>/// 关闭/// </summary>public virtual void Close(){m_OldSid = m_RcdSid;m_WebSocket?.Close();m_WebSocket = null;}/// <summary>/// 重置/// </summary>public virtual void Reset(){m_CacheHistory.Clear();}/// <summary>/// 发送消息/// </summary>/// <param name="history"></param>public virtual void SendMsg(string askContent){try{CacheHistory(askContent);if (m_WebSocket != null && m_WebSocket.IsOpen){if (m_ReadEnd){Send();}else{//中断websocket，然后再发送Close();m_NeedSend = true;}}else{m_NeedSend = true;Connect();}}catch (Exception e){Debug.LogError(e.Message);}}#endregion#region Websocket回调/// <summary>/// 建立连接的事件/// </summary>/// <param name="ws"></param>protected virtual void OnWebSocketOpen(WebSocket ws){Debug.Log("WebSocket Connected!");if (m_NeedSend) Send();}/// <summary>/// 接收到数据的事件/// </summary>/// <param name="ws"></param>/// <param name="message"></param>protected virtual void OnWebSocketMessage(WebSocket ws, string message){// 处理接收到的消息// 这里你可以根据需要进行相应的处理Debug.Log($"Received message: {message}");IIFlyLLMHandler.JsonResponse jsonResponse = JsonConvert.DeserializeObject<IIFlyLLMHandler.JsonResponse>(message);if (jsonResponse.header.code == 10013) // 错误码 10013：输入内容审核不通过，涉嫌违规，请重新调整输入内容{Debug.LogError($"OnWebSocketMessage message {jsonResponse.header.message} ");}else{Debug.Log($"OnWebSocketMessage m_OldSid {m_OldSid}, jsonResponse.header.sid {jsonResponse.header.sid} jsonResponse.header.code {jsonResponse.header.code}");// 判断不是上一次的 sid 的旧的数据流在做出反应if (jsonResponse.header.sid.Equals(m_OldSid) == false){if (jsonResponse.header.code == 10014) // 错误码 10014：输出内容涉及敏感信息，审核不通过，后续结果无法展示给用户{jsonResponse=GetHandleJsonResponse(jsonResponse);}OnMessageReceivedAction?.Invoke(jsonResponse);m_RcdSid = jsonResponse.header.sid;}}if (jsonResponse.payload.choices.status == 0){Debug.Log("Get First IFlyLLM Response");}if (jsonResponse.payload.choices.status == 2){m_ReadEnd = true;Debug.Log("Get Last IFlyLLM Response");m_OldSid = jsonResponse.header.sid;}}/// <summary>/// 连接发生错误的事件/// </summary>/// <param name="ws"></param>/// <param name="ex"></param>protected virtual void OnWebSocketError(WebSocket ws, Exception ex){Debug.LogError($"WebSocket Error: {ex.Message}");}/// <summary>/// 连接关闭的事件/// </summary>/// <param name="ws"></param>/// <param name="code"></param>/// <param name="message"></param>protected virtual void OnWebSocketClosed(WebSocket ws, UInt16 code, string message){Debug.LogFormat("WebSocket Closed!: code={0}, msg={1}", code, message);}#endregion#region protected function/// <summary>/// 真正发送数据/// </summary>protected virtual void Send(){m_NeedSend = false;m_ReadEnd = false;IIFlyLLMHandler.JsonRequest request = CreateRequest();string jsonString = JsonConvert.SerializeObject(request);Debug.LogError(jsonString);m_WebSocket?.Send(jsonString);}/// <summary>/// 创建一个连接请求/// </summary>/// <returns></returns>protected virtual IIFlyLLMHandler.JsonRequest CreateRequest(){return new IIFlyLLMHandler.JsonRequest{header = new IIFlyLLMHandler.Header{app_id = m_Config.IFLY_APPID,uid = Guid.NewGuid().ToString().Substring(0, 10)},parameter = new IIFlyLLMHandler.Parameter{chat = new IIFlyLLMHandler.Chat{domain = "generalv3",temperature = 0.01,max_tokens = 480}},payload = new IIFlyLLMHandler.Payload{message = new IIFlyLLMHandler.Message{text = m_CacheHistory}}};}/// <summary>/// 缓存历史记录数据结构，优化性能/// (缓存需要自己额外补充，这里先做简单的提问)/// </summary>protected virtual void CacheHistory(string askContent){m_CacheHistory.Clear();IIFlyLLMHandler.Content content = new IIFlyLLMHandler.Content();content.role = "user";content.content = askContent;m_CacheHistory.Add(content);}/// <summary>/// 敏感输出内容后的处理/// 重新组织数据(涉及敏感内容不予展示)/// </summary>/// <param name="jsonResponse"></param>/// <returns></returns>protected virtual IIFlyLLMHandler.JsonResponse GetHandleJsonResponse(IIFlyLLMHandler.JsonResponse jsonResponse) {int status = 2;IIFlyLLMHandler.HeaderResponse header = new IIFlyLLMHandler.HeaderResponse() { code = 0, message = "Success", sid = jsonResponse.header.sid, status = status };IIFlyLLMHandler.Text text = new IIFlyLLMHandler.Text() {content="***(涉及敏感内容不予展示)。",role= "assistant",index=0 };List<IIFlyLLMHandler.Text> textlst = new List<IIFlyLLMHandler.Text>();textlst.Add(text);IIFlyLLMHandler.Choices choices = new IIFlyLLMHandler.Choices() { status= status, seq=999,text=textlst};IIFlyLLMHandler.PayloadResponse payload = new IIFlyLLMHandler.PayloadResponse() { choices=choices};IIFlyLLMHandler.JsonResponse reorganizingInformationJRspn = new IIFlyLLMHandler.JsonResponse() {header=header,payload=payload };return reorganizingInformationJRspn;}#endregion#region 字符串操作/// <summary>/// 生成得到授权的 URL /// </summary>/// <returns></returns>private string GetAuthUrl(){string date = DateTime.UtcNow.ToString("r");Uri uri = new Uri(m_Config.IFLY_HOST_URL);StringBuilder builder = new StringBuilder("host: ").Append(uri.Host).Append("\n").//Append("date: ").Append(date).Append("\n").//Append("GET ").Append(uri.LocalPath).Append(" HTTP/1.1");string sha = HMACsha256(m_Config.IFLY_API_SECRET, builder.ToString());string authorization = string.Format("api_key=\"{0}\", algorithm=\"{1}\", headers=\"{2}\", signature=\"{3}\"", m_Config.IFLY_API_KEY, "hmac-sha256", "host date request-line", sha);string NewUrl = "https://" + uri.Host + uri.LocalPath;string path1 = "authorization" + "=" + Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(authorization));date = date.Replace(" ", "%20").Replace(":", "%3A").Replace(",", "%2C");string path2 = "date" + "=" + date;string path3 = "host" + "=" + uri.Host;NewUrl = NewUrl + "?" + path1 + "&" + path2 + "&" + path3;return NewUrl;}/// <summary>/// HMACsha256 /// </summary>/// <param name="apiSecretIsKey"></param>/// <param name="buider"></param>/// <returns></returns>private string HMACsha256(string apiSecretIsKey, string buider){byte[] bytes = System.Text.Encoding.UTF8.GetBytes(apiSecretIsKey);System.Security.Cryptography.HMACSHA256 hMACSHA256 = new System.Security.Cryptography.HMACSHA256(bytes);byte[] date = System.Text.Encoding.UTF8.GetBytes(buider);date = hMACSHA256.ComputeHash(date);hMACSHA256.Clear();return Convert.ToBase64String(date);}#endregion}

4、IIFlyLLMHandler

using System;
using System.Collections.Generic;/// <summary>
/// 讯飞大模型接口类
/// </summary>
public interface IIFlyLLMHandler
{/// <summary>/// 初始化/// </summary>void Initialized();/// <summary>/// 连接/// </summary>void Connect();/// <summary>/// 关闭连接/// </summary>void Close();/// <summary>/// 重置/// </summary>void Reset();/// <summary>/// 发送信息/// </summary>/// <param name="askContent"></param>void SendMsg(string askContent);/// <summary>/// 接收到数据事件/// </summary>Action<JsonResponse> OnMessageReceivedAction { get; set; }#region Request数据类定义public class JsonRequest{public Header header { get; set; }public Parameter parameter { get; set; }public Payload payload { get; set; }}public class Header{public string app_id { get; set; }public string uid { get; set; }}public class Parameter{public Chat chat { get; set; }}public class Chat{public string domain { get; set; }public double temperature { get; set; }public int max_tokens { get; set; }}public class Payload{public Message message { get; set; }}public class Message{public List<Content> text { get; set; }}public class Content{public string role { get; set; }public string content { get; set; }}#endregion#region Response数据类型定义public class JsonResponse{public HeaderResponse header { get; set; }public PayloadResponse payload { get; set; }}public class HeaderResponse{public int code { get; set; }public string message { get; set; }public string sid { get; set; }public int status { get; set; }}public class PayloadResponse{public Choices choices { get; set; }}public class Choices{public int status { get; set; }public int seq { get; set; }public List<Text> text { get; set; }}public class Text{public string content { get; set; }public string role { get; set; }public int index { get; set; }}#endregion
}

5、LLMConfig

public class LLMConfig
{#region 讯飞/// <summary>/// 应用APPID（必须为webapi类型应用，并开通星火认知大模型授权）/// </summary>public readonly string IFLY_APPID = "YOUR_IFLY_APPID";/// <summary>/// 接口密钥（webapi类型应用开通星火认知大模型后，控制台--我的应用---星火认知大模型---相应服务的apikey）/// </summary>public readonly string IFLY_API_SECRET = "YOUR_IFLY_API_SECRET";/// <summary>/// 接口密钥（webapi类型应用开通星火认知大模型后，控制台--我的应用---星火认知大模型---相应服务的apisecret）/// </summary>public readonly string IFLY_API_KEY = "YOUR_IFLY_API_KEY";/// <summary>/// 讯飞大模型访问地址/// </summary>public readonly string IFLY_HOST_URL = "https://spark-api.xf-yun.com/v3.1/chat";/// <summary>/// 讯飞图像识别/// </summary>public readonly string IFLY_IMAGE_RECOGNIZE_URL = "wss://spark-api.cn-huabei-1.xf-yun.com/v2.1/image";#endregion
}

6、AutoReConnectIFlyLLMHandler

using BestHTTP.WebSocket;
using System;
using UnityEngine;/// <summary>
/// 带自动重连的讯飞大模型访问
/// </summary>
public class AutoReConnectIFlyLLMHandler : BaseIFlyLLMHandler
{#region Data/// <summary>/// m_ReconnectTimerId ：自动重连的 TimerId/// </summary>int m_ReconnectTimerId = -1;/// <summary>/// m_ReconnectDelay ：  自动重连的 时间/// </summary>float m_ReconnectDelay = 0.5f;#endregion#region interface function/// <summary>/// 初始化/// </summary>public override void Initialized(){base.Initialized();Connect();}/// <summary>/// 关闭连接/// </summary>public override void Close(){CancelReconnectTimer();base.Close();}/// <summary>/// 自动重连/// </summary>internal void AutoReconnect(){Connect();}/// <summary>/// 发送请求/// </summary>/// <param name="askContent"></param>public override void SendMsg(string askContent){try{CacheHistory(askContent);if (m_WebSocket != null && m_WebSocket.IsOpen){if (m_ReadEnd){Send();}else{//中断websocket，然后再发送Close();m_NeedSend = true;}}else{m_NeedSend = true;}}catch (Exception e){Debug.LogError(e.Message);}}#endregion#region Websocket回调/// <summary>/// 关闭连接事件/// </summary>/// <param name="ws"></param>/// <param name="code"></param>/// <param name="message"></param>protected override void OnWebSocketClosed(WebSocket ws, ushort code, string message){base.OnWebSocketClosed(ws, code, message);RunReconnectTimer();}/// <summary>/// 连接发生错误的事件/// </summary>/// <param name="ws"></param>/// <param name="ex"></param>protected override void OnWebSocketError(WebSocket ws, Exception ex){base.OnWebSocketError(ws, ex);RunReconnectTimer();}#endregion#region 计时器/// <summary>/// 计时自动重新连接/// </summary>private void RunReconnectTimer(){Debug.Log($"Scheduling reconnect in {m_ReconnectDelay} seconds...");CancelReconnectTimer();m_ReconnectTimerId = Timer.Instance.Post2Really((v) => { AutoReconnect(); }, m_ReconnectDelay);}/// <summary>/// 取消重新连接计时/// </summary>private void CancelReconnectTimer(){if (m_ReconnectTimerId != -1){Timer.Instance.Cancel(m_ReconnectTimerId);m_ReconnectTimerId = -1;}}#endregion
}

七、案例下载地址

https://download.csdn.net/download/u014361280/88565465

补充、讯飞大模型-图像识别

1、IFlyLLMImageHandler

using UnityEngine;
using System;
using System.Text;
using Newtonsoft.Json;
using BestHTTP.WebSocket;
using System.Collections.Generic;public class IFlyLLMImageHandler 
{private WebSocket m_WebSocket;private int m_ReconnectTimerId = -1;private float m_ReconnectDelay = 0.5f;private List<Content> m_CacheHistory = new List<Content>();public Action<JsonResponse> OnMessageReceivedAction;private bool m_ReadEnd = true;private bool m_NeedSend = false;private LLMConfig m_Config;public void Initialized(){m_Config = new LLMConfig();Connect();}/// <summary>/// 连接/// </summary>public void Connect(){if (m_WebSocket != null){Close();}string authUrl = GetAuthUrl();string url = authUrl.Replace("http://", "ws://").Replace("https://", "wss://");m_WebSocket = new WebSocket(new Uri(url));m_WebSocket.OnOpen += OnWebSocketOpen;m_WebSocket.OnMessage += OnWebSocketMessage;m_WebSocket.OnError += OnWebSocketError;m_WebSocket.OnClosed += OnWebSocketClosed;m_WebSocket.Open();}/// <summary>/// 关闭/// </summary>public void Close(){CancelReconnectTimer();m_WebSocket?.Close();m_WebSocket = null;}public void Reset(){m_CacheHistory.Clear();}/// <summary>/// 自动重连/// </summary>private void AutoReconnect(){Connect();}/// <summary>/// 发送消息/// </summary>/// <param name="history"></param>public void SendMsg(string base64,string ask){try{ImageDataAsk(base64, ask);if (m_WebSocket != null && m_WebSocket.IsOpen){if (m_ReadEnd){Send();}else{//中断websocket，然后再发送Close();m_NeedSend = true;}}else{m_NeedSend = true;}}catch (Exception e){Debug.LogError(e.Message);}}private void Send(){m_NeedSend = false;m_ReadEnd = false;JsonRequest request = CreateRequest();string jsonString = JsonConvert.SerializeObject(request);Debug.LogError(jsonString);m_WebSocket?.Send(jsonString);}#region Websocket回调private void OnWebSocketOpen(WebSocket ws){Debug.Log("WebSocket Connected!");if (m_NeedSend) Send();}private void OnWebSocketMessage(WebSocket ws, string message){// 处理接收到的消息// 这里你可以根据需要进行相应的处理Debug.Log($"Received message: {message}");JsonResponse jsonResponse = JsonConvert.DeserializeObject<IFlyLLMImageHandler.JsonResponse>(message);OnMessageReceivedAction?.Invoke(jsonResponse);if (jsonResponse.payload.choices.status == 0){Debug.Log("Get First IFlyLLM Response");}if (jsonResponse.payload.choices.status == 2){m_ReadEnd = true;Debug.Log("Get Last IFlyLLM Response");}}private void OnWebSocketError(WebSocket ws, Exception ex){Debug.LogError($"WebSocket Error: {ex.Message}");RunReconnectTimer();}private void OnWebSocketClosed(WebSocket ws, UInt16 code, string message){Debug.LogFormat("WebSocket Closed!: code={0}, msg={1}", code, message);RunReconnectTimer();}#endregionprivate JsonRequest CreateRequest(){return new JsonRequest{header = new Header{app_id = m_Config.IFLY_APPID,uid = Guid.NewGuid().ToString().Substring(0, 10)},parameter = new Parameter{chat = new Chat{domain = "general",temperature = 0.5,top_k= 4,max_tokens = 2048,auditing = "default"}},payload = new Payload{message = new Message{text = m_CacheHistory}}};}private void ImageDataAsk(string base64,string ask) {m_CacheHistory.Clear();Content content = new Content();content.role = "user";content.content = base64;content.content_type = "image";m_CacheHistory.Add(content);content = new Content();content.role = "user";content.content = ask;content.content_type = "text";m_CacheHistory.Add(content);}#region 计时器private void RunReconnectTimer(){Debug.Log($"Scheduling reconnect in {m_ReconnectDelay} seconds...");CancelReconnectTimer();m_ReconnectTimerId = Timer.Instance.Post2Really((v) => { AutoReconnect(); }, m_ReconnectDelay);}private void CancelReconnectTimer(){if (m_ReconnectTimerId != -1){Timer.Instance.Cancel(m_ReconnectTimerId);m_ReconnectTimerId = -1;}}#endregion#region 字符串操作private string GetAuthUrl(){string date = DateTime.UtcNow.ToString("r");Uri uri = new Uri(m_Config.IFLY_IMAGE_RECOGNIZE_URL);StringBuilder builder = new StringBuilder("host: ").Append(uri.Host).Append("\n").//Append("date: ").Append(date).Append("\n").//Append("GET ").Append(uri.LocalPath).Append(" HTTP/1.1");string sha = HMACsha256(m_Config.IFLY_API_SECRET, builder.ToString());string authorization = string.Format("api_key=\"{0}\", algorithm=\"{1}\", headers=\"{2}\", signature=\"{3}\"", m_Config.IFLY_API_KEY, "hmac-sha256", "host date request-line", sha);string NewUrl = "https://" + uri.Host + uri.LocalPath;string path1 = "authorization" + "=" + Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(authorization));date = date.Replace(" ", "%20").Replace(":", "%3A").Replace(",", "%2C");string path2 = "date" + "=" + date;string path3 = "host" + "=" + uri.Host;NewUrl = NewUrl + "?" + path1 + "&" + path2 + "&" + path3;return NewUrl;}private string HMACsha256(string apiSecretIsKey, string buider){byte[] bytes = System.Text.Encoding.UTF8.GetBytes(apiSecretIsKey);System.Security.Cryptography.HMACSHA256 hMACSHA256 = new System.Security.Cryptography.HMACSHA256(bytes);byte[] date = System.Text.Encoding.UTF8.GetBytes(buider);date = hMACSHA256.ComputeHash(date);hMACSHA256.Clear();return Convert.ToBase64String(date);}#endregion#region Request数据类定义public class JsonRequest{public Header header { get; set; }public Parameter parameter { get; set; }public Payload payload { get; set; }}public class Header{public string app_id { get; set; }public string uid { get; set; }}public class Parameter{public Chat chat { get; set; }}public class Chat{public string domain { get; set; }public double temperature { get; set; }public int top_k { get; set; }public int max_tokens { get; set; }public string auditing { get; set; }}public class Payload{public Message message { get; set; }}public class Message{public List<Content> text { get; set; }}public class Content{public string role { get; set; }public string content { get; set; }public string content_type { get; set; }}#endregion#region Response数据类型定义public class JsonResponse{public HeaderResponse header { get; set; }public PayloadResponse payload { get; set; }}public class HeaderResponse{public int code { get; set; }public string message { get; set; }public string sid { get; set; }public int status { get; set; }}public class PayloadResponse{public Choices choices { get; set; }public Usage usage { get; set; }}public class Choices{public int status { get; set; }public int seq { get; set; }public List<Text> text { get; set; }}public class Text{public string content { get; set; }public string role { get; set; }public int index { get; set; }}public class Usage{public UsageText text { get; set; }}public class UsageText { public int completion_tokens { get; set; }public int question_tokens { get; set; }public int prompt_tokens { get; set; }public int total_tokens { get; set; }}#endregion
}

2、TestIFlyLLMImageHandler


using UnityEngine;public class TestIFlyLLMImageHandler : Singleton<TestIFlyLLMImageHandler>
{public void Test() {IFlyLLMImageHandler imageHandler = new IFlyLLMImageHandler();imageHandler.Initialized();imageHandler.OnMessageReceivedAction += (str) => {Debug.Log(" TestIFlyLLMImageHandler OnMessageReceivedAction " + str.payload.choices.text[0].content);};CameraCaptureImageForBase64.Instance.OnGetBase64ImageSuccess = (base64) =>{imageHandler.SendMsg(base64, "这是什么，尽可能详细的回答");};CameraCaptureImageForBase64.Instance.OnGetBase64ImageFailed = async (str) =>{string base64 = await ImageLoader.Instance.LoadImageAndConvertToBase64();imageHandler.SendMsg(base64, "这是什么，尽可能详细的回答");};CameraCaptureImageForBase64.Instance.CaptureImage();}
}

3、CameraCaptureImageForBase64


using System;
using System.Collections;
using UnityEngine;
using UnityEngine.Android;/// <summary>
/// 打开 Camera  捕捉图片，获取图片 base64 信息
/// </summary>
public class CameraCaptureImageForBase64 : MonoSingleton<CameraCaptureImageForBase64>
{#region data/// <summary>/// 获取图片成功事件/// </summary>public Action<string> OnGetBase64ImageSuccess;/// <summary>/// 获取图片失败事件/// </summary>public Action<string> OnGetBase64ImageFailed;private WebCamTexture webcamTexture; // 用于获取相机图像private Coroutine m_Coroutine;#endregion/// <summary>/// 捕捉图片/// </summary>public void CaptureImage(){m_Coroutine = StartCoroutine(CaptureRoutine());}/// <summary>/// 停止捕捉/// </summary>public void StopCapture(){if (m_Coroutine!=null){StopCoroutine(m_Coroutine);m_Coroutine = null;}}#region private functionprivate IEnumerator CaptureRoutine(){// 检查是否有相机权限if (!Permission.HasUserAuthorizedPermission(Permission.Camera)){// 如果没有相机权限，等待用户授权yield return RequestCameraPermission();}// 启动相机StartCamera();// 等待用户拍照yield return WaitForUserToTakePhoto();// 拍照Texture2D photo = TakePhoto();if (photo != null){// 将Texture2D转换为byte数组byte[] imageBytes = photo.EncodeToPNG();// 将byte数组转换为base64字符串string base64String = Convert.ToBase64String(imageBytes);Debug.Log("Base64 Image: " + base64String);// 在这里执行你的异步任务，例如上传base64图片到服务器OnGetBase64ImageSuccess?.Invoke(base64String);}else {OnGetBase64ImageFailed?.Invoke("capture image failed.");}// 停止相机StopCamera();}private IEnumerator RequestCameraPermission(){// 请求相机权限Permission.RequestUserPermission(Permission.Camera);// 等待用户授权yield return new WaitUntil(() => Permission.HasUserAuthorizedPermission(Permission.Camera));}private void StartCamera(){// 获取可用的相机设备WebCamDevice[] devices = WebCamTexture.devices;if (devices.Length > 0){// 使用第一个相机设备webcamTexture = new WebCamTexture(devices[0].name);webcamTexture.Play();}else{Debug.LogError("No camera device available.");OnGetBase64ImageFailed?.Invoke("No camera device available.");}}private IEnumerator WaitForUserToTakePhoto(){// 可以其他按键操作拍照// 这里直接等到若干秒yield return new WaitForSecondsRealtime(0.1f);}private Texture2D TakePhoto(){if (webcamTexture != null && webcamTexture.isPlaying){Texture2D photo = new Texture2D(webcamTexture.width, webcamTexture.height);photo.SetPixels(webcamTexture.GetPixels());photo.Apply();return photo;}return null;}private void StopCamera(){if (webcamTexture != null && webcamTexture.isPlaying){webcamTexture.Stop();}}#endregion
}

4、ImageLoader


using System;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.AddressableAssets;
using UnityEngine.ResourceManagement.AsyncOperations;public class ImageLoader : Singleton<ImageLoader>
{[SerializeField] private string imageAddress = "车"; // 设置为你图片在Addressables中的地址public async Task<string> LoadImageAndConvertToBase64(){string base64String = string.Empty;// 使用Addressables加载图片AsyncOperationHandle<Texture2D> handle = Addressables.LoadAssetAsync<Texture2D>(imageAddress);// 等待加载完成await handle.Task;if (handle.Status == AsyncOperationStatus.Succeeded){Texture2D texture = handle.Result;// 将Texture2D转换为byte数组byte[] imageBytes = texture.EncodeToPNG();// 将byte数组转换为base64字符串base64String = Convert.ToBase64String(imageBytes);Debug.Log("Base64 Image: " + base64String);}else{Debug.LogError("Failed to load image: " + handle.OperationException);}// 释放资源Addressables.Release(handle);return base64String;}
}