next-chat/app/client/platforms/openai.ts

"use client";
// azure and openai, using same models. so using same LLMApi.
import {
  ApiPath,
  DEFAULT_API_HOST,
  DEFAULT_MODELS,
  OpenaiPath,
  Azure,
  REQUEST_TIMEOUT_MS,
  ServiceProvider,
} from "@/app/constant";
import {
  ChatMessageTool,
  useAccessStore,
  useAppConfig,
  useChatStore,
} from "@/app/store";
import { collectModelsWithDefaultModel } from "@/app/utils/model";
import {
  preProcessImageContent,
  uploadImage,
  base64Image2Blob,
} from "@/app/utils/chat";
import { cloudflareAIGatewayUrl } from "@/app/utils/cloudflare";
import { DalleSize, DalleQuality, DalleStyle } from "@/app/typing";

import {
  ChatOptions,
  getHeaders,
  LLMApi,
  LLMModel,
  LLMUsage,
  MultimodalContent,
} from "../api";
import Locale from "../../locales";
import {
  EventStreamContentType,
  fetchEventSource,
} from "@fortaine/fetch-event-source";
import { prettyObject } from "@/app/utils/format";
import { getClientConfig } from "@/app/config/client";
import {
  getMessageTextContent,
  getMessageImages,
  isVisionModel,
  isDalle3 as _isDalle3,
} from "@/app/utils";

export interface OpenAIListModelResponse {
  object: string;
  data: Array<{
    id: string;
    object: string;
    root: string;
  }>;
}

export interface RequestPayload {
  messages: {
    role: "system" | "user" | "assistant";
    content: string | MultimodalContent[];
  }[];
  stream?: boolean;
  model: string;
  temperature: number;
  presence_penalty: number;
  frequency_penalty: number;
  top_p: number;
  max_tokens?: number;
}

export interface DalleRequestPayload {
  model: string;
  prompt: string;
  response_format: "url" | "b64_json";
  n: number;
  size: DalleSize;
  quality: DalleQuality;
  style: DalleStyle;
}

export class ChatGPTApi implements LLMApi {
  private disableListModels = true;

  path(path: string): string {
    const accessStore = useAccessStore.getState();

    let baseUrl = "";

    const isAzure = path.includes("deployments");
    if (accessStore.useCustomConfig) {
      if (isAzure && !accessStore.isValidAzure()) {
        throw Error(
          "incomplete azure config, please check it in your settings page",
        );
      }

      baseUrl = isAzure ? accessStore.azureUrl : accessStore.openaiUrl;
    }

    if (baseUrl.length === 0) {
      const isApp = !!getClientConfig()?.isApp;
      const apiPath = isAzure ? ApiPath.Azure : ApiPath.OpenAI;
      baseUrl = isApp ? DEFAULT_API_HOST + "/proxy" + apiPath : apiPath;
    }

    if (baseUrl.endsWith("/")) {
      baseUrl = baseUrl.slice(0, baseUrl.length - 1);
    }
    if (
      !baseUrl.startsWith("http") &&
      !isAzure &&
      !baseUrl.startsWith(ApiPath.OpenAI)
    ) {
      baseUrl = "https://" + baseUrl;
    }

    console.log("[Proxy Endpoint] ", baseUrl, path);

    // try rebuild url, when using cloudflare ai gateway in client
    return cloudflareAIGatewayUrl([baseUrl, path].join("/"));
  }

  async extractMessage(res: any) {
    if (res.error) {
      return "```\n" + JSON.stringify(res, null, 4) + "\n```";
    }
    // dalle3 model return url, using url create image message
    if (res.data) {
      let url = res.data?.at(0)?.url ?? "";
      const b64_json = res.data?.at(0)?.b64_json ?? "";
      if (!url && b64_json) {
        // uploadImage
        url = await uploadImage(base64Image2Blob(b64_json, "image/png"));
      }
      return [
        {
          type: "image_url",
          image_url: {
            url,
          },
        },
      ];
    }
    return res.choices?.at(0)?.message?.content ?? res;
  }

  async chat(options: ChatOptions) {
    const modelConfig = {
      ...useAppConfig.getState().modelConfig,
      ...useChatStore.getState().currentSession().mask.modelConfig,
      ...{
        model: options.config.model,
        providerName: options.config.providerName,
      },
    };

    let requestPayload: RequestPayload | DalleRequestPayload;

    const isDalle3 = _isDalle3(options.config.model);
    if (isDalle3) {
      const prompt = getMessageTextContent(
        options.messages.slice(-1)?.pop() as any,
      );
      requestPayload = {
        model: options.config.model,
        prompt,
        // URLs are only valid for 60 minutes after the image has been generated.
        response_format: "b64_json", // using b64_json, and save image in CacheStorage
        n: 1,
        size: options.config?.size ?? "1024x1024",
        quality: options.config?.quality ?? "standard",
        style: options.config?.style ?? "vivid",
      };
    } else {
      const visionModel = isVisionModel(options.config.model);
      const messages: ChatOptions["messages"] = [];
      for (const v of options.messages) {
        const content = visionModel
          ? await preProcessImageContent(v.content)
          : getMessageTextContent(v);
        messages.push({ role: v.role, content });
      }

      requestPayload = {
        messages,
        stream: options.config.stream,
        model: modelConfig.model,
        temperature: modelConfig.temperature,
        presence_penalty: modelConfig.presence_penalty,
        frequency_penalty: modelConfig.frequency_penalty,
        top_p: modelConfig.top_p,
        // max_tokens: Math.max(modelConfig.max_tokens, 1024),
        // Please do not ask me why not send max_tokens, no reason, this param is just shit, I dont want to explain anymore.
      };

      // add max_tokens to vision model
      if (visionModel && modelConfig.model.includes("preview")) {
        requestPayload["max_tokens"] = Math.max(modelConfig.max_tokens, 4000);
      }
    }

    console.log("[Request] openai payload: ", requestPayload);

    const shouldStream = !isDalle3 && !!options.config.stream;
    const controller = new AbortController();
    options.onController?.(controller);

    try {
      let chatPath = "";
      if (modelConfig.providerName === ServiceProvider.Azure) {
        // find model, and get displayName as deployName
        const { models: configModels, customModels: configCustomModels } =
          useAppConfig.getState();
        const {
          defaultModel,
          customModels: accessCustomModels,
          useCustomConfig,
        } = useAccessStore.getState();
        const models = collectModelsWithDefaultModel(
          configModels,
          [configCustomModels, accessCustomModels].join(","),
          defaultModel,
        );
        const model = models.find(
          (model) =>
            model.name === modelConfig.model &&
            model?.provider?.providerName === ServiceProvider.Azure,
        );
        chatPath = this.path(
          (isDalle3 ? Azure.ImagePath : Azure.ChatPath)(
            (model?.displayName ?? model?.name) as string,
            useCustomConfig ? useAccessStore.getState().azureApiVersion : "",
          ),
        );
      } else {
        chatPath = this.path(
          isDalle3 ? OpenaiPath.ImagePath : OpenaiPath.ChatPath,
        );
      }
      const chatPayload = {
        method: "POST",
        body: JSON.stringify(requestPayload),
        signal: controller.signal,
        headers: getHeaders(),
      };

      // make a fetch request
      const requestTimeoutId = setTimeout(
        () => controller.abort(),
        isDalle3 ? REQUEST_TIMEOUT_MS * 2 : REQUEST_TIMEOUT_MS, // dalle3 using b64_json is slow.
      );

      if (shouldStream) {
        let responseText = "";
        let remainText = "";
        let finished = false;
        let running = false;
        let runTools: ChatMessageTool[] = [];

        // animate response to make it looks smooth
        function animateResponseText() {
          if (finished || controller.signal.aborted) {
            responseText += remainText;
            console.log("[Response Animation] finished");
            if (responseText?.length === 0) {
              options.onError?.(new Error("empty response from server"));
            }
            return;
          }

          if (remainText.length > 0) {
            const fetchCount = Math.max(1, Math.round(remainText.length / 60));
            const fetchText = remainText.slice(0, fetchCount);
            responseText += fetchText;
            remainText = remainText.slice(fetchCount);
            options.onUpdate?.(responseText, fetchText);
          }

          requestAnimationFrame(animateResponseText);
        }

        // start animaion
        animateResponseText();

        // TODO 后面这里是从选择的plugins中获取function列表
        const funcs = {
          get_current_weather: (args: any) => {
            console.log("call get_current_weather", args);
            return new Promise((resolve) => {
              setTimeout(() => resolve("30"), 3000);
            });
          },
        };
        const finish = () => {
          if (!finished) {
            console.log("try run tools", runTools.length, finished, running);
            if (!running && runTools.length > 0) {
              const toolCallMessage = {
                role: "assistant",
                tool_calls: [...runTools],
              };
              running = true;
              runTools.splice(0, runTools.length); // empty runTools
              return Promise.all(
                toolCallMessage.tool_calls.map((tool) => {
                  options?.onBeforeTool?.(tool);
                  return Promise.resolve(
                    // @ts-ignore
                    funcs[tool.function.name](
                      // @ts-ignore
                      JSON.parse(tool.function.arguments),
                    ),
                  )
                    .then((content) => {
                      options?.onAfterTool?.({
                        ...tool,
                        content,
                        isError: false,
                      });
                      return content;
                    })
                    .catch((e) => {
                      options?.onAfterTool?.({ ...tool, isError: true });
                      return e.toString();
                    })
                    .then((content) => ({
                      role: "tool",
                      content,
                      tool_call_id: tool.id,
                    }));
                }),
              ).then((toolCallResult) => {
                console.log("end runTools", toolCallMessage, toolCallResult);
                // @ts-ignore
                requestPayload?.messages?.splice(
                  // @ts-ignore
                  requestPayload?.messages?.length,
                  0,
                  toolCallMessage,
                  ...toolCallResult,
                );
                setTimeout(() => {
                  // call again
                  console.log("start again");
                  running = false;
                  chatApi(chatPath, requestPayload as RequestPayload); // call fetchEventSource
                }, 60);
              });
              console.log("try run tools", runTools.length, finished);
              return;
            }
            if (running) {
              return;
            }
            finished = true;
            options.onFinish(responseText + remainText);
          }
        };

        controller.signal.onabort = finish;

        function chatApi(chatPath: string, requestPayload: RequestPayload) {
          const chatPayload = {
            method: "POST",
            body: JSON.stringify({
              ...requestPayload,
              // TODO 这里暂时写死的，后面从store.tools中按照当前session中选择的获取
              tools: [
                {
                  type: "function",
                  function: {
                    name: "get_current_weather",
                    description: "Get the current weather",
                    parameters: {
                      type: "object",
                      properties: {
                        location: {
                          type: "string",
                          description:
                            "The city and country, eg. San Francisco, USA",
                        },
                        format: {
                          type: "string",
                          enum: ["celsius", "fahrenheit"],
                        },
                      },
                      required: ["location", "format"],
                    },
                  },
                },
              ],
            }),
            signal: controller.signal,
            headers: getHeaders(),
          };
          console.log("chatApi", chatPath, requestPayload, chatPayload);
          fetchEventSource(chatPath, {
            ...chatPayload,
            async onopen(res) {
              clearTimeout(requestTimeoutId);
              const contentType = res.headers.get("content-type");
              console.log(
                "[OpenAI] request response content type: ",
                contentType,
              );

              if (contentType?.startsWith("text/plain")) {
                responseText = await res.clone().text();
                return finish();
              }

              if (
                !res.ok ||
                !res.headers
                  .get("content-type")
                  ?.startsWith(EventStreamContentType) ||
                res.status !== 200
              ) {
                const responseTexts = [responseText];
                let extraInfo = await res.clone().text();
                try {
                  const resJson = await res.clone().json();
                  extraInfo = prettyObject(resJson);
                } catch {}

                if (res.status === 401) {
                  responseTexts.push(Locale.Error.Unauthorized);
                }

                if (extraInfo) {
                  responseTexts.push(extraInfo);
                }

                responseText = responseTexts.join("\n\n");

                return finish();
              }
            },
            onmessage(msg) {
              if (msg.data === "[DONE]" || finished) {
                return finish();
              }
              const text = msg.data;
              try {
                const json = JSON.parse(text);
                const choices = json.choices as Array<{
                  delta: {
                    content: string;
                    tool_calls: ChatMessageTool[];
                  };
                }>;
                console.log("choices", choices);
                const delta = choices[0]?.delta?.content;
                const tool_calls = choices[0]?.delta?.tool_calls;
                const textmoderation = json?.prompt_filter_results;

                if (delta) {
                  remainText += delta;
                }
                if (tool_calls?.length > 0) {
                  const index = tool_calls[0]?.index;
                  const id = tool_calls[0]?.id;
                  const args = tool_calls[0]?.function?.arguments;
                  if (id) {
                    runTools.push({
                      id,
                      type: tool_calls[0]?.type,
                      function: {
                        name: tool_calls[0]?.function?.name as string,
                        arguments: args,
                      },
                    });
                  } else {
                    // @ts-ignore
                    runTools[index]["function"]["arguments"] += args;
                  }
                }

                console.log("runTools", runTools);

                if (
                  textmoderation &&
                  textmoderation.length > 0 &&
                  ServiceProvider.Azure
                ) {
                  const contentFilterResults =
                    textmoderation[0]?.content_filter_results;
                  console.log(
                    `[${ServiceProvider.Azure}] [Text Moderation] flagged categories result:`,
                    contentFilterResults,
                  );
                }
              } catch (e) {
                console.error("[Request] parse error", text, msg);
              }
            },
            onclose() {
              finish();
            },
            onerror(e) {
              options.onError?.(e);
              throw e;
            },
            openWhenHidden: true,
          });
        }
        chatApi(chatPath, requestPayload as RequestPayload); // call fetchEventSource
      } else {
        const res = await fetch(chatPath, chatPayload);
        clearTimeout(requestTimeoutId);

        const resJson = await res.json();
        const message = await this.extractMessage(resJson);
        options.onFinish(message);
      }
    } catch (e) {
      console.log("[Request] failed to make a chat request", e);
      options.onError?.(e as Error);
    }
  }
  async usage() {
    const formatDate = (d: Date) =>
      `${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, "0")}-${d
        .getDate()
        .toString()
        .padStart(2, "0")}`;
    const ONE_DAY = 1 * 24 * 60 * 60 * 1000;
    const now = new Date();
    const startOfMonth = new Date(now.getFullYear(), now.getMonth(), 1);
    const startDate = formatDate(startOfMonth);
    const endDate = formatDate(new Date(Date.now() + ONE_DAY));

    const [used, subs] = await Promise.all([
      fetch(
        this.path(
          `${OpenaiPath.UsagePath}?start_date=${startDate}&end_date=${endDate}`,
        ),
        {
          method: "GET",
          headers: getHeaders(),
        },
      ),
      fetch(this.path(OpenaiPath.SubsPath), {
        method: "GET",
        headers: getHeaders(),
      }),
    ]);

    if (used.status === 401) {
      throw new Error(Locale.Error.Unauthorized);
    }

    if (!used.ok || !subs.ok) {
      throw new Error("Failed to query usage from openai");
    }

    const response = (await used.json()) as {
      total_usage?: number;
      error?: {
        type: string;
        message: string;
      };
    };

    const total = (await subs.json()) as {
      hard_limit_usd?: number;
    };

    if (response.error && response.error.type) {
      throw Error(response.error.message);
    }

    if (response.total_usage) {
      response.total_usage = Math.round(response.total_usage) / 100;
    }

    if (total.hard_limit_usd) {
      total.hard_limit_usd = Math.round(total.hard_limit_usd * 100) / 100;
    }

    return {
      used: response.total_usage,
      total: total.hard_limit_usd,
    } as LLMUsage;
  }

  async models(): Promise<LLMModel[]> {
    if (this.disableListModels) {
      return DEFAULT_MODELS.slice();
    }

    const res = await fetch(this.path(OpenaiPath.ListModelPath), {
      method: "GET",
      headers: {
        ...getHeaders(),
      },
    });

    const resJson = (await res.json()) as OpenAIListModelResponse;
    const chatModels = resJson.data?.filter((m) => m.id.startsWith("gpt-"));
    console.log("[Models]", chatModels);

    if (!chatModels) {
      return [];
    }

    //由于目前 OpenAI 的 disableListModels 默认为 true，所以当前实际不会运行到这场
    let seq = 1000; //同 Constant.ts 中的排序保持一致
    return chatModels.map((m) => ({
      name: m.id,
      available: true,
      sorted: seq++,
      provider: {
        id: "openai",
        providerName: "OpenAI",
        providerType: "openai",
        sorted: 1,
      },
    }));
  }
}
export { OpenaiPath };