66 lines
1.8 KiB
TypeScript
66 lines
1.8 KiB
TypeScript
import os from "node:os";
|
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
import { type ModelMessage, streamText } from "ai";
|
|
|
|
import { renderStream } from "./src/render";
|
|
|
|
const home = os.homedir();
|
|
const convoFile = Bun.file(`${home}/.aicl_convo.json`);
|
|
const reasoningFile = Bun.file(`${home}/.aicl_reasoning.md`);
|
|
const responsePath = `${home}/.aicl_response.md`;
|
|
const responseFile = Bun.file(responsePath);
|
|
|
|
const provider = createOpenAICompatible({
|
|
name: "llama.cpp",
|
|
apiKey: "local",
|
|
baseURL: "http://127.0.0.1:8080/v1",
|
|
includeUsage: true, // Include usage information in streaming responses
|
|
});
|
|
|
|
const messages: ModelMessage[] = [];
|
|
|
|
try {
|
|
const convo = await convoFile.json();
|
|
messages.push(...convo);
|
|
} catch {}
|
|
|
|
const prompt = Bun.argv[2] ?? "Introduce yourself.";
|
|
messages.push({ role: "user", content: prompt });
|
|
|
|
const { fullStream, textStream, reasoningText, text, totalUsage } = streamText({
|
|
model: provider("qwen3.5-35b-a3b"),
|
|
messages,
|
|
});
|
|
|
|
const start = Date.now();
|
|
|
|
const streamRender = renderStream(fullStream);
|
|
|
|
const [width] = process.stdout.getWindowSize();
|
|
const glow = Bun.spawn({
|
|
cmd: ["glow", "-w", `${width}`, "-"],
|
|
stdin: textStream,
|
|
stdout: "inherit",
|
|
});
|
|
|
|
const reasoning = await reasoningText;
|
|
const response = await text;
|
|
|
|
const stop = Date.now();
|
|
|
|
messages.push({ role: "assistant", content: response });
|
|
await convoFile.write(JSON.stringify(messages));
|
|
|
|
await reasoningFile.write(reasoning ?? "");
|
|
await responseFile.write(response);
|
|
|
|
await streamRender;
|
|
await glow.exited;
|
|
|
|
const { inputTokens, outputTokens } = await totalUsage;
|
|
const context = (inputTokens ?? 0) / 1000;
|
|
const output = (outputTokens ?? 0) / 1000;
|
|
const time = (stop - start) / 1000;
|
|
process.stderr.write(
|
|
`\u001b[2mContext: ${context.toFixed(1)}k, Output: ${output.toFixed(1)}k, Time: ${time.toFixed(2)}s\u001b[22m\n`,
|
|
);
|