87 lines
2.5 KiB
TypeScript
87 lines
2.5 KiB
TypeScript
import { streamText } from "ai";
|
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
|
|
const provider = createOpenAICompatible({
|
|
name: "llama.cpp",
|
|
apiKey: "local",
|
|
baseURL: "http://127.0.0.1:8080/v1",
|
|
includeUsage: true, // Include usage information in streaming responses
|
|
});
|
|
|
|
const { fullStream, reasoningText, text } = streamText({
|
|
model: provider("qwen3.5-35b-a3b"),
|
|
prompt: Bun.argv[2]!,
|
|
});
|
|
|
|
const moveCursor = (x: number, y: number) =>
|
|
new Promise<void>((resolve) => process.stdout.moveCursor(x, y, resolve));
|
|
const clearScreenDown = () =>
|
|
new Promise<void>((resolve) => process.stdout.clearScreenDown(resolve));
|
|
|
|
let lineCount = 0;
|
|
let charsSinceNL = 0;
|
|
const [numColumns, numRows] = process.stdout.getWindowSize();
|
|
|
|
for await (const part of fullStream) {
|
|
if (part.type === "reasoning-start") {
|
|
process.stdout.write("\u001b[2m\n");
|
|
} else if (part.type === "reasoning-delta") {
|
|
process.stdout.write(part.text);
|
|
|
|
charsSinceNL += part.text.length;
|
|
if (part.text.includes("\n")) {
|
|
lineCount += part.text.match(/\n/g)?.length ?? 1;
|
|
charsSinceNL = 0;
|
|
} else if (charsSinceNL >= numColumns) {
|
|
lineCount++;
|
|
charsSinceNL = charsSinceNL - numColumns;
|
|
}
|
|
|
|
if (lineCount >= numRows / 2) {
|
|
await moveCursor(0, -lineCount);
|
|
await clearScreenDown();
|
|
lineCount = 0;
|
|
charsSinceNL = 0;
|
|
}
|
|
} else if (part.type === "reasoning-end") {
|
|
process.stdout.write("\n\u001b[22m");
|
|
await moveCursor(0, -lineCount - 2);
|
|
await clearScreenDown();
|
|
lineCount = 0;
|
|
charsSinceNL = 0;
|
|
} else if (part.type === "text-start") {
|
|
process.stdout.write("\n");
|
|
} else if (part.type === "text-delta") {
|
|
process.stdout.write(part.text);
|
|
|
|
charsSinceNL += part.text.length;
|
|
if (part.text.includes("\n")) {
|
|
lineCount += part.text.match(/\n/g)?.length ?? 1;
|
|
charsSinceNL = 0;
|
|
} else if (charsSinceNL >= numColumns) {
|
|
lineCount++;
|
|
charsSinceNL = charsSinceNL - numColumns;
|
|
}
|
|
|
|
if (lineCount >= numRows / 2) {
|
|
await moveCursor(0, -lineCount);
|
|
await clearScreenDown();
|
|
lineCount = 0;
|
|
charsSinceNL = 0;
|
|
}
|
|
} else if (part.type === "text-end") {
|
|
await moveCursor(0, -lineCount - 1);
|
|
await clearScreenDown();
|
|
}
|
|
}
|
|
|
|
const reasoning = await reasoningText;
|
|
const response = await text;
|
|
|
|
Bun.file("reasoning.md").write(reasoning ?? "");
|
|
Bun.file("response.md").write(response);
|
|
|
|
Bun.spawnSync({
|
|
cmd: ["glow", "response.md"],
|
|
stdio: ["inherit", "inherit", "inherit"],
|
|
});
|