30 lines
929 B
TypeScript
30 lines
929 B
TypeScript
import { streamText } from "ai";
|
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
|
|
const provider = createOpenAICompatible({
|
|
name: "llama.cpp",
|
|
apiKey: "local",
|
|
baseURL: "http://127.0.0.1:8080/v1",
|
|
includeUsage: true, // Include usage information in streaming responses
|
|
});
|
|
|
|
const { fullStream } = streamText({
|
|
model: provider("qwen3.5-35b-a3b"),
|
|
prompt: Bun.argv[2]!,
|
|
});
|
|
|
|
for await (const part of fullStream) {
|
|
if (part.type === "reasoning-start") {
|
|
process.stdout.write("<think>\n");
|
|
} else if (part.type === "reasoning-delta") {
|
|
process.stdout.write(part.text);
|
|
} else if (part.type === "reasoning-end") {
|
|
process.stdout.write("\n</think>");
|
|
} else if (part.type === "text-start") {
|
|
process.stdout.write("\n");
|
|
} else if (part.type === "text-delta") {
|
|
process.stdout.write(part.text);
|
|
} else if (part.type === "text-end") {
|
|
process.stdout.write("\n");
|
|
}
|
|
}
|