feat: add args parsing; refactor scripts
This commit is contained in:
parent
b3cef4dd03
commit
bcd9771cd5
3 changed files with 59 additions and 41 deletions
94
index.ts
94
index.ts
|
|
@ -1,58 +1,78 @@
|
||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
|
import { parseArgs } from "node:util";
|
||||||
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
||||||
import { type ModelMessage, streamText } from "ai";
|
import { type ModelMessage, streamText } from "ai";
|
||||||
|
|
||||||
import { renderStream } from "./src/render";
|
import { renderStream } from "./src/render";
|
||||||
|
|
||||||
const home = os.homedir();
|
const home = os.homedir();
|
||||||
const convoFile = Bun.file(`${home}/.aicl_convo.json`);
|
const convoFile = Bun.file(`${home}/.aicl_convo.json`);
|
||||||
|
|
||||||
const provider = createOpenAICompatible({
|
const { values: args, positionals } = parseArgs({
|
||||||
name: "llama.cpp",
|
args: Bun.argv,
|
||||||
apiKey: "local",
|
options: {
|
||||||
baseURL: "http://127.0.0.1:8080/v1",
|
model: {
|
||||||
includeUsage: true, // Include usage information in streaming responses
|
type: "string",
|
||||||
|
},
|
||||||
|
new: {
|
||||||
|
type: "boolean",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
strict: true,
|
||||||
|
allowPositionals: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
const messages: ModelMessage[] = [];
|
async function main() {
|
||||||
|
const provider = createOpenAICompatible({
|
||||||
|
name: "llama.cpp",
|
||||||
|
apiKey: "local",
|
||||||
|
baseURL: "http://127.0.0.1:8080/v1",
|
||||||
|
includeUsage: true, // Include usage information in streaming responses
|
||||||
|
});
|
||||||
|
|
||||||
try {
|
const messages: ModelMessage[] = [];
|
||||||
const convo = await convoFile.json();
|
|
||||||
messages.push(...convo);
|
|
||||||
} catch {}
|
|
||||||
|
|
||||||
const prompt = Bun.argv[2] ?? "Introduce yourself.";
|
if (!args.new) {
|
||||||
messages.push({ role: "user", content: prompt });
|
try {
|
||||||
|
const convo = await convoFile.json();
|
||||||
|
messages.push(...convo);
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
|
||||||
const { fullStream, textStream, text, totalUsage } = streamText({
|
const prompt = positionals[2] ?? "Introduce yourself.";
|
||||||
model: provider("gemma4-26b-a4b"),
|
messages.push({ role: "user", content: prompt });
|
||||||
messages,
|
|
||||||
});
|
|
||||||
|
|
||||||
const start = Date.now();
|
const { fullStream, textStream, text, totalUsage } = streamText({
|
||||||
|
model: provider(args.model ?? "gemma4-26b-a4b"),
|
||||||
|
messages,
|
||||||
|
});
|
||||||
|
|
||||||
const streamRender = renderStream(fullStream);
|
const start = Date.now();
|
||||||
|
|
||||||
const [width] = process.stdout.getWindowSize();
|
const streamRender = renderStream(fullStream);
|
||||||
const glow = Bun.spawn({
|
|
||||||
cmd: ["glow", "-w", `${width}`, "-"],
|
|
||||||
stdin: textStream,
|
|
||||||
stdout: "inherit",
|
|
||||||
});
|
|
||||||
|
|
||||||
await streamRender;
|
const [width] = process.stdout.getWindowSize();
|
||||||
|
const glow = Bun.spawn({
|
||||||
|
cmd: ["glow", "-w", `${width}`, "-"],
|
||||||
|
stdin: textStream,
|
||||||
|
stdout: "inherit",
|
||||||
|
});
|
||||||
|
|
||||||
const stop = Date.now();
|
await streamRender;
|
||||||
|
|
||||||
await glow.exited;
|
const stop = Date.now();
|
||||||
|
|
||||||
messages.push({ role: "assistant", content: await text });
|
await glow.exited;
|
||||||
await convoFile.write(JSON.stringify(messages));
|
|
||||||
|
|
||||||
const { inputTokens, outputTokens } = await totalUsage;
|
messages.push({ role: "assistant", content: await text });
|
||||||
const context = (inputTokens ?? 0) / 1000;
|
await convoFile.write(JSON.stringify(messages));
|
||||||
const output = (outputTokens ?? 0) / 1000;
|
|
||||||
const time = (stop - start) / 1000;
|
const { inputTokens, outputTokens } = await totalUsage;
|
||||||
process.stderr.write(
|
const context = (inputTokens ?? 0) / 1000;
|
||||||
`\u001b[2mContext: ${context.toFixed(1)}k, Output: ${output.toFixed(1)}k, Time: ${time.toFixed(2)}s\u001b[22m\n`,
|
const output = (outputTokens ?? 0) / 1000;
|
||||||
);
|
const time = (stop - start) / 1000;
|
||||||
|
process.stderr.write(
|
||||||
|
`\u001b[2mContext: ${context.toFixed(1)}k, Output: ${output.toFixed(1)}k, Time: ${time.toFixed(2)}s\u001b[22m\n`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await main();
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
"dev": "bun run index.ts",
|
"dev": "bun run index.ts",
|
||||||
"prod": "bun run out/index.js",
|
"prod": "bun run out/index.js",
|
||||||
"check": "bunx --bun @biomejs/biome check --write",
|
"check": "bunx --bun @biomejs/biome check --write",
|
||||||
"build": "bun run check && bun run build:bundle && bun run shebang",
|
"build:js": "bun run check && bun run build:bundle && bun run shebang",
|
||||||
"build:bundle": "bun build --target=bun --production --sourcemap --outdir=out index.ts",
|
"build:bundle": "bun build --target=bun --production --sourcemap --outdir=out index.ts",
|
||||||
"build:bin": "bun build --compile --outfile=out/aicl index.ts",
|
"build:bin": "bun build --compile --outfile=out/aicl index.ts",
|
||||||
"shebang": "sed -i '1i#!\\/usr\\/bin\\/env bun' out/index.js"
|
"shebang": "sed -i '1i#!\\/usr\\/bin\\/env bun' out/index.js"
|
||||||
|
|
|
||||||
|
|
@ -59,9 +59,7 @@ export async function renderStream(
|
||||||
resetCounts();
|
resetCounts();
|
||||||
}
|
}
|
||||||
|
|
||||||
while (!memOk) {
|
while (!memOk) await Bun.sleep(10);
|
||||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue