GitHub Repository Analysis & Explainer Agent: DeepWiki Explained, and How to Build Your Own with Hono and OpenRouter

https://github.com/honojs/hono

https://deepwiki.com/honojs/hono

claude mcp add -s user -t http deepwiki https://mcp.deepwiki.com/mcp

{
  "mcpServers": {
    "deepwiki": { "url": "https://mcp.deepwiki.com/mcp" }
  }
}

{
  "repo_notes": [
    {
      "content": "StackBudget is a Claude Code skill. The scanner is in skills/stack-budget/scripts, pricing data lives in assets/engine.js and references/pricing.md. examples/ holds generated reports, not source."
    }
  ],
  "pages": [
    { "title": "Overview", "purpose": "What StackBudget does and who it is for" },
    { "title": "How the scanner works", "purpose": "How manifests, env keys and configs become stack.json", "parent": "Overview" },
    { "title": "Pricing engine", "purpose": "How rates are stored, dated and applied to presets" },
    { "title": "Adding a new provider", "purpose": "Step-by-step guide for contributors" }
  ]
}

mkdir localwiki && cd localwiki
npm init -y
npm i hono @hono/node-server openai
npm i -D typescript tsx @types/node

export OPENROUTER_API_KEY=sk-or-...
export GITHUB_TOKEN=ghp_...   # optional, but strongly recommended

localwiki/
├── src/
│   ├── github.ts   # fetch tree + files from a public repo
│   ├── llm.ts      # OpenRouter client: chat + embeddings
│   ├── store.ts    # JSON file store, chunking, cosine search
│   ├── wiki.ts     # build step: summary, index, embeddings
│   └── index.ts    # Hono routes
└── data/           # cached wikis (created on first run)

// Pull a public repo's file tree and the text files worth reading.
const GH = "https://api.github.com";
const headers: Record<string, string> = {
  Accept: "application/vnd.github+json",
  "User-Agent": "localwiki",
  ...(process.env.GITHUB_TOKEN ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } : {}),
};

const SKIP_DIRS = /(^|\/)(node_modules|dist|build|\.next|vendor|coverage|\.git)\//;
const SKIP_FILES = /(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb|\.min\.js|\.map)$/;
const TEXT_EXT = /\.(md|mdx|ts|tsx|js|jsx|mjs|py|go|rs|rb|java|kt|swift|php|vue|svelte|astro|json|toml|yaml|yml|sql|sh|css)$/i;
const MAX_BYTES = 60_000; // skip giant generated files
const MAX_FILES = 300;    // keep the first version cheap

export type RepoFile = { path: string; content: string };
export type RepoMeta = { owner: string; repo: string; branch: string; description: string | null; stars: number };

export function parseRepoUrl(url: string) {
  const m = url.match(/github\.com\/([^/]+)\/([^/#?]+)/) ?? url.match(/^([^/]+)\/([^/]+)$/);
  if (!m) throw new Error("Pass a GitHub URL or owner/repo");
  return { owner: m[1], repo: m[2].replace(/\.git$/, "") };
}

async function gh<T>(path: string): Promise<T> {
  const res = await fetch(`${GH}${path}`, { headers });
  if (!res.ok) throw new Error(`GitHub ${res.status} on ${path}`);
  return res.json() as Promise<T>;
}

export async function fetchRepo(owner: string, repo: string) {
  const info = await gh<{ default_branch: string; description: string | null; stargazers_count: number }>(
    `/repos/${owner}/${repo}`,
  );
  const branch = info.default_branch;
  const tree = await gh<{ tree: { path: string; type: string; size?: number }[]; truncated: boolean }>(
    `/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`,
  );

  const allPaths = tree.tree.filter((t) => t.type === "blob").map((t) => t.path);
  const wanted = tree.tree
    .filter((t) => t.type === "blob" && (t.size ?? 0) < MAX_BYTES)
    .filter((t) => TEXT_EXT.test(t.path) && !SKIP_DIRS.test(t.path) && !SKIP_FILES.test(t.path))
    // README and manifests first, so the summary always has them
    .sort((a, b) => score(b.path) - score(a.path))
    .slice(0, MAX_FILES);

  // raw.githubusercontent.com doesn't count against the REST API rate limit
  const files: RepoFile[] = [];
  for (const batch of chunk(wanted, 10)) {
    const got = await Promise.all(
      batch.map(async (t) => {
        const r = await fetch(`https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${t.path}`);
        return r.ok ? { path: t.path, content: await r.text() } : null;
      }),
    );
    files.push(...(got.filter(Boolean) as RepoFile[]));
  }

  const meta: RepoMeta = { owner, repo, branch, description: info.description, stars: info.stargazers_count };
  return { meta, allPaths, files, truncated: tree.truncated };
}

function score(p: string) {
  if (/^readme/i.test(p)) return 100;
  if (/^(package\.json|pyproject\.toml|go\.mod|cargo\.toml)$/i.test(p)) return 90;
  if (/(^|\/)(index|main|app|server)\.(ts|js|py|go)$/.test(p)) return 50;
  return 10 - p.split("/").length; // shallow files before deep ones
}

const chunk = <T,>(a: T[], n: number) =>
  Array.from({ length: Math.ceil(a.length / n) }, (_, i) => a.slice(i * n, i * n + n));

// OpenRouter speaks the OpenAI API, so the official SDK works with a different baseURL.
import OpenAI from "openai";

export const ai = new OpenAI({
  baseURL: "https://openrouter.ai/api/v1",
  apiKey: process.env.OPENROUTER_API_KEY,
  defaultHeaders: { "HTTP-Referer": "http://localhost:8787", "X-Title": "localwiki" },
});

export const CHAT_MODEL = process.env.CHAT_MODEL ?? "google/gemini-2.5-flash";
export const EMBED_MODEL = process.env.EMBED_MODEL ?? "openai/text-embedding-3-small";

export async function complete(system: string, user: string) {
  const r = await ai.chat.completions.create({
    model: CHAT_MODEL,
    messages: [{ role: "system", content: system }, { role: "user", content: user }],
  });
  return r.choices[0]?.message?.content ?? "";
}

export async function embed(texts: string[]) {
  const out: number[][] = [];
  for (let i = 0; i < texts.length; i += 64) {
    const r = await ai.embeddings.create({ model: EMBED_MODEL, input: texts.slice(i, i + 64) });
    out.push(...r.data.map((d) => d.embedding));
  }
  return out;
}

// The "vector DB" is a JSON file per repo. Fine for a few thousand chunks.
import { mkdir, readFile, writeFile } from "node:fs/promises";
import type { RepoMeta } from "./github.js";

export type Chunk = { path: string; start: number; text: string; vector: number[] };
export type Wiki = {
  meta: RepoMeta;
  summary: string;
  index: { folder: string; files: string[]; about: string }[];
  chunks: Chunk[];
  builtAt: string;
};

const dir = "./data";
const file = (o: string, r: string) => `${dir}/${o}__${r}.json`;

export async function save(w: Wiki) {
  await mkdir(dir, { recursive: true });
  await writeFile(file(w.meta.owner, w.meta.repo), JSON.stringify(w));
}

export async function load(o: string, r: string): Promise<Wiki | null> {
  try { return JSON.parse(await readFile(file(o, r), "utf8")); } catch { return null; }
}

export function splitIntoChunks(path: string, content: string, size = 1500, overlap = 200) {
  const out: { path: string; start: number; text: string }[] = [];
  for (let i = 0; i < content.length; i += size - overlap) {
    out.push({ path, start: i, text: content.slice(i, i + size) });
    if (i + size >= content.length) break;
  }
  return out;
}

export function topK(chunks: Chunk[], q: number[], k = 8) {
  const cos = (a: number[], b: number[]) => {
    let d = 0, na = 0, nb = 0;
    for (let i = 0; i < a.length; i++) { d += a[i] * b[i]; na += a[i] ** 2; nb += b[i] ** 2; }
    return d / (Math.sqrt(na) * Math.sqrt(nb));
  };
  return chunks.map((c) => ({ c, s: cos(c.vector, q) })).sort((x, y) => y.s - x.s).slice(0, k).map((x) => x.c);
}

// Build step: summary + folder index + embedded chunks.
import { fetchRepo } from "./github.js";
import { complete, embed } from "./llm.js";
import { splitIntoChunks, type Wiki } from "./store.js";

export async function buildWiki(owner: string, repo: string): Promise<Wiki> {
  const { meta, allPaths, files } = await fetchRepo(owner, repo);

  // 1. Summary: README + manifest + the file tree is usually enough
  const readme = files.find((f) => /^readme/i.test(f.path))?.content.slice(0, 12_000) ?? "";
  const manifest = files.find((f) => /^(package\.json|pyproject\.toml|go\.mod|cargo\.toml)$/i.test(f.path))?.content.slice(0, 4_000) ?? "";
  const summary = await complete(
    "You write short, accurate repository overviews for developers. Use only the material given. Say 'not clear from the code' instead of guessing.",
    `Repo: ${owner}/${repo}\nDescription: ${meta.description ?? "none"}\n\nFILE TREE:\n${allPaths.slice(0, 400).join("\n")}\n\nMANIFEST:\n${manifest}\n\nREADME:\n${readme}\n\n` +
      "Write: 1) what it is in two sentences, 2) tech stack, 3) how to run it, 4) the 5 most important folders and why.",
  );

  // 2. Index: one line per top-level folder
  const folders = new Map<string, string[]>();
  for (const f of files) {
    const top = f.path.includes("/") ? f.path.split("/")[0] : "(root)";
    folders.set(top, [...(folders.get(top) ?? []), f.path]);
  }
  const index = await Promise.all(
    [...folders].map(async ([folder, paths]) => {
      const sample = files.filter((f) => paths.includes(f.path)).slice(0, 4)
        .map((f) => `--- ${f.path}\n${f.content.slice(0, 1_500)}`).join("\n");
      const about = await complete(
        "Describe what this folder does in one sentence. No fluff.",
        `Folder: ${folder}\nFiles: ${paths.join(", ")}\n\n${sample}`,
      );
      return { folder, files: paths, about };
    }),
  );

  // 3. Chunks + embeddings for the chatbot
  const pieces = files.flatMap((f) => splitIntoChunks(f.path, f.content));
  const vectors = await embed(pieces.map((p) => `${p.path}\n${p.text}`));
  const chunks = pieces.map((p, i) => ({ ...p, vector: vectors[i] }));

  return { meta, summary, index, chunks, builtAt: new Date().toISOString() };
}

import { Hono } from "hono";
import { serve } from "@hono/node-server";
import { streamSSE } from "hono/streaming";
import { parseRepoUrl } from "./github.js";
import { buildWiki } from "./wiki.js";
import { load, save, topK } from "./store.js";
import { ai, CHAT_MODEL, embed } from "./llm.js";

const app = new Hono();

// POST /wiki { "url": "https://github.com/honojs/hono" } -> builds and caches the wiki
app.post("/wiki", async (c) => {
  const { url } = await c.req.json<{ url: string }>();
  const { owner, repo } = parseRepoUrl(url);
  const cached = await load(owner, repo);
  if (cached) return c.json({ status: "cached", repo: `${owner}/${repo}`, builtAt: cached.builtAt });
  const wiki = await buildWiki(owner, repo);
  await save(wiki);
  return c.json({ status: "built", repo: `${owner}/${repo}`, chunks: wiki.chunks.length });
});

app.get("/wiki/:owner/:repo/summary", async (c) => {
  const w = await load(c.req.param("owner"), c.req.param("repo"));
  return w ? c.json({ meta: w.meta, summary: w.summary }) : c.json({ error: "build it first" }, 404);
});

app.get("/wiki/:owner/:repo/index", async (c) => {
  const w = await load(c.req.param("owner"), c.req.param("repo"));
  return w ? c.json(w.index) : c.json({ error: "build it first" }, 404);
});

// POST /wiki/:owner/:repo/chat { "question": "..." } -> streamed answer with file citations
app.post("/wiki/:owner/:repo/chat", async (c) => {
  const w = await load(c.req.param("owner"), c.req.param("repo"));
  if (!w) return c.json({ error: "build it first" }, 404);
  const { question } = await c.req.json<{ question: string }>();

  const [qv] = await embed([question]);
  const hits = topK(w.chunks, qv, 8);
  const context = hits.map((h) => `### ${h.path} (offset ${h.start})\n${h.text}`).join("\n\n");

  return streamSSE(c, async (stream) => {
    await stream.writeSSE({ event: "sources", data: JSON.stringify([...new Set(hits.map((h) => h.path))]) });
    const res = await ai.chat.completions.create({
      model: CHAT_MODEL,
      stream: true,
      messages: [
        {
          role: "system",
          content: `You answer questions about ${w.meta.owner}/${w.meta.repo}. Use only the code context. Cite file paths in backticks. If the context doesn't contain the answer, say so.\n\nRepo summary:\n${w.summary}`,
        },
        { role: "user", content: `${question}\n\nCODE CONTEXT:\n${context}` },
      ],
    });
    for await (const part of res) {
      const token = part.choices[0]?.delta?.content;
      if (token) await stream.writeSSE({ event: "token", data: token });
    }
    await stream.writeSSE({ event: "done", data: "" });
  });
});

serve({ fetch: app.fetch, port: 8787 }, (i) => console.log(`localwiki on http://localhost:${i.port}`));

npx tsx src/index.ts
# localwiki on http://localhost:8787

curl -X POST localhost:8787/wiki \
  -H "content-type: application/json" \
  -d '{"url":"https://github.com/honojs/hono"}'

curl localhost:8787/wiki/honojs/hono/summary
curl localhost:8787/wiki/honojs/hono/index

curl -N -X POST localhost:8787/wiki/honojs/hono/chat \
  -H "content-type: application/json" \
  -d '{"question":"How does Hono pick which router to use?"}'