{ "version": 1, "read_on": "Oct 17, 2026", "blocks": { "screen": { "name": "Screen", "tech": "UI", "plain": "What a person sees and clicks: a web page, an app tab, a report.", "whatif": "What does the person see while it loads, or when something fails?" }, "api": { "name": "Door", "tech": "API", "plain": "A fixed way for one program to ask another for something, often with a key that says who is asking. MCP is a door made for AI apps.", "whatif": "What if the other side is slow, down, or changes its answer format?" }, "job": { "name": "Timer", "tech": "scheduled job", "plain": "Code that runs by itself at set times, with nobody clicking.", "whatif": "What if a run is missed, or two runs overlap?" }, "workflow": { "name": "Steps that survive", "tech": "workflow / queue", "plain": "Work cut into steps that are written down as each finishes, so a crash or restart picks up where it stopped.", "whatif": "What if the machine restarts halfway through?" }, "store": { "name": "Memory", "tech": "storage", "plain": "Where data stays between runs: files (JSON, CSV), a database (SQLite, Postgres), or git history.", "whatif": "What if two writers change it at once, or it gets ten times bigger?" }, "cache": { "name": "Saved copy", "tech": "cache", "plain": "A copy kept so slow or costly work isn't redone. It can go stale.", "whatif": "How old can the copy be before it is wrong?" }, "llm": { "name": "AI model call", "tech": "LLM", "plain": "Send a prompt, get text or structured data back. Paid per token, and it can be wrong while sounding sure.", "whatif": "What if the model is wrong, slow, or its price doubles?" }, "meaning": { "name": "Search by meaning", "tech": "embeddings", "plain": "Text turned into lists of numbers so similar meanings sit close together; then find the nearest ones.", "whatif": "How do you know the nearest match is a good match?" }, "stats": { "name": "Numbers model", "tech": "statistics", "plain": "Math that turns data into a number you can act on: a forecast, a probability, a waiting time, a cost.", "whatif": "What does it assume, and what happens when that is not true?" }, "check": { "name": "Grader", "tech": "tests / evals", "plain": "Known answers the system must match before anything ships. For AI: how often it is right, and whether its confidence can be trusted.", "whatif": "How do you know it is right, and right compared with what baseline?" }, "browser": { "name": "Robot hands", "tech": "browser automation", "plain": "Code driving a real browser: open pages, click, read forms (Playwright, Stagehand).", "whatif": "What if the page changes, or the site does not allow automated reading?" }, "ship": { "name": "Shipping", "tech": "CI / deploy", "plain": "Every change runs the tests, then is built and put live, ideally by itself.", "whatif": "How do you undo a bad release?" }, "watch": { "name": "Camera", "tech": "logs / analytics", "plain": "Records of what ran, what failed and who visited, so problems show up before someone complains.", "whatif": "How would you know it broke before a user tells you?" } }, "tracks": { "tools": { "label": "Our tools", "note": "Live on this site. Sheets drawn from the real code." }, "oss": { "label": "Open source projects", "note": "Read from each project's docs and code." } }, "status": { "live": "Live", "oss": "Open source" }, "sheets": [ { "id": "modelshelf", "no": "T-01", "track": "tools", "status": "live", "name": "ModelShelf", "link": "/products/modelshelf/", "sub": "AI model prices, every change to them, and a 90 day outlook", "breath": "A daily batch job: the public price list of AI models in; every price change since 2023, the chance of a price cut within 90 days, and a page to look up and compare models out.", "lanes": { "in": [ { "n": "LiteLLM's public price catalogue (MIT)", "b": "api", "go": "litellm" }, { "n": "LMArena quality ratings (CC BY 4.0)", "b": "api" } ], "work": [ { "n": "A daily job at 06:17 UTC", "b": "ship", "note": "A hosted CI runner installs the code, runs the tests, runs the pipeline and publishes the page." }, { "n": "Tests first", "b": "check", "note": "If a test fails, the page is not rebuilt that day." }, { "n": "Find every price change", "b": "job", "note": "Each change is one event: a model, a field, the old and new value, the day." }, { "n": "Survival analysis in SQL: the chance of a cut within 90 days", "b": "stats" } ], "keep": [ { "n": "Event and estimate tables (CSV)", "b": "store" }, { "n": "Version history: every day's catalogue", "b": "store", "note": "Each day's copy is kept, so any past price can be checked." } ], "out": [ { "n": "The page: Ask, Models, Price history, About", "b": "screen" }, { "n": "A guess at the kind of work in a prompt (69.7% right)", "b": "stats", "note": "Measured on Arena prompts held out from the fit." }, { "n": "Daily prices for BillCheck", "b": "cache", "go": "billcheck" } ] }, "when": "Daily at 06:17 UTC. This page picks up each new day within a few hours.", "decisions": [ { "t": "No server", "chose": "a static page rebuilt daily", "over": "a live API server", "because": "prices change at most daily, so daily is enough; free, and nothing to secure", "cost": "the data can be up to a day old" }, { "t": "Survival analysis", "chose": "time until a price cut, with censoring", "over": "an average time between cuts", "because": "models with no cut yet count as 'not yet', not as 'never'", "cost": "harder to explain in one sentence" }, { "t": "Open licences only", "chose": "LiteLLM (MIT) and Arena (CC BY)", "over": "commercial benchmark data", "because": "the page is free to read and reuse, so its sources must be too", "cost": "no data on how many tokens a task takes" } ], "whatif": [ { "q": "The daily run is skipped?", "a": "It ran 8 hours late once, and hosted schedules switch themselves off when a project goes quiet. The job turns its own schedule back on at every run." }, { "q": "The catalogue has a wrong price?", "a": "The page shows the source's number, and big jumps are flagged instead of trusted blindly." }, { "q": "Ten times more models?", "a": "The page's data file grows. Split it by view, or serve a lean daily file that other tools can share." } ], "talk": [ { "q": "Explain survival analysis in plain words.", "k": "Waiting time", "a": "It measures how long until something happens, here a price cut. Some models haven't had a cut yet. They are not dropped; they count as 'not yet'. That is called censoring." }, { "q": "Why no backend?", "k": "Daily is enough", "a": "Prices change at most once a day, so the page is rebuilt once a day. There is no server to pay for or secure." }, { "q": "How do you know the numbers are right?", "k": "Tests first", "a": "Every daily run starts with the tests. If one fails, the page is not rebuilt. The method has a worked example checked by hand." } ] }, { "id": "billcheck", "no": "T-02", "track": "tools", "status": "live", "name": "BillCheck", "link": "/products/billcheck/", "sub": "Is your AI bill counted right? With a chargeback file for finance", "breath": "A check of a team's AI bill: LiteLLM's spend logs in; every request priced again at the price of its own day; then what to fix in plain words, who spent what, when budgets run out, cheaper options, and a FOCUS file finance can charge each team with. It runs in your browser.", "lanes": { "in": [ { "n": "LiteLLM spend logs: an export or the proxy's API", "b": "store", "note": "The page gives a SQL query that only reads.", "go": "litellm" }, { "n": "Budget tables (optional)", "b": "store" }, { "n": "LiteLLM's public price list", "b": "cache", "note": "Token prices only (MIT)." }, { "n": "Daily price history from ModelShelf", "b": "cache", "note": "The price on any past day.", "go": "modelshelf" } ], "work": [ { "n": "All the checks run in this page", "b": "screen", "note": "Nothing is uploaded; the logs never leave the browser." }, { "n": "Reprice each request at that day's price", "b": "stats" }, { "n": "One verdict per row: right price, old price, logged $0, own rates, upstream's cost, unexplained", "b": "check" }, { "n": "Recover a deployment's own rates from its logs", "b": "stats", "note": "A least squares fit on the logged costs; in the example, a contract price 20% below list came out exactly." }, { "n": "Plain findings: what happened, why, what to do", "b": "check" }, { "n": "Budget pace and cheaper model math", "b": "stats" } ], "keep": [ { "n": "Nothing: the page keeps no logs", "b": "store" } ], "out": [ { "n": "Report: What to fix, Who spent what, Budgets, Cheaper options, For finance", "b": "screen" }, { "n": "A chargeback file in FOCUS 1.4", "b": "api", "note": "One row per day, team, key, model and token type, at the right price, with LiteLLM's own number next to it." }, { "n": "A test set: 3,882 requests through a real LiteLLM proxy", "b": "check", "note": "A price list two months old, a stand in model server and invented traffic; every row's right answer was planned in advance." } ] }, "when": "Whenever a visitor runs it. A weekly check fits most teams.", "decisions": [ { "t": "In the browser", "chose": "run in the visitor's browser", "over": "upload to a server", "because": "spend logs name a company's teams and keys, and there is nothing to host", "cost": "very large logs are slow in a browser" }, { "t": "Reprice by date", "chose": "the price as it stood that day", "over": "today's price", "because": "a bill is right or wrong at its own day's price, and ModelShelf already keeps that history", "cost": "on the day a price changed, both prices have to count" }, { "t": "Use the proxy's own records too", "chose": "repricing plus the price entry the proxy logged", "over": "repricing alone", "because": "repricing alone was 89% right on the test set; with the records, 100%; trusting the proxy, 42%", "cost": "older proxies don't write that record (95% then)" }, { "t": "FOCUS for finance", "chose": "FOCUS 1.4, as an internal chargeback", "over": "our own CSV columns", "because": "finance and FinOps tools already read FOCUS; teams map to billing accounts, keys to sub accounts", "cost": "strict column rules; LiteLLM's own FOCUS export copies its logged cost, so a company uses one of the two, not both" } ], "whatif": [ { "q": "A million rows?", "a": "About 4,000 rows a second in a browser: a few weeks of a busy proxy is fine, a year is not." }, { "q": "A model isn't in the catalogue?", "a": "It is marked unpriced, never guessed. A $0 row with no price anywhere is flagged, because no budget can see it." }, { "q": "A team pays a negotiated price?", "a": "The rates that reproduce every logged cost are recovered from the logs and kept as the bill. In the FOCUS file they show as the contracted price." } ], "talk": [ { "q": "What problem does it solve?", "k": "Bills drift", "a": "LiteLLM logs a cost for every request from the price list it loaded. When that list is old, the cost is wrong, and $0 rows hide from budgets. BillCheck prices every row again at that day's price and says what to fix." }, { "q": "Why would finance care?", "k": "Chargeback", "a": "Companies charge each team for the AI it used. BillCheck writes the right amounts in FOCUS, the format finance tools already read. In the example, one team LiteLLM charged $19.82 had used $41.57." }, { "q": "What did the error analysis find?", "k": "Fake discount", "a": "Without the proxy's logged price entry, rows at an old price also fit one set of rates, so they looked like a contract discount. Trying older prices first fixed it: from 67% to 95% of requests right." }, { "q": "Did it find anything in LiteLLM itself?", "k": "Streams", "a": "Yes: a streamed request can be billed at the provider's own number while the same plain request is billed at the price list. A fix with tests is ready for LiteLLM's maintainers." } ] }, { "id": "calibration", "no": "T-03", "track": "tools", "status": "live", "name": "Calibration Check", "link": "/products/calibration-check/", "sub": "Is your AI's 90% really 90%?", "breath": "A page that tests whether an AI's confidence can be trusted: labeled cases and probabilities in; calibration error, a tested fix, the cheapest cutoff and the worst mistakes out.", "lanes": { "in": [ { "n": "A CSV: right answers and the model's probabilities", "b": "store" }, { "n": "Demo: Laya on 1,795 Amazon drinkware reviews", "b": "check", "note": "Amazon Reviews 2023 (McAuley Lab, UC San Diego). Laya is Apache 2.0." } ], "work": [ { "n": "Split: half to fit the fixes, half to judge them", "b": "check" }, { "n": "Calibration error", "b": "stats" }, { "n": "Two fixes compared: temperature and isotonic", "b": "stats" }, { "n": "Cost of every cutoff at your prices", "b": "stats" } ], "keep": [ { "n": "Nothing: the file never leaves the page", "b": "store" } ], "out": [ { "n": "Reliability chart and the numbers", "b": "screen" }, { "n": "The most confident mistakes to read", "b": "check" } ] }, "when": "Whenever a visitor runs it.", "decisions": [ { "t": "Judge on held out cases", "chose": "fit on one half, judge on the other", "over": "fit and judge on the same cases", "because": "otherwise the fix looks better than it is", "cost": "needs enough cases" }, { "t": "Two fixes", "chose": "temperature (one number) and isotonic (a step curve)", "over": "one method", "because": "temperature is safe with little data; isotonic bends more but needs more data", "cost": "two results to explain" }, { "t": "In the browser", "chose": "no upload", "over": "a server", "because": "people's labeled data stays with them", "cost": "file size is limited by the browser" } ], "whatif": [ { "q": "Only 50 labeled cases?", "a": "Temperature still works; isotonic will overfit. The page shows both, so the gap is visible." }, { "q": "Three or more classes?", "a": "Check each class against the rest, or check the top answer's confidence." }, { "q": "Right but always unsure?", "a": "That is under confidence. The fix pushes the probabilities outward, and the best cutoff moves." } ], "talk": [ { "q": "What is calibration?", "k": "90 means 90", "a": "When a model says it is 90% sure, it should be right about 90 times out of 100. Calibration checks that." }, { "q": "How do you fix an overconfident model?", "k": "Temperature", "a": "Divide its scores by one number before turning them into probabilities. A number above 1 softens them. Pick it on one half of the data and check it on the other half." }, { "q": "Why does it matter in a business?", "k": "Cutoffs cost money", "a": "If cases below 90% go to a person, the 90 has to be real. Otherwise you pay for reviews you don't need, or miss the ones you do." } ] }, { "id": "litellm", "no": "O-01", "track": "oss", "status": "oss", "name": "LiteLLM", "sub": "One gateway in front of 100+ AI model providers", "license": "MIT (outside its enterprise folder)", "home": "https://github.com/BerriAI/litellm", "docs": "https://docs.litellm.ai", "breath": "A gateway between a company's apps and the AI providers: every app speaks one format (OpenAI's); LiteLLM checks the key and the budget, picks a deployment, translates the request for that provider, and writes down what it cost.", "lanes": { "in": [ { "n": "Apps call it like OpenAI, with a key", "b": "api", "note": "For example /chat/completions on port 4000; a virtual key in the Authorization header says which key, user or team is asking." }, { "n": "config.yaml: model names to deployments", "b": "store", "note": "One model name can stand for several deployments: other keys, regions or providers." }, { "n": "A public price list, read at start", "b": "cache", "note": "model_prices_and_context_window.json from the repo, with a copy inside the package; an admin can reload it." } ], "work": [ { "n": "Check the key and its budget", "b": "check", "note": "From the in memory or Redis cache first; the database only on a miss." }, { "n": "Rate limits for the server, key, user and team", "b": "stats", "note": "Requests and tokens per minute; with Redis, several copies of the proxy share the counters." }, { "n": "Router: pick a deployment, retry, fall back", "b": "api", "note": "Retries stay inside the model group; fallbacks move to the next group. The default pick is a simple shuffle." }, { "n": "Translate to the provider's format and back", "b": "api", "note": "OpenAI's format in and out, for 100+ providers." } ], "keep": [ { "n": "Postgres: keys, teams, budgets, spend logs", "b": "store", "note": "Prisma tables such as LiteLLM_VerificationToken and LiteLLM_SpendLogs." }, { "n": "Redis, optional: caches, counters, a spend buffer", "b": "cache" } ], "out": [ { "n": "The answer, in OpenAI's shape", "b": "screen" }, { "n": "Spend written after the answer, in batches", "b": "store", "note": "Every 10 to 15 seconds by default, so no database write sits in the request path." }, { "n": "Logs to tools such as Langfuse and Datadog", "b": "watch" }, { "n": "An admin screen for keys, teams and spend", "b": "screen" } ] }, "when": "On every request. Spend reaches the database 10 to 15 seconds later. The price list is read at start and reloaded only when an admin asks or sets a schedule.", "decisions": [ { "t": "One API shape for every provider", "chose": "OpenAI's request and response format", "over": "each provider's own SDK", "because": "an app changes providers by changing a model name, not its code", "cost": "what one provider has and OpenAI's shape lacks needs extra parameters" }, { "t": "Keep the database out of the request", "chose": "a cached key check, and spend written after the answer in batches", "over": "reading and writing the database on every call", "because": "a gateway sits in front of every call, so it must not add waiting time", "cost": "spend totals in the database lag 10 to 15 seconds; budget checks use live counters, shared across copies only with Redis" }, { "t": "Prices as a public file", "chose": "one JSON file of model prices in the repo", "over": "a pricing service or each provider's billing API", "because": "anyone can read it, fix it and reuse it", "cost": "a proxy keeps the prices it loaded until someone reloads them, so costs can drift" }, { "t": "Redis only when you scale out", "chose": "memory by default, Redis when several copies run", "over": "Redis always", "because": "one copy needs nothing extra", "cost": "without Redis, limits and caches are per copy" } ], "whatif": [ { "q": "A provider goes down?", "a": "The router asks the same model group again, which may pick another deployment, then falls back to the next model group in the config." }, { "q": "Ten copies behind a load balancer?", "a": "Limits and caches move to Redis so the copies agree, and spend updates can be buffered there before the database write." }, { "q": "A price changes while it runs?", "a": "The proxy keeps the price it loaded, so costs drift until an admin reloads the list. BillCheck exists to catch that drift." } ], "talk": [ { "q": "What is an AI gateway?", "k": "One door", "a": "A server every app sends its AI calls through. It holds the keys, picks the provider, enforces budgets and logs every call." }, { "q": "Retry or fallback?", "k": "Same group or next", "a": "A retry asks the same model group again, which may pick another deployment. A fallback switches to a different model group when the first keeps failing." }, { "q": "How does it know what a call cost?", "k": "Price list", "a": "Tokens in and out, times the prices in its price file. When that file is old, the cost is wrong." } ], "sources": [ [ "LiteLLM docs: Life of a Request", "https://docs.litellm.ai/docs/proxy/architecture" ], [ "LiteLLM on GitHub", "https://github.com/BerriAI/litellm" ] ], "read": "Read on Oct 17, 2026 from the docs and the code of version 1.105.1." }, { "id": "fastapi", "no": "O-02", "track": "oss", "status": "oss", "name": "FastAPI", "sub": "A Python web API framework built on type hints", "license": "MIT", "home": "https://github.com/fastapi/fastapi", "docs": "https://fastapi.tiangolo.com", "breath": "A way to write a web API in Python: you write ordinary functions with type hints, and FastAPI turns each one into an endpoint that checks its input, sends JSON back, and documents itself.", "lanes": { "in": [ { "n": "An HTTP request, received by an ASGI server such as Uvicorn", "b": "api", "note": "The server is a separate program; FastAPI is the framework it runs." }, { "n": "Your function, with type hints", "b": "store", "note": "The hints say what each input must be. One declaration drives the checks, the editor's help and the docs." } ], "work": [ { "n": "Starlette: routing, requests, responses, WebSockets", "b": "api", "note": "The FastAPI class is a subclass of Starlette's." }, { "n": "Pydantic checks and converts the data", "b": "check", "note": "A wrong type is refused with a 422 and the reason, before your code runs." }, { "n": "Dependencies run first (Depends)", "b": "job", "note": "Shared steps such as a database session or the current user, written once." }, { "n": "async def on the event loop; plain def in a thread pool", "b": "workflow", "note": "So a function that waits on a disk or a network does not stop the server." } ], "keep": [ { "n": "Nothing by itself: you bring the database", "b": "store" } ], "out": [ { "n": "JSON responses", "b": "screen" }, { "n": "An OpenAPI schema, generated from the code", "b": "api", "note": "The open standard for describing APIs; client generators and test tools read it." }, { "n": "Interactive docs at /docs and /redoc", "b": "screen", "note": "Swagger UI and ReDoc, built in." } ] }, "when": "On every request, while a server such as Uvicorn runs it.", "decisions": [ { "t": "Build on Starlette", "chose": "Starlette for the web layer", "over": "writing its own", "because": "a fast, tested ASGI toolkit already handled routing, requests and WebSockets", "cost": "FastAPI moves with Starlette's releases" }, { "t": "Type hints as the one source", "chose": "standard Python type hints, read by Pydantic", "over": "separate schema classes, or YAML in docstrings", "because": "one declaration gives the checks, the editor help and the docs, so they can't drift apart", "cost": "the code needs type hints wherever it matters" }, { "t": "An open standard for the docs", "chose": "OpenAPI, with Swagger UI and ReDoc", "over": "a custom schema", "because": "other tools already read OpenAPI", "cost": "what OpenAPI can't describe needs extra care" }, { "t": "Dependency injection", "chose": "Depends() for shared steps", "over": "copying setup into each endpoint", "because": "less repetition, and easy to swap in tests", "cost": "one more idea to learn" } ], "whatif": [ { "q": "A slow blocking database call inside an async def endpoint?", "a": "It holds up every other request on that worker. Use an async driver, or make the endpoint a plain def so it runs in the thread pool." }, { "q": "Ten times the traffic?", "a": "Run more worker processes or more copies behind a load balancer. FastAPI keeps no state between requests by itself, so copies are easy." }, { "q": "A client sends the wrong type?", "a": "Pydantic refuses it with a 422 that names the field and the reason, before your function runs." } ], "talk": [ { "q": "Why FastAPI over Flask?", "k": "Types do the work", "a": "Flask leaves checking and docs to add ons. FastAPI reads the type hints and does both, and it runs on ASGI, so async works out of the box." }, { "q": "What is ASGI?", "k": "The async plug", "a": "The standard plug between a Python web app and its server, made for async code and WebSockets. WSGI is the older one, made before async." }, { "q": "Where do the docs come from?", "k": "Generated", "a": "From the same type hints that check the data, so the docs match the code." } ], "sources": [ [ "FastAPI docs: Alternatives, Inspiration and Comparisons", "https://fastapi.tiangolo.com/alternatives/" ], [ "FastAPI docs: Concurrency and async / await", "https://fastapi.tiangolo.com/async/" ], [ "FastAPI on GitHub", "https://github.com/fastapi/fastapi" ] ], "read": "Read on Oct 17, 2026 from the FastAPI docs." }, { "id": "ollama", "no": "O-03", "track": "oss", "status": "oss", "name": "Ollama", "sub": "Run open AI models on your own computer", "license": "MIT", "home": "https://github.com/ollama/ollama", "docs": "https://docs.ollama.com", "breath": "A program that runs open models on your own machine: it downloads a model once, keeps it in memory while you use it, and answers on a local web address that apps call like a cloud API.", "lanes": { "in": [ { "n": "Apps and the ollama command call 127.0.0.1:11434", "b": "api", "note": "Its own API (/api/chat, /api/generate, /api/embed) and an OpenAI compatible one (/v1/chat/completions)." }, { "n": "Models pulled from the ollama.com library", "b": "api" }, { "n": "A Modelfile to make your own", "b": "store", "note": "FROM a base model, then PARAMETER, TEMPLATE and SYSTEM lines, much like a Dockerfile." } ], "work": [ { "n": "A Go server with a scheduler", "b": "workflow", "note": "It decides which model loads where, and holds requests while a model loads." }, { "n": "A runner process per loaded model", "b": "llm", "note": "llama.cpp's server for GGUF models; an MLX engine for safetensors models, on by default on Apple Silicon and also offered for NVIDIA GPUs." }, { "n": "The chat template turns messages into the model's prompt", "b": "job" } ], "keep": [ { "n": "Model layers, each named by its SHA-256", "b": "store", "note": "A manifest lists the layers, like a container image, so a shared layer is stored once." }, { "n": "A loaded model stays in memory 5 minutes", "b": "cache", "note": "keep_alive; it can be set longer, shorter or forever." } ], "out": [ { "n": "Tokens streamed back as they are made", "b": "screen" }, { "n": "Embeddings for search by meaning", "b": "meaning" }, { "n": "A 503 when too many wait for a model to load", "b": "watch", "note": "OLLAMA_MAX_QUEUE: 512 by default." } ] }, "when": "Whenever an app calls it. A model loads on first use and unloads after 5 idle minutes.", "decisions": [ { "t": "A local server, not a library", "chose": "one background server on the machine", "over": "a library each app loads", "because": "any app in any language can use it, and one server decides what stays in memory and shares loaded models", "cost": "one more program running; it listens only on the machine itself unless OLLAMA_HOST says otherwise" }, { "t": "Layers named by their hash", "chose": "content addressed layers and a manifest", "over": "one big file per model", "because": "downloads can be checked, and parts two models share are stored once", "cost": "the files on disk have no friendly names" }, { "t": "Keep it loaded for a while", "chose": "unload after 5 idle minutes", "over": "always loaded, or a fresh load per request", "because": "follow up requests are fast, and memory comes back when you stop", "cost": "the first request after a pause waits for the load" }, { "t": "Reuse proven engines", "chose": "llama.cpp and MLX", "over": "writing its own engine", "because": "wide hardware support and compressed (quantized) models", "cost": "keeping up with the engine's changes; Ollama pins a version and patches it" } ], "whatif": [ { "q": "Two apps want two big models on a laptop?", "a": "If both fit, both load. If not, the new request waits while an idle model is unloaded to make room." }, { "q": "Six hundred requests at once?", "a": "Each model works on one request at a time by default (OLLAMA_NUM_PARALLEL), and the rest wait their turn. The 512 limit, and its 503, cover requests waiting for a model to load." }, { "q": "You want it on your network?", "a": "Set OLLAMA_HOST to listen beyond the machine, and put your own access control in front: the local API has no sign in of its own." } ], "talk": [ { "q": "Why run a model locally?", "k": "Private and free", "a": "The text never leaves the machine and each call costs nothing. The trade is speed, and a smaller model than the best cloud ones." }, { "q": "What is quantization?", "k": "Smaller numbers", "a": "Storing a model's weights with fewer bits, so a big model fits in a laptop's memory, at a small cost in quality." }, { "q": "Why keep it loaded for 5 minutes?", "k": "Fast follow ups", "a": "Loading a model takes a while. Keeping it warm makes the next request fast, and it still frees the memory when you stop." } ], "sources": [ [ "Ollama docs: FAQ", "https://docs.ollama.com/faq" ], [ "Ollama on GitHub", "https://github.com/ollama/ollama" ] ], "read": "Read on Oct 17, 2026 from the docs and the code (commit d1ed3fd)." } ], "tips": [ { "term": "API", "plain": "A fixed way for one program to ask another for something." }, { "term": "gateway", "plain": "A server that every request passes through on its way to the real service; it can check keys, count and log." }, { "term": "proxy", "plain": "A program that sits between apps and a service and passes requests along, often checking or logging them." }, { "term": "tokens", "plain": "The pieces of text a model reads and writes. Providers charge per million tokens." }, { "term": "virtual key", "plain": "A key the gateway makes for a team or an app, so the real provider keys stay hidden." }, { "term": "Redis", "plain": "A very fast in memory store, often used for caches and shared counters." }, { "term": "Postgres", "plain": "A widely used open source database." }, { "term": "Prisma", "plain": "A tool that defines database tables in one schema file and talks to the database for the code." }, { "term": "batches", "plain": "Many small writes saved up and written together, which is cheaper than one at a time." }, { "term": "OpenAPI", "plain": "The open standard file that describes every endpoint of an API, its inputs and its outputs." }, { "term": "ASGI", "plain": "The standard plug between a Python web app and its server, made for async code." }, { "term": "type hints", "plain": "Notes in Python code that say what type a value is, like name: str." }, { "term": "Pydantic", "plain": "A Python library that checks and converts data using type hints." }, { "term": "Starlette", "plain": "A small, fast toolkit for Python web apps; FastAPI is built on it." }, { "term": "thread pool", "plain": "A few worker threads kept ready to run blocking functions without stopping the main loop." }, { "term": "event loop", "plain": "The part of an async program that switches between tasks while each one waits." }, { "term": "GGUF", "plain": "A single file format for model weights, made for llama.cpp and quantized models." }, { "term": "safetensors", "plain": "A safe, fast file format for model weights, common on Hugging Face." }, { "term": "MLX", "plain": "Apple's open source library for running models on Apple Silicon chips." }, { "term": "llama.cpp", "plain": "An open source C++ engine that runs language models on ordinary computers." }, { "term": "manifest", "plain": "A small file that lists the parts (layers) a download is made of." }, { "term": "SHA-256", "plain": "A fingerprint of a file: change one byte and the fingerprint changes." }, { "term": "quantized", "plain": "Stored with fewer bits per number, so the model is smaller and faster, with a small loss in quality." }, { "term": "survival analysis", "plain": "Statistics for how long until something happens, counting cases where it has not happened yet." }, { "term": "censoring", "plain": "Keeping cases where the event has not happened yet, as 'not yet' instead of dropping them." }, { "term": "calibration", "plain": "Whether a model's confidence matches how often it is right." }, { "term": "isotonic", "plain": "A fix that bends the probabilities with a step curve that only goes up." }, { "term": "FOCUS", "plain": "The FinOps Foundation's open format for billing data, read by finance and cloud cost tools." }, { "term": "chargeback", "plain": "Charging each team for what it used." }, { "term": "least squares", "plain": "Fitting numbers so the squared differences from the data are as small as possible." }, { "term": "CI runner", "plain": "A machine that runs the tests and builds automatically, for example on every change or every day." }, { "term": "held out", "plain": "Kept aside while fitting, so the check is on cases the fit never saw." }, { "term": "LMArena", "plain": "A public site where people compare two models' answers blind; the votes become ratings." }, { "term": "Uvicorn", "plain": "A fast ASGI server for Python apps." }, { "term": "Swagger UI", "plain": "A web page that lists an API's endpoints and lets you try them, built from its OpenAPI file." }, { "term": "422", "plain": "The HTTP code for 'I understood the request, but the data in it is not valid'." }, { "term": "503", "plain": "The HTTP code for 'busy or down right now, try again later'." }, { "term": "Modelfile", "plain": "Ollama's recipe file for building a model: the base model, its settings and its prompt template." } ], "frames": [ { "t": "The C4 model", "by": "Simon Brown, c4model.com", "url": "https://c4model.com", "p": [ "Four zoom levels for drawing any software system: Context (the system and who uses it), Containers (the apps, databases and services), Components (the parts inside one app), and Code.", "Each sheet here is the first two levels. Design interviews mostly live there too." ] }, { "t": "Architecture Decision Records", "by": "Michael Nygard, 2011; adr.github.io", "url": "https://adr.github.io", "p": [ "One short note per big choice: the situation, what was chosen, and what it costs. The Decisions column on every sheet uses this shape.", "'Why did you choose X?' is the most common design question, and this is the shape of a good answer." ] }, { "t": "The parts of an AI product", "by": "Chip Huyen, Building A Generative AI Platform (2024)", "url": "https://huyenchip.com/2024/07/25/genai-platform.html", "p": [ "Most AI products are the same parts around a model: context (search, tools), guardrails on what goes in and out, a router and gateway that pick the model and log the cost, a cache, evaluation, and monitoring.", "LiteLLM is a gateway; BillCheck checks the gateway's bill; Calibration Check is evaluation." ] }, { "t": "A design interview, in order", "by": "The common format", "url": "", "p": [ "Ask what it must do and for whom. Name the doors (the API). Say what data it keeps and where. Draw the boxes and arrows. Say what breaks at ten times the load, and the trade offs.", "The What if column on each sheet is practice for the last step." ] } ], "five": [ [ "Who is it for, and what one job does it do?", "One sentence: the In one breath line." ], [ "What comes in, and what goes out?", "The first and last lanes." ], [ "Which blocks does it use?", "The boxes in the middle lanes." ], [ "Follow one item from start to end.", "One request, one price, one file." ], [ "What did they choose, what did it cost, and what breaks at ten times the size?", "The Decisions and What if columns." ] ] }