Fine-tune with a Hosted API

Build the create-job request

// Build the create-job request from your config, the way the repo's createJob() does.
const config = { baseModel: "gpt-4o-mini-2024-07-18", suffix: "tickets-v2", nEpochs: 3, seed: 42 };
const datasetSha256 = "9c1e0d7a52b4f6e8a1c3d5f7b9e0a2c4d6f8b1a3c5e7d9f0b2a4c6e8d0f1a3b5";

const body = {
  model: config.baseModel,
  training_file: "file-tt-train-0001",   // from POST /v1/files
  validation_file: "file-tt-val-0001",
  suffix: config.suffix,                 // becomes part of the model name
  seed: config.seed,                     // same seed, same shuffling
  metadata: { dataset_sha256: datasetSha256.slice(0, 16) },
  method: { type: "supervised", supervised: { hyperparameters: { n_epochs: config.nEpochs } } },
};
console.log("POST /v1/fine_tuning/jobs");
console.log(JSON.stringify(body, null, 2));

Build the create-job request

using System.Text.Encodings.Web;
using System.Text.Json;

// Build the create-job request from your config, the way the repo's CreateJobAsync() does.
var config = new { BaseModel = "gpt-4o-mini-2024-07-18", Suffix = "tickets-v2", NEpochs = 3, Seed = 42 };
var datasetSha256 = "9c1e0d7a52b4f6e8a1c3d5f7b9e0a2c4d6f8b1a3c5e7d9f0b2a4c6e8d0f1a3b5";

var body = new Dictionary<string, object>
{
    ["model"] = config.BaseModel,
    ["training_file"] = "file-tt-train-0001",   // from POST /v1/files
    ["validation_file"] = "file-tt-val-0001",
    ["suffix"] = config.Suffix,                 // becomes part of the model name
    ["seed"] = config.Seed,                     // same seed, same shuffling
    ["metadata"] = new { dataset_sha256 = datasetSha256[..16] },
    ["method"] = new { type = "supervised", supervised = new { hyperparameters = new { n_epochs = config.NEpochs } } },
};
Console.WriteLine("POST /v1/fine_tuning/jobs");
Console.WriteLine(JsonSerializer.Serialize(body, new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }));

Poll a job until it ends

// Poll a job until it ends. These are the statuses the recorded job went through.
type Job = { status: string; fine_tuned_model: string | null };
const polls: Job[] = [
  { status: "validating_files", fine_tuned_model: null },
  { status: "queued", fine_tuned_model: null },
  { status: "running", fine_tuned_model: null },
  { status: "running", fine_tuned_model: null },
  { status: "succeeded", fine_tuned_model: "ft:gpt-4o-mini-2024-07-18:example-org:tickets-v2:Tt9xQ2b1" },
];

function nextAction(job: Job): { action: "wait" | "done" | "stop"; waitMs: number } {
  if (job.status === "validating_files") return { action: "wait", waitMs: 15000 };
  if (job.status === "queued") return { action: "wait", waitMs: 30000 };
  if (job.status === "running") return { action: "wait", waitMs: 60000 };
  if (job.status === "succeeded" && job.fine_tuned_model) return { action: "done", waitMs: 0 };
  return { action: "stop", waitMs: 0 };
}

let waited = 0;
for (const job of polls) {
  const next = nextAction(job);
  console.log(`${job.status.padEnd(17)} -> ${next.action}${next.waitMs ? ` (poll again in ${next.waitMs / 1000} s)` : ""}`);
  waited += next.waitMs;
  if (next.action !== "wait") {
    console.log(next.action === "done" ? `model: ${job.fine_tuned_model}` : "job ended without a model");
    break;
  }
}
console.log(`time spent waiting between polls: ${waited / 1000} s`);

Poll a job until it ends

// Poll a job until it ends. These are the statuses the recorded job went through.
var polls = new List<Job>
{
    new("validating_files", null),
    new("queued", null),
    new("running", null),
    new("running", null),
    new("succeeded", "ft:gpt-4o-mini-2024-07-18:example-org:tickets-v2:Tt9xQ2b1"),
};

(string Action, int WaitMs) NextAction(Job job)
{
    if (job.Status == "validating_files") return ("wait", 15000);
    if (job.Status == "queued") return ("wait", 30000);
    if (job.Status == "running") return ("wait", 60000);
    if (job.Status == "succeeded" && job.FineTunedModel != null) return ("done", 0);
    return ("stop", 0);
}

var waited = 0;
foreach (var job in polls)
{
    var next = NextAction(job);
    Console.WriteLine($"{job.Status,-17} -> {next.Action}{(next.WaitMs > 0 ? $" (poll again in {next.WaitMs / 1000} s)" : "")}");
    waited += next.WaitMs;
    if (next.Action != "wait")
    {
        Console.WriteLine(next.Action == "done" ? $"model: {job.FineTunedModel}" : "job ended without a model");
        break;
    }
}
Console.WriteLine($"time spent waiting between polls: {waited / 1000} s");

record Job(string Status, string? FineTunedModel);

Find the best point in the job's events

// The events endpoint lists newest first. Sort by step, then find the best point.
const events = [
  { step: 105, train: 0.364, valid: 0.544 }, { step: 100, train: 0.369, valid: 0.527 },
  { step: 90, train: 0.377, valid: 0.497 }, { step: 80, train: 0.392, valid: 0.472 },
  { step: 70, train: 0.416, valid: 0.456 }, { step: 60, train: 0.455, valid: 0.495 },
  { step: 50, train: 0.515, valid: 0.555 }, { step: 40, train: 0.61, valid: 0.65 },
];

const curve = [...events].sort((a, b) => a.step - b.step);
let best = curve[0];
for (const p of curve) if (p.valid < best.valid) best = p;
for (const p of curve) console.log(`step ${String(p.step).padStart(3)}  train ${p.train.toFixed(3)}  valid ${p.valid.toFixed(3)}${p === best ? "  <- lowest" : ""}`);

const last = curve[curve.length - 1];
const totalSteps = 105, epochs = 3;
const bestEpoch = Math.ceil((best.step / totalSteps) * epochs);
console.log(`validation loss rose from ${best.valid} to ${last.valid} after step ${best.step}`);
console.log(`next run: n_epochs ${bestEpoch} instead of ${epochs}, then compare both on the eval set`);

Find the best point in the job's events

// The events endpoint lists newest first. Sort by step, then find the best point.
var events = new List<Point>
{
    new(105, 0.364, 0.544), new(100, 0.369, 0.527),
    new(90, 0.377, 0.497), new(80, 0.392, 0.472),
    new(70, 0.416, 0.456), new(60, 0.455, 0.495),
    new(50, 0.515, 0.555), new(40, 0.61, 0.65),
};

var curve = events.OrderBy(p => p.Step).ToList();
var best = curve[0];
foreach (var p in curve) if (p.Valid < best.Valid) best = p;
foreach (var p in curve) Console.WriteLine($"step {p.Step,3}  train {p.Train:F3}  valid {p.Valid:F3}{(p == best ? "  <- lowest" : "")}");

var last = curve[^1];
int totalSteps = 105, epochs = 3;
var bestEpoch = (int)Math.Ceiling((double)best.Step / totalSteps * epochs);
Console.WriteLine($"validation loss rose from {best.Valid} to {last.Valid} after step {best.Step}");
Console.WriteLine($"next run: n_epochs {bestEpoch} instead of {epochs}, then compare both on the eval set");

record Point(int Step, double Train, double Valid);

Estimate the trained tokens

// Roughly how many tokens will the job train on? Count before you upload.
// Hosted providers bill training by tokens processed: the file's tokens × epochs.
const trainingLines = [
  { chars: 412 }, { chars: 388 }, { chars: 455 }, { chars: 401 }, { chars: 1960 },
];
const linesInFile = 560;            // wc -l train.jsonl
const epochs = 3;
const pricePerMillion = 3.0;        // illustrative only: use your provider's current price

const avgChars = trainingLines.reduce((s, l) => s + l.chars, 0) / trainingLines.length;
const tokensPerLine = Math.ceil(avgChars / 4);       // about 4 characters per token in English
const fileTokens = tokensPerLine * linesInFile;
const trained = fileTokens * epochs;
console.log(`about ${tokensPerLine} tokens per line, ${fileTokens.toLocaleString("en-US")} in the file`);
console.log(`× ${epochs} epochs = ${trained.toLocaleString("en-US")} trained tokens`);
console.log(`at $${pricePerMillion.toFixed(2)} per million: about $${((trained / 1e6) * pricePerMillion).toFixed(2)} per run`);

Estimate the trained tokens

// Roughly how many tokens will the job train on? Count before you upload.
// Hosted providers bill training by tokens processed: the file's tokens × epochs.
int[] lineChars = [412, 388, 455, 401, 1960];
var linesInFile = 560;            // wc -l train.jsonl
var epochs = 3;
var pricePerMillion = 3.0;        // illustrative only: use your provider's current price

var avgChars = lineChars.Average();
var tokensPerLine = (int)Math.Ceiling(avgChars / 4);   // about 4 characters per token in English
var fileTokens = tokensPerLine * linesInFile;
var trained = fileTokens * epochs;
Console.WriteLine($"about {tokensPerLine} tokens per line, {fileTokens:N0} in the file");
Console.WriteLine($"× {epochs} epochs = {trained:N0} trained tokens");
Console.WriteLine($"at ${pricePerMillion:F2} per million: about ${trained / 1e6 * pricePerMillion:F2} per run");