From 3e75113ceeb7862eeb4f62494620c3a4409a2f84 Mon Sep 17 00:00:00 2001 From: Morpheus Date: Thu, 20 Aug 2026 20:34:56 +0000 Subject: [PATCH] Add overnight flux2klein smoke benchmark runner --- benchmarks/flux2klein/README.md | 38 +++ .../flux2klein/canonical-inputs/README.md | 5 + benchmarks/flux2klein/overnight-state.json | 10 + optimized/notes/process.md | 15 +- scripts/run-flux2klein-overnight-smoke.mjs | 286 ++++++++++++++++++ 5 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 benchmarks/flux2klein/README.md create mode 100644 benchmarks/flux2klein/canonical-inputs/README.md create mode 100644 benchmarks/flux2klein/overnight-state.json create mode 100644 scripts/run-flux2klein-overnight-smoke.mjs diff --git a/benchmarks/flux2klein/README.md b/benchmarks/flux2klein/README.md new file mode 100644 index 0000000..7dbbb67 --- /dev/null +++ b/benchmarks/flux2klein/README.md @@ -0,0 +1,38 @@ +# FLUX.2-klein Benchmarks + +This folder holds durable benchmark inputs, outputs, and state for low-cost `flux2klein` testing. + +## Current intent + +- Keep overnight tests cheap and reviewable. +- Prefer one or two single-image smoke tests over broad matrices. +- Log enough context to reproduce a result without trawling temporary folders. + +## Layout + +- `canonical-inputs/` + - durable reference inputs used for repeatable comparisons +- `runs/` + - dated run batches +- `overnight-state.json` + - handoff file between the `3am` optimisation pass and the `6am` review pass + +## Batch contents + +Each case folder should keep: + +- `input-1.*` +- `resolved-prompt.json` +- `run-record.json` +- the saved output image + +## Current nightly default + +- family: `flux2klein` +- workflow: `models/flux2klein/image-edit-9b-distilled/workflow.json` +- output policy: single image only +- test style: smoke test, not broad sweep + +## Review rule + +Do not call a workflow fully optimal from one benchmark image. Use these batches to decide whether a path is promising enough for follow-up. diff --git a/benchmarks/flux2klein/canonical-inputs/README.md b/benchmarks/flux2klein/canonical-inputs/README.md new file mode 100644 index 0000000..6351c95 --- /dev/null +++ b/benchmarks/flux2klein/canonical-inputs/README.md @@ -0,0 +1,5 @@ +# Canonical Inputs + +Put durable benchmark reference images here when a test input becomes part of the standing comparison set. + +For now, the overnight runner can also copy an external source image into each run case so the used input is still captured in-repo even before it graduates into this folder. diff --git a/benchmarks/flux2klein/overnight-state.json b/benchmarks/flux2klein/overnight-state.json new file mode 100644 index 0000000..f717821 --- /dev/null +++ b/benchmarks/flux2klein/overnight-state.json @@ -0,0 +1,10 @@ +{ + "date": null, + "testsSet": false, + "serverBusy": false, + "reason": "No overnight run has written state yet.", + "cases": [], + "reviewPending": false, + "optimizedPaths": [], + "verdict": null +} diff --git a/optimized/notes/process.md b/optimized/notes/process.md index 6f8f583..ad4cbb9 100644 --- a/optimized/notes/process.md +++ b/optimized/notes/process.md @@ -7,7 +7,7 @@ - Do not claim "optimal" when the evidence is still thin; "best current" is acceptable. - Prefer removing dead branches, duplicated wiring, and stale custom-node clutter. -## Current weekly review priorities +## Current review priorities 1. `flux2klein` 2. `zimage` @@ -26,7 +26,18 @@ - `optimized/` = maintained masters - `optimized/notes/` = accumulated reasoning and unresolved questions -## What to record after each Sunday review +## Overnight flow + +- `03:00 UK` + - optimize maintained workflows where the evidence is strong enough + - prefer low-cost `flux2klein` smoke tests first + - only submit a test if the ComfyUI queue is idle +- `06:00 UK` + - inspect overnight single-image outputs + - update benchmark notes and any affected maintained notes + - clear the overnight review-pending state + +## What to record after each review - Which optimized categories changed - Which source workflows influenced the update diff --git a/scripts/run-flux2klein-overnight-smoke.mjs b/scripts/run-flux2klein-overnight-smoke.mjs new file mode 100644 index 0000000..419e5e7 --- /dev/null +++ b/scripts/run-flux2klein-overnight-smoke.mjs @@ -0,0 +1,286 @@ +#!/usr/bin/env node + +import fs from 'fs'; +import path from 'path'; +import { execFileSync } from 'child_process'; + +const REPO_ROOT = '/home/node/.openclaw/workspace/work/comfyui-workflows'; +const DEFAULT_SERVER = process.env.COMFYUI_URL || 'http://192.168.1.202:8188'; +const DEFAULT_WORKFLOW = path.join(REPO_ROOT, 'models/flux2klein/image-edit-9b-distilled/workflow.json'); +const DEFAULT_INPUT = '/home/node/.openclaw/workspace/tmp/flux2klein-inputs/2026-08-13-pinup-ref.jpg'; +const DEFAULT_STATE_FILE = path.join(REPO_ROOT, 'benchmarks/flux2klein/overnight-state.json'); +const DEFAULT_RUNS_ROOT = path.join(REPO_ROOT, 'benchmarks/flux2klein/runs'); +const SUBMIT_SCRIPT = path.join(REPO_ROOT, 'scripts/submit-flux2klein-edit-test.mjs'); + +const DEFAULT_PROMPT = 'Restage the exact same woman as a clean full-body studio reference photo on a pure white seamless background. Face the camera directly in a neutral standing pose. Preserve identical facial features, platinum blonde hair, skin tone, gold corset dress, black thigh-high boots, proportions, and age. Even soft studio lighting, no props, no street, no buildings, no text.'; + +function printUsage() { + console.log(`Usage: + node scripts/run-flux2klein-overnight-smoke.mjs [options] + +Options: + --server ComfyUI base URL. Default: ${DEFAULT_SERVER} + --workflow Workflow path. Default: ${DEFAULT_WORKFLOW} + --image Input image path. Default: ${DEFAULT_INPUT} + --prompt Prompt override + --steps Default: 4 + --cfg Default: 1.0 + --width Default: 1536 + --height Default: 864 + --prefix Output filename prefix. Default: flux2klein-overnight-smoke + --batch-label Batch label appended to the run folder name + --state-file State file path. Default: ${DEFAULT_STATE_FILE} + --optimized-path Relative repo path touched by the optimisation pass. Repeatable. + --help Show this message +`); +} + +function parseArgs(argv) { + const options = { + server: DEFAULT_SERVER, + workflow: DEFAULT_WORKFLOW, + image: DEFAULT_INPUT, + prompt: DEFAULT_PROMPT, + steps: 4, + cfg: 1.0, + width: 1536, + height: 864, + prefix: 'flux2klein-overnight-smoke', + batchLabel: 'single-smoke', + stateFile: DEFAULT_STATE_FILE, + optimizedPaths: [], + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--help' || arg === '-h') { + printUsage(); + process.exit(0); + } + const next = argv[i + 1]; + const take = () => { + if (next === undefined) throw new Error(`Missing value for ${arg}`); + i += 1; + return next; + }; + switch (arg) { + case '--server': options.server = take(); break; + case '--workflow': options.workflow = take(); break; + case '--image': options.image = take(); break; + case '--prompt': options.prompt = take(); break; + case '--steps': options.steps = Number(take()); break; + case '--cfg': options.cfg = Number(take()); break; + case '--width': options.width = Number(take()); break; + case '--height': options.height = Number(take()); break; + case '--prefix': options.prefix = take(); break; + case '--batch-label': options.batchLabel = take(); break; + case '--state-file': options.stateFile = take(); break; + case '--optimized-path': options.optimizedPaths.push(take()); break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + return options; +} + +function makeSlug(value) { + return String(value || 'run') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80) || 'run'; +} + +function repoRelative(filePath) { + return path.relative(REPO_ROOT, filePath).replace(/\\/g, '/'); +} + +function ensureDir(filePath) { + fs.mkdirSync(filePath, { recursive: true }); +} + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, 'utf8')); +} + +function writeJson(filePath, data) { + ensureDir(path.dirname(filePath)); + fs.writeFileSync(filePath, `${JSON.stringify(data, null, 2)}\n`); +} + +async function fetchQueue(server) { + const base = server.replace(/\/$/, ''); + const endpoints = [`${base}/api/queue`, `${base}/queue`]; + let lastError = null; + for (const endpoint of endpoints) { + try { + const response = await fetch(endpoint); + if (!response.ok) { + lastError = new Error(`Queue check failed at ${endpoint} (${response.status})`); + continue; + } + return await response.json(); + } catch (error) { + lastError = error; + } + } + throw lastError || new Error('Queue check failed'); +} + +function summarizeBusy(queueData) { + const running = Array.isArray(queueData?.queue_running) ? queueData.queue_running.length : 0; + const pending = Array.isArray(queueData?.queue_pending) ? queueData.queue_pending.length : 0; + return { + running, + pending, + busy: running > 0 || pending > 0, + }; +} + +function writeState(options, patch) { + const previous = fs.existsSync(options.stateFile) ? readJson(options.stateFile) : {}; + const next = { + date: new Date().toISOString(), + testsSet: false, + serverBusy: false, + reason: '', + cases: [], + reviewPending: false, + optimizedPaths: options.optimizedPaths, + verdict: null, + ...previous, + ...patch, + }; + writeJson(options.stateFile, next); + return next; +} + +function copyInputImage(sourcePath, caseDir) { + const ext = path.extname(sourcePath) || '.bin'; + const destination = path.join(caseDir, `input-1${ext}`); + fs.copyFileSync(sourcePath, destination); + return destination; +} + +function runSubmitScript(options, caseDir) { + const stdout = execFileSync('node', [ + SUBMIT_SCRIPT, + '--workflow', options.workflow, + '--image', options.image, + '--target', 'single', + '--prompt', options.prompt, + '--steps', String(options.steps), + '--cfg', String(options.cfg), + '--width', String(options.width), + '--height', String(options.height), + '--prefix', options.prefix, + '--out-dir', caseDir, + '--server', options.server, + ], { + cwd: REPO_ROOT, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse(stdout); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + options.workflow = path.resolve(options.workflow); + options.image = path.resolve(options.image); + options.stateFile = path.resolve(options.stateFile); + + if (!fs.existsSync(options.workflow)) { + const state = writeState(options, { + reason: `Workflow not found: ${repoRelative(options.workflow)}`, + }); + console.log(JSON.stringify({ ok: false, state }, null, 2)); + return; + } + + if (!fs.existsSync(options.image)) { + const state = writeState(options, { + reason: `Input image not found: ${options.image}`, + }); + console.log(JSON.stringify({ ok: false, state }, null, 2)); + return; + } + + let queueData; + try { + queueData = await fetchQueue(options.server); + } catch (error) { + const state = writeState(options, { + reason: `Could not verify queue state: ${error.message}`, + }); + console.log(JSON.stringify({ ok: false, state }, null, 2)); + return; + } + + const queue = summarizeBusy(queueData); + if (queue.busy) { + const state = writeState(options, { + serverBusy: true, + reason: `Skipped smoke test because ComfyUI was busy (running=${queue.running}, pending=${queue.pending}).`, + cases: [], + reviewPending: false, + }); + console.log(JSON.stringify({ ok: true, skipped: true, queue, state }, null, 2)); + return; + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const batchDir = path.join(DEFAULT_RUNS_ROOT, `${timestamp}-${makeSlug(options.batchLabel)}`); + const caseDir = path.join(batchDir, 'cases', 'case01'); + ensureDir(caseDir); + const copiedInput = copyInputImage(options.image, caseDir); + + const result = runSubmitScript(options, caseDir); + const runRecordPath = path.join(caseDir, 'run-record.json'); + const promptPath = path.join(caseDir, 'resolved-prompt.json'); + const runRecord = readJson(runRecordPath); + const outputFile = Array.isArray(runRecord.files) && runRecord.files.length + ? runRecord.files[0].localPath + : null; + + const manifest = { + workflow: repoRelative(options.workflow), + input: repoRelative(copiedInput), + output: outputFile ? repoRelative(outputFile) : null, + runRecord: repoRelative(runRecordPath), + resolvedPrompt: repoRelative(promptPath), + promptId: result.promptId || runRecord.promptId || null, + status: result.status || runRecord?.status?.status_str || 'unknown', + overrides: { + steps: options.steps, + cfg: options.cfg, + width: options.width, + height: options.height, + prefix: options.prefix, + }, + }; + writeJson(path.join(batchDir, 'manifest.json'), manifest); + + const state = writeState(options, { + testsSet: true, + serverBusy: false, + reason: 'Overnight flux2klein smoke test submitted successfully.', + cases: [manifest], + reviewPending: true, + verdict: null, + }); + + console.log(JSON.stringify({ + ok: true, + queue, + batchDir: repoRelative(batchDir), + caseDir: repoRelative(caseDir), + state, + }, null, 2)); +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +});