Add portal browser testing and QA issue automation
Test, build, and deploy ComfyUI Jobs / test-build (push) Successful in 4m41s
Test, build, and deploy ComfyUI Jobs / deploy (push) Successful in 14s
Test, build, and deploy ComfyUI Jobs / browser-test (push) Failing after 7m5s
Test, build, and deploy ComfyUI Jobs / qa-gate (push) Failing after 4s

This commit is contained in:
2026-10-26 14:16:26 +00:00
parent 362f2c9d56
commit 96be6412a8
6 changed files with 285 additions and 1 deletions
+76
View File
@@ -130,3 +130,79 @@ jobs:
"
for attempt in 1 2 3 4 5 6; do curl --fail --silent --show-error --max-time 15 "$HEALTH_URL/api/health" && exit 0 || sleep 5; done
exit 1
browser-test:
if: ${{ gitea.event_name == 'push' && (gitea.ref == 'refs/heads/develop' || gitea.ref == 'refs/heads/qa') }}
needs: deploy
timeout-minutes: 35
runs-on: docker
env:
PLAYWRIGHT_BROWSERS_PATH: .playwright-browsers
steps:
- name: Checkout
uses: actions/checkout@v4
with:
github-server-url: https://git.dumas.ddns.net
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: 20
- name: Install dependencies
run: npm ci
- name: Install Playwright Chromium
run: npx playwright install --with-deps chromium
- name: Resolve browser test target
shell: bash
env:
GITEA_REF: ${{ gitea.ref }}
run: |
set -Eeuo pipefail
case "$GITEA_REF" in
refs/heads/develop) echo "E2E_BASE_URL=http://192.168.1.15:7002" >> "$GITHUB_ENV"; echo "E2E_ENVIRONMENT=dev" >> "$GITHUB_ENV" ;;
refs/heads/qa) echo "E2E_BASE_URL=http://192.168.1.15:6002" >> "$GITHUB_ENV"; echo "E2E_ENVIRONMENT=qa" >> "$GITHUB_ENV" ;;
*) echo "Unexpected browser-test ref: $GITEA_REF"; exit 1 ;;
esac
if [ -z "${{ secrets.COMFYUI_PORTAL_E2E_USERNAME }}" ] || [ -z "${{ secrets.COMFYUI_PORTAL_E2E_PASSWORD }}" ]; then
echo "COMFYUI_PORTAL_E2E_USERNAME and COMFYUI_PORTAL_E2E_PASSWORD must be configured before browser tests can run."
exit 1
fi
- name: Run browser tests against selected environment
id: browser
continue-on-error: true
env:
E2E_USERNAME: ${{ secrets.COMFYUI_PORTAL_E2E_USERNAME }}
E2E_PASSWORD: ${{ secrets.COMFYUI_PORTAL_E2E_PASSWORD }}
run: |
set +e
timeout --signal=TERM --kill-after=30s 30m npm run test:playwright
status=$?
exit "$status"
- name: Track browser failures in Gitea
if: always()
env:
GITEA_API_TOKEN: ${{ secrets.COMFYUI_PORTAL_ISSUE_TOKEN }}
GITEA_REPOSITORY: ${{ gitea.repository }}
GITEA_RUN_ID: ${{ gitea.run_id }}
GITEA_SHA: ${{ gitea.sha }}
E2E_ENVIRONMENT: ${{ gitea.ref == 'refs/heads/qa' && 'qa' || 'dev' }}
BROWSER_TEST_OUTCOME: ${{ steps.browser.outcome }}
run: npm run qa:report
- name: Fail browser job when tests fail
if: ${{ steps.browser.outcome == 'failure' }}
run: exit 1
qa-gate:
if: ${{ always() && (gitea.event_name == 'push' || gitea.event_name == 'pull_request') }}
needs: [test-build, deploy, browser-test]
timeout-minutes: 5
runs-on: docker
steps:
- name: Checkout
uses: actions/checkout@v4
with:
github-server-url: https://git.dumas.ddns.net
- name: Check automated QA blockers
env:
GITEA_API_TOKEN: ${{ secrets.COMFYUI_PORTAL_ISSUE_TOKEN }}
GITEA_REPOSITORY: ${{ gitea.repository }}
run: npm run qa:gate
+11
View File
@@ -16,3 +16,14 @@ The first run seeds `data/users.json` from `ADMIN_USERNAME` and `ADMIN_PASSWORD`
Gitea Actions deploys `develop`, `qa`, and `main` to the existing ComfyUiPortal target directories and preserves the existing 7002/6002/5002 port contract. On first startup of a fresh data directory, provide `ADMIN_USERNAME` and `ADMIN_PASSWORD` in the deployment host environment. Existing target data is retained during repository cutover.
After validation succeeds, the workflow builds one immutable image tagged with the source commit SHA on the Docker host and publishes it to the Gitea container registry. The deploy stage only pulls that exact image and restarts the service; it does not build. If the SHA-tagged image already exists, the workflow reuses it instead of rebuilding. Configure the masked `REGISTRY_USERNAME` and `REGISTRY_PASSWORD` repository Actions secrets with a Gitea account/token that can write container packages. Compose receives the exact image reference through `IMAGE_REF`, so the same artifact can be redeployed or promoted without rebuilding.
## Automated testing
The project maintains both server tests and Playwright browser tests. The browser suite runs after deployment on develop and qa only; production is never browser-tested automatically. It covers authentication, protected-route redirects, the job monitor's stable states, sign-out, and mobile layout overflow.
Configure these masked repository Actions secrets before enabling the browser job:
- COMFYUI_PORTAL_E2E_USERNAME — a dedicated admin test account
- COMFYUI_PORTAL_E2E_PASSWORD — its password
- COMFYUI_PORTAL_ISSUE_TOKEN — a Gitea token that can create, update, comment on, and close issues
Failed tests are fingerprinted by environment and test name. The workflow reuses the matching Gitea issue instead of creating duplicates, closes it after a later passing run, and blocks the QA gate while automated UI blockers remain open. Screenshots and video are disabled; Playwright traces are retained only on failure.
+61
View File
@@ -0,0 +1,61 @@
const { test, expect } = require('@playwright/test');
const username = process.env.E2E_USERNAME;
const password = process.env.E2E_PASSWORD;
const requiresCredentials = username && password;
test.describe('ComfyUI Portal browser flow', () => {
test.beforeEach(async ({ page }) => {
test.skip(!requiresCredentials, 'E2E_USERNAME and E2E_PASSWORD are required for deployed portal tests');
await page.goto('/login');
});
async function signIn(page, user = username, pass = password) {
await page.getByLabel('Username').fill(user);
await page.getByLabel('Password').fill(pass);
await page.getByRole('button', { name: 'Sign in' }).click();
}
test('valid credentials open the protected job monitor', async ({ page }) => {
await signIn(page);
await expect(page).toHaveURL(/\/jobs/);
await expect(page.getByRole('heading', { name: 'Job monitor' })).toBeVisible();
await expect(page.getByText('ComfyUI status')).toBeVisible();
});
test('invalid credentials remain on sign in with a clear error', async ({ page }) => {
await signIn(page, username, String(password) + '-invalid');
await expect(page).toHaveURL(/\/login/);
await expect(page.getByText('Invalid username or password.')).toBeVisible();
});
test('protected jobs route redirects an unauthenticated visitor', async ({ page }) => {
await page.context().clearCookies();
await page.goto('/jobs');
await expect(page).toHaveURL(/\/login/);
await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible();
});
test('the job monitor exposes stable loading or empty/error states', async ({ page }) => {
await signIn(page);
await expect(page.getByRole('heading', { name: 'Job monitor' })).toBeVisible();
await expect(page.locator('.server-card')).toBeVisible();
await expect(page.locator('.loading, .empty, .notice, .grid .job, .server-error').first()).toBeVisible({ timeout: 15000 });
});
test('an authenticated user can sign out', async ({ page }) => {
await signIn(page);
await expect(page.getByRole('button', { name: 'Sign out' })).toBeVisible();
await page.getByRole('button', { name: 'Sign out' }).click();
await expect(page).toHaveURL(/\/login/);
await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible();
});
test('the portal remains usable at mobile width without horizontal overflow', async ({ page }) => {
await page.setViewportSize({ width: 390, height: 844 });
await signIn(page);
await expect(page.getByRole('heading', { name: 'Job monitor' })).toBeVisible();
const overflow = await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth + 1);
expect(overflow, 'mobile layout should not create horizontal scrolling').toBe(false);
});
});
+3 -1
View File
@@ -11,7 +11,9 @@
"dev": "npm run build:frontend && node server.js",
"start": "node server.js",
"test": "node --test server.test.js",
"test:playwright": "node ./scripts/setup-playwright-runtime.js && playwright test --config=playwright.config.js"
"test:playwright": "playwright test --config=playwright.config.js",
"qa:report": "node scripts/qa-issues.mjs report e2e/test-results.json",
"qa:gate": "node scripts/qa-issues.mjs gate"
},
"engines": {
"node": ">=20"
+22
View File
@@ -0,0 +1,22 @@
const { defineConfig, devices } = require('@playwright/test');
const baseURL = process.env.E2E_BASE_URL || 'http://127.0.0.1:7002';
module.exports = defineConfig({
testDir: './e2e',
timeout: 30000,
expect: { timeout: 10000 },
fullyParallel: false,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 1 : 0,
workers: 1,
reporter: [['list'], ['json', { outputFile: 'e2e/test-results.json' }]],
use: {
baseURL,
headless: true,
trace: 'retain-on-failure',
screenshot: 'off',
video: 'off',
...devices['Desktop Chrome'],
},
});
+112
View File
@@ -0,0 +1,112 @@
import crypto from "node:crypto";
import fs from "node:fs";
const apiRoot = (process.env.GITEA_API_URL || "https://git.dumas.ddns.net/api/v1").replace(/\/$/, "");
const repository = process.env.GITEA_REPOSITORY || "chris.dumas/ComfyUiPortalNext";
const token = process.env.GITEA_API_TOKEN;
const environment = process.env.E2E_ENVIRONMENT || "unknown";
const runId = process.env.GITEA_RUN_ID || "local";
const sha = process.env.GITEA_SHA || "unknown";
const mode = process.argv[2] || "report";
const resultPath = process.argv[3] || "e2e/test-results.json";
const serverRoot = apiRoot.replace(/\/api\/v1$/, "");
function fail(message) { throw new Error(message); }
function headers() { return { Accept: "application/json", "Content-Type": "application/json", Authorization: "token " + token }; }
async function request(path, options = {}) {
const response = await fetch(apiRoot + path, { ...options, headers: { ...headers(), ...(options.headers || {}) } });
const body = response.status === 204 ? null : await response.json().catch(() => null);
if (!response.ok) throw new Error("Gitea API " + response.status + ": " + (body?.message || response.statusText));
return body;
}
function labels(issue) { return (issue.labels || []).map((item) => typeof item === "string" ? item : item.name); }
function keyFor(name) { return "ui-test:" + environment + ":" + name; }
function marker(key) { return "<!-- qa-automation-key: " + key + " -->"; }
function fingerprint(key) { return crypto.createHash("sha256").update(key).digest("hex").slice(0, 16); }
function issueTitle(name) { return "[" + environment + "] UI test failure: " + name; }
function flattenSuites(suites, output = []) {
for (const suite of suites || []) {
for (const spec of suite.specs || []) {
const results = (spec.tests || []).flatMap((item) => item.results || []);
const failed = results.find((result) => result.status === "failed" || result.error);
output.push({
name: [suite.title, spec.title].filter(Boolean).join(" › "),
passed: Boolean(spec.ok) && !failed,
durationMs: results.reduce((total, result) => total + (result.duration || 0), 0),
error: failed?.error?.message || failed?.error?.stack || (failed ? "Browser test status: " + failed.status : ""),
});
}
flattenSuites(suite.suites, output);
}
return output;
}
function readResults() {
if (!fs.existsSync(resultPath)) return [];
try { return flattenSuites(JSON.parse(fs.readFileSync(resultPath, "utf8")).suites); } catch { return []; }
}
async function allIssues() {
const result = [];
for (let page = 1; page < 20; page += 1) {
const batch = await request("/repos/" + repository + "/issues?state=all&limit=50&page=" + page);
result.push(...batch);
if (batch.length < 50) break;
}
return result.filter((issue) => labels(issue).includes("automated") && labels(issue).includes("ui-test"));
}
async function ensureLabel(name, color) {
const existing = await request("/repos/" + repository + "/labels?limit=50");
const found = existing.find((label) => label.name === name);
if (found) return found.id;
return (await request("/repos/" + repository + "/labels", { method: "POST", body: JSON.stringify({ name, color }) })).id;
}
async function comment(issue, body) {
const comments = await request("/repos/" + repository + "/issues/" + issue.number + "/comments?limit=50");
if (comments.some((item) => item.body?.includes("qa-run: " + runId))) return;
await request("/repos/" + repository + "/issues/" + issue.number + "/comments", { method: "POST", body: JSON.stringify({ body: body + "\n\nqa-run: " + runId }) });
}
async function closeResolved(issue, name) {
if (!issue || issue.state !== "open") return;
await comment(issue, marker(keyFor(name)) + "\nAutomated UI test passed on commit " + sha + "; closing this blocker.");
await request("/repos/" + repository + "/issues/" + issue.number, { method: "PATCH", body: JSON.stringify({ state: "closed" }) });
}
async function reportFailure(issue, test, labelIds) {
const key = keyFor(test.name);
const body = [
marker(key), "**Automated UI failure**", "",
"- Environment: " + environment, "- Test: " + test.name, "- Commit: " + sha,
"- Run: [" + runId + "](" + serverRoot + "/" + repository + "/actions/runs/" + runId + ")",
"- Fingerprint: " + fingerprint(key), "- Duration: " + test.durationMs + " ms", "",
"Failure details:", test.error || "The browser test failed without a serialized error.",
].join("\n");
if (!issue) issue = await request("/repos/" + repository + "/issues", { method: "POST", body: JSON.stringify({ title: issueTitle(test.name), body, labels: labelIds }) });
else {
if (issue.state === "closed") await request("/repos/" + repository + "/issues/" + issue.number, { method: "PATCH", body: JSON.stringify({ state: "open" }) });
await comment(issue, body);
}
console.log("Tracked UI failure in issue #" + issue.number);
}
async function main() {
if (!token) fail("GITEA_API_TOKEN is required for QA issue automation");
const issues = await allIssues();
if (mode === "gate") {
const blockers = issues.filter((issue) => issue.state === "open" && labels(issue).includes("qa-blocker"));
if (blockers.length) {
console.error("Open automated QA blockers prevent this build:");
blockers.forEach((issue) => console.error("- #" + issue.number + ": " + issue.title));
process.exitCode = 1;
} else console.log("No open automated QA blockers.");
return;
}
const tests = readResults();
if (!tests.length && process.env.BROWSER_TEST_OUTCOME === "failure") tests.push({ name: "browser-suite", passed: false, durationMs: 0, error: "Browser suite failed before producing parseable results." });
const labelIds = [await ensureLabel("automated", "5319e7"), await ensureLabel("ui-test", "1d76db"), await ensureLabel("qa-blocker", "d73a4a")];
for (const test of tests) {
const issue = issues.find((item) => item.body?.includes(marker(keyFor(test.name))));
if (test.passed) await closeResolved(issue, test.name);
else await reportFailure(issue, test, labelIds);
}
if (tests.some((test) => !test.passed)) process.exitCode = 1;
}
main().catch((error) => { console.error(error.message); process.exitCode = 1; });