From 96be6412a8e6b8d9fb1c2d8c1d7804d8c27d5846 Mon Sep 17 00:00:00 2001 From: "chris.dumas" Date: Mon, 26 Oct 2026 14:16:26 +0000 Subject: [PATCH] Add portal browser testing and QA issue automation --- .gitea/workflows/deploy.yml | 76 ++++++++++++++++++++++++ README.md | 11 ++++ e2e/portal.spec.js | 61 ++++++++++++++++++++ package.json | 4 +- playwright.config.js | 22 +++++++ scripts/qa-issues.mjs | 112 ++++++++++++++++++++++++++++++++++++ 6 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 e2e/portal.spec.js create mode 100644 playwright.config.js create mode 100644 scripts/qa-issues.mjs diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 72d708e..6fc36ad 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -130,3 +130,79 @@ jobs: " for attempt in 1 2 3 4 5 6; do curl --fail --silent --show-error --max-time 15 "$HEALTH_URL/api/health" && exit 0 || sleep 5; done exit 1 + + browser-test: + if: ${{ gitea.event_name == 'push' && (gitea.ref == 'refs/heads/develop' || gitea.ref == 'refs/heads/qa') }} + needs: deploy + timeout-minutes: 35 + runs-on: docker + env: + PLAYWRIGHT_BROWSERS_PATH: .playwright-browsers + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + github-server-url: https://git.dumas.ddns.net + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Install dependencies + run: npm ci + - name: Install Playwright Chromium + run: npx playwright install --with-deps chromium + - name: Resolve browser test target + shell: bash + env: + GITEA_REF: ${{ gitea.ref }} + run: | + set -Eeuo pipefail + case "$GITEA_REF" in + refs/heads/develop) echo "E2E_BASE_URL=http://192.168.1.15:7002" >> "$GITHUB_ENV"; echo "E2E_ENVIRONMENT=dev" >> "$GITHUB_ENV" ;; + refs/heads/qa) echo "E2E_BASE_URL=http://192.168.1.15:6002" >> "$GITHUB_ENV"; echo "E2E_ENVIRONMENT=qa" >> "$GITHUB_ENV" ;; + *) echo "Unexpected browser-test ref: $GITEA_REF"; exit 1 ;; + esac + if [ -z "${{ secrets.COMFYUI_PORTAL_E2E_USERNAME }}" ] || [ -z "${{ secrets.COMFYUI_PORTAL_E2E_PASSWORD }}" ]; then + echo "COMFYUI_PORTAL_E2E_USERNAME and COMFYUI_PORTAL_E2E_PASSWORD must be configured before browser tests can run." + exit 1 + fi + - name: Run browser tests against selected environment + id: browser + continue-on-error: true + env: + E2E_USERNAME: ${{ secrets.COMFYUI_PORTAL_E2E_USERNAME }} + E2E_PASSWORD: ${{ secrets.COMFYUI_PORTAL_E2E_PASSWORD }} + run: | + set +e + timeout --signal=TERM --kill-after=30s 30m npm run test:playwright + status=$? + exit "$status" + - name: Track browser failures in Gitea + if: always() + env: + GITEA_API_TOKEN: ${{ secrets.COMFYUI_PORTAL_ISSUE_TOKEN }} + GITEA_REPOSITORY: ${{ gitea.repository }} + GITEA_RUN_ID: ${{ gitea.run_id }} + GITEA_SHA: ${{ gitea.sha }} + E2E_ENVIRONMENT: ${{ gitea.ref == 'refs/heads/qa' && 'qa' || 'dev' }} + BROWSER_TEST_OUTCOME: ${{ steps.browser.outcome }} + run: npm run qa:report + - name: Fail browser job when tests fail + if: ${{ steps.browser.outcome == 'failure' }} + run: exit 1 + + qa-gate: + if: ${{ always() && (gitea.event_name == 'push' || gitea.event_name == 'pull_request') }} + needs: [test-build, deploy, browser-test] + timeout-minutes: 5 + runs-on: docker + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + github-server-url: https://git.dumas.ddns.net + - name: Check automated QA blockers + env: + GITEA_API_TOKEN: ${{ secrets.COMFYUI_PORTAL_ISSUE_TOKEN }} + GITEA_REPOSITORY: ${{ gitea.repository }} + run: npm run qa:gate diff --git a/README.md b/README.md index 4f4adb8..ef472aa 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,14 @@ The first run seeds `data/users.json` from `ADMIN_USERNAME` and `ADMIN_PASSWORD` Gitea Actions deploys `develop`, `qa`, and `main` to the existing ComfyUiPortal target directories and preserves the existing 7002/6002/5002 port contract. On first startup of a fresh data directory, provide `ADMIN_USERNAME` and `ADMIN_PASSWORD` in the deployment host environment. Existing target data is retained during repository cutover. After validation succeeds, the workflow builds one immutable image tagged with the source commit SHA on the Docker host and publishes it to the Gitea container registry. The deploy stage only pulls that exact image and restarts the service; it does not build. If the SHA-tagged image already exists, the workflow reuses it instead of rebuilding. Configure the masked `REGISTRY_USERNAME` and `REGISTRY_PASSWORD` repository Actions secrets with a Gitea account/token that can write container packages. Compose receives the exact image reference through `IMAGE_REF`, so the same artifact can be redeployed or promoted without rebuilding. +## Automated testing + +The project maintains both server tests and Playwright browser tests. The browser suite runs after deployment on develop and qa only; production is never browser-tested automatically. It covers authentication, protected-route redirects, the job monitor's stable states, sign-out, and mobile layout overflow. + +Configure these masked repository Actions secrets before enabling the browser job: + +- COMFYUI_PORTAL_E2E_USERNAME — a dedicated admin test account +- COMFYUI_PORTAL_E2E_PASSWORD — its password +- COMFYUI_PORTAL_ISSUE_TOKEN — a Gitea token that can create, update, comment on, and close issues + +Failed tests are fingerprinted by environment and test name. The workflow reuses the matching Gitea issue instead of creating duplicates, closes it after a later passing run, and blocks the QA gate while automated UI blockers remain open. Screenshots and video are disabled; Playwright traces are retained only on failure. diff --git a/e2e/portal.spec.js b/e2e/portal.spec.js new file mode 100644 index 0000000..a7de988 --- /dev/null +++ b/e2e/portal.spec.js @@ -0,0 +1,61 @@ +const { test, expect } = require('@playwright/test'); + +const username = process.env.E2E_USERNAME; +const password = process.env.E2E_PASSWORD; +const requiresCredentials = username && password; + +test.describe('ComfyUI Portal browser flow', () => { + test.beforeEach(async ({ page }) => { + test.skip(!requiresCredentials, 'E2E_USERNAME and E2E_PASSWORD are required for deployed portal tests'); + await page.goto('/login'); + }); + + async function signIn(page, user = username, pass = password) { + await page.getByLabel('Username').fill(user); + await page.getByLabel('Password').fill(pass); + await page.getByRole('button', { name: 'Sign in' }).click(); + } + + test('valid credentials open the protected job monitor', async ({ page }) => { + await signIn(page); + await expect(page).toHaveURL(/\/jobs/); + await expect(page.getByRole('heading', { name: 'Job monitor' })).toBeVisible(); + await expect(page.getByText('ComfyUI status')).toBeVisible(); + }); + + test('invalid credentials remain on sign in with a clear error', async ({ page }) => { + await signIn(page, username, String(password) + '-invalid'); + await expect(page).toHaveURL(/\/login/); + await expect(page.getByText('Invalid username or password.')).toBeVisible(); + }); + + test('protected jobs route redirects an unauthenticated visitor', async ({ page }) => { + await page.context().clearCookies(); + await page.goto('/jobs'); + await expect(page).toHaveURL(/\/login/); + await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible(); + }); + + test('the job monitor exposes stable loading or empty/error states', async ({ page }) => { + await signIn(page); + await expect(page.getByRole('heading', { name: 'Job monitor' })).toBeVisible(); + await expect(page.locator('.server-card')).toBeVisible(); + await expect(page.locator('.loading, .empty, .notice, .grid .job, .server-error').first()).toBeVisible({ timeout: 15000 }); + }); + + test('an authenticated user can sign out', async ({ page }) => { + await signIn(page); + await expect(page.getByRole('button', { name: 'Sign out' })).toBeVisible(); + await page.getByRole('button', { name: 'Sign out' }).click(); + await expect(page).toHaveURL(/\/login/); + await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible(); + }); + + test('the portal remains usable at mobile width without horizontal overflow', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await signIn(page); + await expect(page.getByRole('heading', { name: 'Job monitor' })).toBeVisible(); + const overflow = await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth + 1); + expect(overflow, 'mobile layout should not create horizontal scrolling').toBe(false); + }); +}); diff --git a/package.json b/package.json index 377f8cc..5813a4a 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,9 @@ "dev": "npm run build:frontend && node server.js", "start": "node server.js", "test": "node --test server.test.js", - "test:playwright": "node ./scripts/setup-playwright-runtime.js && playwright test --config=playwright.config.js" + "test:playwright": "playwright test --config=playwright.config.js", + "qa:report": "node scripts/qa-issues.mjs report e2e/test-results.json", + "qa:gate": "node scripts/qa-issues.mjs gate" }, "engines": { "node": ">=20" diff --git a/playwright.config.js b/playwright.config.js new file mode 100644 index 0000000..a31acb2 --- /dev/null +++ b/playwright.config.js @@ -0,0 +1,22 @@ +const { defineConfig, devices } = require('@playwright/test'); + +const baseURL = process.env.E2E_BASE_URL || 'http://127.0.0.1:7002'; + +module.exports = defineConfig({ + testDir: './e2e', + timeout: 30000, + expect: { timeout: 10000 }, + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + workers: 1, + reporter: [['list'], ['json', { outputFile: 'e2e/test-results.json' }]], + use: { + baseURL, + headless: true, + trace: 'retain-on-failure', + screenshot: 'off', + video: 'off', + ...devices['Desktop Chrome'], + }, +}); diff --git a/scripts/qa-issues.mjs b/scripts/qa-issues.mjs new file mode 100644 index 0000000..27c46ea --- /dev/null +++ b/scripts/qa-issues.mjs @@ -0,0 +1,112 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; + +const apiRoot = (process.env.GITEA_API_URL || "https://git.dumas.ddns.net/api/v1").replace(/\/$/, ""); +const repository = process.env.GITEA_REPOSITORY || "chris.dumas/ComfyUiPortalNext"; +const token = process.env.GITEA_API_TOKEN; +const environment = process.env.E2E_ENVIRONMENT || "unknown"; +const runId = process.env.GITEA_RUN_ID || "local"; +const sha = process.env.GITEA_SHA || "unknown"; +const mode = process.argv[2] || "report"; +const resultPath = process.argv[3] || "e2e/test-results.json"; +const serverRoot = apiRoot.replace(/\/api\/v1$/, ""); + +function fail(message) { throw new Error(message); } +function headers() { return { Accept: "application/json", "Content-Type": "application/json", Authorization: "token " + token }; } +async function request(path, options = {}) { + const response = await fetch(apiRoot + path, { ...options, headers: { ...headers(), ...(options.headers || {}) } }); + const body = response.status === 204 ? null : await response.json().catch(() => null); + if (!response.ok) throw new Error("Gitea API " + response.status + ": " + (body?.message || response.statusText)); + return body; +} +function labels(issue) { return (issue.labels || []).map((item) => typeof item === "string" ? item : item.name); } +function keyFor(name) { return "ui-test:" + environment + ":" + name; } +function marker(key) { return ""; } +function fingerprint(key) { return crypto.createHash("sha256").update(key).digest("hex").slice(0, 16); } +function issueTitle(name) { return "[" + environment + "] UI test failure: " + name; } + +function flattenSuites(suites, output = []) { + for (const suite of suites || []) { + for (const spec of suite.specs || []) { + const results = (spec.tests || []).flatMap((item) => item.results || []); + const failed = results.find((result) => result.status === "failed" || result.error); + output.push({ + name: [suite.title, spec.title].filter(Boolean).join(" › "), + passed: Boolean(spec.ok) && !failed, + durationMs: results.reduce((total, result) => total + (result.duration || 0), 0), + error: failed?.error?.message || failed?.error?.stack || (failed ? "Browser test status: " + failed.status : ""), + }); + } + flattenSuites(suite.suites, output); + } + return output; +} +function readResults() { + if (!fs.existsSync(resultPath)) return []; + try { return flattenSuites(JSON.parse(fs.readFileSync(resultPath, "utf8")).suites); } catch { return []; } +} +async function allIssues() { + const result = []; + for (let page = 1; page < 20; page += 1) { + const batch = await request("/repos/" + repository + "/issues?state=all&limit=50&page=" + page); + result.push(...batch); + if (batch.length < 50) break; + } + return result.filter((issue) => labels(issue).includes("automated") && labels(issue).includes("ui-test")); +} +async function ensureLabel(name, color) { + const existing = await request("/repos/" + repository + "/labels?limit=50"); + const found = existing.find((label) => label.name === name); + if (found) return found.id; + return (await request("/repos/" + repository + "/labels", { method: "POST", body: JSON.stringify({ name, color }) })).id; +} +async function comment(issue, body) { + const comments = await request("/repos/" + repository + "/issues/" + issue.number + "/comments?limit=50"); + if (comments.some((item) => item.body?.includes("qa-run: " + runId))) return; + await request("/repos/" + repository + "/issues/" + issue.number + "/comments", { method: "POST", body: JSON.stringify({ body: body + "\n\nqa-run: " + runId }) }); +} +async function closeResolved(issue, name) { + if (!issue || issue.state !== "open") return; + await comment(issue, marker(keyFor(name)) + "\nAutomated UI test passed on commit " + sha + "; closing this blocker."); + await request("/repos/" + repository + "/issues/" + issue.number, { method: "PATCH", body: JSON.stringify({ state: "closed" }) }); +} +async function reportFailure(issue, test, labelIds) { + const key = keyFor(test.name); + const body = [ + marker(key), "**Automated UI failure**", "", + "- Environment: " + environment, "- Test: " + test.name, "- Commit: " + sha, + "- Run: [" + runId + "](" + serverRoot + "/" + repository + "/actions/runs/" + runId + ")", + "- Fingerprint: " + fingerprint(key), "- Duration: " + test.durationMs + " ms", "", + "Failure details:", test.error || "The browser test failed without a serialized error.", + ].join("\n"); + if (!issue) issue = await request("/repos/" + repository + "/issues", { method: "POST", body: JSON.stringify({ title: issueTitle(test.name), body, labels: labelIds }) }); + else { + if (issue.state === "closed") await request("/repos/" + repository + "/issues/" + issue.number, { method: "PATCH", body: JSON.stringify({ state: "open" }) }); + await comment(issue, body); + } + console.log("Tracked UI failure in issue #" + issue.number); +} + +async function main() { + if (!token) fail("GITEA_API_TOKEN is required for QA issue automation"); + const issues = await allIssues(); + if (mode === "gate") { + const blockers = issues.filter((issue) => issue.state === "open" && labels(issue).includes("qa-blocker")); + if (blockers.length) { + console.error("Open automated QA blockers prevent this build:"); + blockers.forEach((issue) => console.error("- #" + issue.number + ": " + issue.title)); + process.exitCode = 1; + } else console.log("No open automated QA blockers."); + return; + } + const tests = readResults(); + if (!tests.length && process.env.BROWSER_TEST_OUTCOME === "failure") tests.push({ name: "browser-suite", passed: false, durationMs: 0, error: "Browser suite failed before producing parseable results." }); + const labelIds = [await ensureLabel("automated", "5319e7"), await ensureLabel("ui-test", "1d76db"), await ensureLabel("qa-blocker", "d73a4a")]; + for (const test of tests) { + const issue = issues.find((item) => item.body?.includes(marker(keyFor(test.name)))); + if (test.passed) await closeResolved(issue, test.name); + else await reportFailure(issue, test, labelIds); + } + if (tests.some((test) => !test.passed)) process.exitCode = 1; +} +main().catch((error) => { console.error(error.message); process.exitCode = 1; });