From 2f7f2cc37500249939bc54098a49ea05d3eb294f Mon Sep 17 00:00:00 2001 From: "chris.dumas" Date: Mon, 26 Oct 2026 17:19:11 +0000 Subject: [PATCH] Persist job history and show model usage stats --- README.md | 8 +- e2e/portal.spec.js | 16 +++- frontend/src/app/pages/jobs.page.html | 1 + frontend/src/app/pages/jobs.page.scss | 1 + frontend/src/app/pages/jobs.page.ts | 130 +++++++++++++++++++++++++- job-store.js | 82 ++++++++++++++++ server.js | 7 +- server.test.js | 13 +++ 8 files changed, 252 insertions(+), 6 deletions(-) create mode 100644 job-store.js diff --git a/README.md b/README.md index ef472aa..8b7f8c3 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,15 @@ The first run seeds `data/users.json` from `ADMIN_USERNAME` and `ADMIN_PASSWORD` Gitea Actions deploys `develop`, `qa`, and `main` to the existing ComfyUiPortal target directories and preserves the existing 7002/6002/5002 port contract. On first startup of a fresh data directory, provide `ADMIN_USERNAME` and `ADMIN_PASSWORD` in the deployment host environment. Existing target data is retained during repository cutover. After validation succeeds, the workflow builds one immutable image tagged with the source commit SHA on the Docker host and publishes it to the Gitea container registry. The deploy stage only pulls that exact image and restarts the service; it does not build. If the SHA-tagged image already exists, the workflow reuses it instead of rebuilding. Configure the masked `REGISTRY_USERNAME` and `REGISTRY_PASSWORD` repository Actions secrets with a Gitea account/token that can write container packages. Compose receives the exact image reference through `IMAGE_REF`, so the same artifact can be redeployed or promoted without rebuilding. +## Persistent job and model history + +Each successful `/api/jobs` read synchronises the returned ComfyUI history and currently running job into `data/job-history.json`. The prompt ID is the stable key, so repeated polling and server restarts do not count the same job twice. The deployment Compose files mount `data/` from the environment-specific host directory and the deployment cleanup explicitly preserves it. + +The store retains job status, title, timestamps, output count, and the distinct model names extracted from each prompt. Model usage is counted once per model per prompt ID. `GET /api/model-stats` returns the tracked job count and model ranking used by the **Most used models** panel. This endpoint is authenticated and read-only. Removing the environment's `data/job-history.json` intentionally resets the history. + ## Automated testing -The project maintains both server tests and Playwright browser tests. The browser suite runs after deployment on develop and qa only; production is never browser-tested automatically. It covers authentication, protected-route redirects, the job monitor's stable states, sign-out, and mobile layout overflow. +The project maintains both server tests and Playwright browser tests. The browser suite runs after deployment on develop and qa only; production is never browser-tested automatically. It covers authentication, protected-route redirects, persistent model statistics, server/job/asset states, Manager UI behavior using mocked control requests, sign-out, and mobile layout overflow. Configure these masked repository Actions secrets before enabling the browser job: diff --git a/e2e/portal.spec.js b/e2e/portal.spec.js index 7bb8ec0..53d83bd 100644 --- a/e2e/portal.spec.js +++ b/e2e/portal.spec.js @@ -71,8 +71,9 @@ const connectedServer = { queue: { running: 0, pending: 0, total: 0 }, }; const managerInfo = { managerVersion: '3.0.0', currentVersion: '0.3.0', versions: ['0.3.0', '0.2.9'] }; +const modelStats = { totalJobs: 3, totalModels: 2, models: [{ name: 'model-a.safetensors', jobCount: 2 }, { name: 'style.safetensors', jobCount: 1 }] }; -async function mockReadOnlyApi(page, { user = adminUser, jobs = [], running = null, server = connectedServer, manager = managerInfo } = {}) { +async function mockReadOnlyApi(page, { user = adminUser, jobs = [], running = null, server = connectedServer, manager = managerInfo, stats = modelStats } = {}) { test.skip(!requiresCredentials, 'deployed read-only UI tests require the configured E2E account'); if (user) { await page.context().addCookies([{ name: 'cp_auth', value: username, url: process.env.E2E_BASE_URL || 'http://127.0.0.1:7002' }]); @@ -85,6 +86,7 @@ async function mockReadOnlyApi(page, { user = adminUser, jobs = [], running = nu if (jobs === 'unavailable') return route.fulfill({ status: 503, contentType: 'application/json', body: JSON.stringify({ error: 'Jobs unavailable' }) }); return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ jobs, running }) }); } + if (url.pathname === '/api/model-stats') return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(stats) }); if (url.pathname === '/api/server-status') { if (server === 'unavailable') return route.fulfill({ status: 502, contentType: 'application/json', body: JSON.stringify({ error: 'ComfyUI is unavailable right now.' }) }); return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ status: server }) }); @@ -116,6 +118,7 @@ test.describe('ComfyUI Portal read-only UI states', () => { await page.route('**/api/**', async (route) => { const pathname = new URL(route.request().url()).pathname; if (pathname === '/api/me') return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ user: adminUser }) }); + if (pathname === '/api/model-stats') return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(modelStats) }); if (pathname === '/api/server-status') return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ status: connectedServer }) }); if (pathname === '/api/comfy/manager') return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(managerInfo) }); if (pathname === '/api/jobs') { @@ -192,6 +195,17 @@ test.describe('ComfyUI Portal read-only UI states', () => { await expect(page.getByText('6.0 GB free of 8.0 GB')).toBeVisible(); }); + test('renders persistent most-used model statistics without counting duplicate jobs twice', async ({ page }) => { + await mockReadOnlyApi(page); + await page.goto('/jobs'); + await expect(page.getByRole('heading', { name: 'Most used models' })).toBeVisible(); + await expect(page.getByText('3 tracked jobs · 2 models')).toBeVisible(); + await expect(page.locator('.stats-list li')).toHaveCount(2); + await expect(page.locator('.stats-list li').first()).toContainText('model-a.safetensors'); + await expect(page.locator('.stats-list li').first()).toContainText('2 jobs'); + await expect(page.locator('.stats-list li').nth(1)).toContainText('style.safetensors'); + }); + test('renders video assets with metadata and safe output links', async ({ page }) => { const jobs = [{ promptId: 'video-1', title: 'Finished video', status: 'completed', queuedAt: 1000, diff --git a/frontend/src/app/pages/jobs.page.html b/frontend/src/app/pages/jobs.page.html index 6455ff6..96a1fd3 100644 --- a/frontend/src/app/pages/jobs.page.html +++ b/frontend/src/app/pages/jobs.page.html @@ -1,5 +1,6 @@
REMOTE JOBS

Job monitor

Read-only view of the remote ComfyUI queue and history.

+
PERSISTENT HISTORY

Most used models

{{modelStats.totalJobs}} tracked jobs · {{modelStats.totalModels}} models
No model usage has been recorded yet.
  1. {{rank + 1}}{{model.name}}{{model.jobCount}} job{{model.jobCount === 1 ? '' : 's'}}
SERVER

ComfyUI status

{{server?'Connected':'Unavailable'}}
{{serverError}}
{{server.queue.total}}jobs queued
{{server.queue.running}}running
{{server.queue.pending}}waiting
Version{{server.version || 'Unknown'}}
Operating system{{server.os || 'Unknown'}}
Python{{server.pythonVersion || 'Unknown'}}
{{device.name}}{{device.type}} · {{formatBytes(device.vramFree)}} free of {{formatBytes(device.vramTotal)}}
ComfyUI Manager {{manager?.managerVersion || '—'}}Runtime controls
Current {{manager.currentVersion}}
{{controlMessage}}
Update results
ComfyUI: {{updateResult.comfyui || 'No result reported'}}
Updated custom nodes
  • {{item.title}}
Already up to date
  • {{item.title}}
Could not update
  • {{item.title}} — {{item.error}}
diff --git a/frontend/src/app/pages/jobs.page.scss b/frontend/src/app/pages/jobs.page.scss index b829718..2b7dce0 100644 --- a/frontend/src/app/pages/jobs.page.scss +++ b/frontend/src/app/pages/jobs.page.scss @@ -1,4 +1,5 @@ .heading{display:flex;justify-content:space-between;align-items:start;gap:1rem;margin-bottom:2rem}.eyebrow{font-size:.75rem;font-weight:800;letter-spacing:.12em;color:#3b82f6}h1{margin:.35rem 0}.heading p{color:#63738a}.refresh{padding:.65rem 1rem;border:1px solid #cbd5e1;border-radius:8px;background:#fff;cursor:pointer}.server-card{background:#18283d;color:#fff;border-radius:14px;padding:1.25rem;margin-bottom:1.5rem}.server-heading{display:flex;justify-content:space-between;align-items:start}.server-heading h2{margin:.35rem 0 0;font-size:1.05rem}.connection{background:#bbf7d0;color:#166534;border-radius:999px;padding:.3rem .6rem;font-size:.75rem;font-weight:700}.connection.offline{background:#fecaca;color:#991b1b}.queue-panel{display:flex;gap:1.5rem;align-items:center;margin-top:1.25rem;padding:1rem;background:#243952;border-radius:10px}.queue-panel div{display:flex;flex-direction:column;gap:.15rem}.queue-panel strong{font-size:1.35rem}.queue-panel span{color:#aab8cb;font-size:.75rem}.server-info{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:1rem;margin-top:1.25rem}.server-info div{display:flex;flex-direction:column;gap:.25rem}.server-info span,.device span{color:#aab8cb;font-size:.75rem}.server-info strong{font-size:.85rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.devices{display:grid;gap:.5rem;margin-top:1.25rem}.device{display:flex;justify-content:space-between;gap:1rem;border-top:1px solid #34465d;padding-top:.7rem}.server-error{color:#fecaca;margin-top:1rem;font-size:.85rem}.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:1rem}.job{background:#fff;border:1px solid #dce3ed;border-radius:14px;padding:1.25rem}.jobhead{display:flex;justify-content:space-between;gap:1rem}.job h2{font-size:1rem;margin:0;color:#18283d}.status{font-size:.75rem;border-radius:999px;padding:.25rem .55rem;background:#e8eef7}.status.completed{background:#dcfce7;color:#166534}.status.failed{background:#fee2e2;color:#991b1b}.meta{display:flex;flex-wrap:wrap;gap:.5rem 1rem;color:#63738a;font-size:.8rem;margin-top:.8rem}.models{display:flex;flex-wrap:wrap;align-items:center;gap:.4rem;margin-top:1rem;font-size:.75rem}.models strong{color:#42526b}.models span{background:#eef2ff;color:#3730a3;border-radius:6px;padding:.3rem .45rem;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.assets{display:grid;grid-template-columns:repeat(auto-fill,minmax(120px,1fr));gap:.6rem;margin-top:1rem}.assets figure{margin:0;min-width:0}.assets a{display:block;aspect-ratio:1;background:#f1f5f9;border-radius:8px;overflow:hidden}.assets img,.assets video{display:block;width:100%;height:100%;object-fit:cover}.assets figcaption{font-size:.7rem;color:#63738a;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;margin-top:.3rem}.no-assets{margin-top:1rem;color:#94a3b8;font-size:.8rem}.notice{padding:1rem;background:#fff1f0;color:#b42318;border-radius:10px}.loading,.empty{padding:3rem;text-align:center;color:#63738a} +.model-stats{background:#fff;border:1px solid #dce3ed;border-radius:14px;padding:1.1rem 1.25rem;margin-bottom:1.5rem}.stats-heading{display:flex;justify-content:space-between;align-items:end;gap:1rem}.stats-heading h2{margin:.35rem 0 0;font-size:1.05rem;color:#18283d}.stats-heading>span{color:#63738a;font-size:.75rem}.stats-list{list-style:none;margin:1rem 0 0;padding:0;display:grid;gap:.5rem}.stats-list li{display:grid;grid-template-columns:1.6rem minmax(0,1fr) auto;align-items:center;gap:.6rem;padding:.55rem .65rem;background:#f8fafc;border-radius:8px}.stats-list strong{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:#18283d;font-size:.85rem}.rank{color:#64748b;font-size:.75rem;font-weight:700}.usage{color:#63738a;font-size:.75rem}.stats-empty{padding:1rem 0 0;color:#63738a;font-size:.8rem} .running-card{background:#fff;border:1px solid #93c5fd;border-radius:14px;padding:1.25rem;margin-bottom:1.5rem;box-shadow:0 6px 20px rgba(59,130,246,.08)}.running-heading{display:flex;justify-content:space-between;align-items:start;gap:1rem}.running-heading h2{margin:.35rem 0 0;font-size:1.05rem;color:#18283d}.progress-track{height:10px;background:#dbeafe;border-radius:999px;overflow:hidden;margin-top:1.25rem}.progress-value{height:100%;background:#2563eb;border-radius:inherit;transition:width .4s ease}.progress-track.indeterminate .progress-value{width:35%!important;animation:progress-slide 1.4s ease-in-out infinite}.progress-label{color:#42526b;font-size:.75rem;margin-top:.45rem}@keyframes progress-slide{0%{margin-left:-35%}100%{margin-left:100%}} .assets figcaption{display:flex;flex-direction:column;gap:.2rem;font-size:.7rem;color:#63738a;overflow:hidden;margin-top:.3rem}.assets figcaption strong{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-weight:600}.asset-meta{display:flex;gap:.5rem;color:#94a3b8;font-size:.68rem} .manager-controls{margin-top:1.25rem;border-top:1px solid #34465d;padding-top:1rem}.manager-heading{display:flex;justify-content:space-between;align-items:end;gap:1rem;color:#aab8cb;font-size:.75rem}.manager-heading div{display:flex;flex-direction:column;gap:.25rem}.manager-heading strong{color:#fff;font-size:.9rem}.control-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin-top:.75rem}.control-actions button,.control-actions select{border:1px solid #526983;border-radius:7px;background:#243952;color:#fff;padding:.5rem .65rem;font-size:.75rem}.control-actions button{cursor:pointer}.control-actions button:disabled,.control-actions select:disabled{opacity:.55;cursor:not-allowed}.control-message{color:#bfdbfe;font-size:.75rem;margin-top:.65rem} diff --git a/frontend/src/app/pages/jobs.page.ts b/frontend/src/app/pages/jobs.page.ts index 09103ac..a4d7f99 100644 --- a/frontend/src/app/pages/jobs.page.ts +++ b/frontend/src/app/pages/jobs.page.ts @@ -1,2 +1,128 @@ -import { CommonModule } from '@angular/common'; import { Component,OnDestroy,inject } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { AuthService,AuthUser } from '../auth.service'; interface Asset{filename:string;url:string;type:string;width?:number|null;height?:number|null;sizeBytes?:number|null;} interface Model{nodeId:string;type:string;name:string;} interface Progress{value:number;max:number;percent:number;} interface Job{promptId:string;title:string;status:string;queuedAt:number|null;durationMs:number|null;outputCount:number;assets:Asset[];models:Model[];progress:Progress|null;} interface Device{name:string;type:string;vramTotal:number;vramFree:number;} interface ServerStatus{reachable:boolean;version:string;os:string;pythonVersion:string;devices:Device[];queue:{running:number;pending:number;total:number};} interface ManagerInfo{managerVersion:string;currentVersion:string;versions:string[];} interface UpdateStatus{total_count:number;done_count:number;in_progress_count:number;is_processing:boolean;} interface UpdateProgress{target:string;uiTarget:string;totalCount:number;doneCount:number;} interface UpdateItem{id:string;title:string;url:string;error:string;} interface UpdateResult{comfyui:string|null;updated:UpdateItem[];skipped:UpdateItem[];failed:UpdateItem[];completedAt:number;} -@Component({standalone:true,imports:[CommonModule],templateUrl:'./jobs.page.html',styleUrls:['./jobs.page.scss']}) export class JobsPage implements OnDestroy { jobs:Job[]=[];runningJob:Job|null=null;server:ServerStatus|null=null;manager:ManagerInfo|null=null;user:AuthUser|null=null;selectedVersion='';controlBusy='';controlMessage='';updateResult:UpdateResult|null=null;updateProgress:UpdateProgress|null=null;loading=true;error='';serverError='';now=Date.now();private http=inject(HttpClient);private auth=inject(AuthService);private refreshTimer:ReturnType;private elapsedTimer:ReturnType;private updateStatusTimer:ReturnType|null=null;constructor(){this.auth.user$.subscribe(user=>this.user=user);this.refreshTimer=setInterval(()=>this.reload(false),5000);this.elapsedTimer=setInterval(()=>this.now=Date.now(),1000);this.reload(true);}ngOnDestroy():void{clearInterval(this.refreshTimer);clearInterval(this.elapsedTimer);this.stopUpdateStatusPolling();}reload(showLoading=true):void{if(showLoading)this.loading=true;this.error='';this.loadServerStatus();this.loadManager();this.http.get<{jobs:Job[];running:Job|null}>('/api/jobs').subscribe({next:x=>{this.jobs=x.jobs;this.runningJob=x.running;this.now=Date.now();this.loadAssetMetadata(this.jobs);this.loading=false;},error:e=>{this.error=e.status===502?'ComfyUI is unavailable right now.':'Unable to load jobs.';this.loading=false;}});}loadAssetMetadata(jobs:Job[]):void{for(const job of jobs)for(const asset of job.assets)this.http.get<{width:number|null;height:number|null;sizeBytes:number|null}>(asset.url.replace('/api/assets?', '/api/assets/meta?')).subscribe({next:meta=>Object.assign(asset,meta)});}loadServerStatus():void{this.serverError='';this.http.get<{status:ServerStatus}>('/api/server-status').subscribe({next:x=>this.server=x.status,error:()=>{this.server=null;this.serverError='Unable to read server status';}});}loadManager():void{this.http.get('/api/comfy/manager').subscribe({next:x=>{this.manager=x;if(!this.selectedVersion||!x.versions.includes(this.selectedVersion))this.selectedVersion=x.versions[0]||'';},error:()=>this.manager=null});}startUpdateStatusPolling():void{this.stopUpdateStatusPolling();this.updateStatusTimer=setInterval(()=>this.loadUpdateStatus(),5000);}stopUpdateStatusPolling():void{if(this.updateStatusTimer){clearInterval(this.updateStatusTimer);this.updateStatusTimer=null;}}loadUpdateStatus():void{this.http.get<{status:UpdateStatus;progress:UpdateProgress|null;result:UpdateResult|null}>('/api/comfy/update-status').subscribe({next:x=>{if(this.controlBusy!=='update-all')return;if(x.progress)this.updateProgress=x.progress;if(x.result)this.updateResult=x.result;const status=x.status;if(status.is_processing||status.total_count>status.done_count){const current=x.progress?.target?` Current task: ${x.progress.target}.`:'';this.controlMessage=`Update all in progress: ${status.done_count} of ${status.total_count} tasks complete.${current}`;}else if(x.result){this.controlMessage='Update all completed. See the results below.';this.controlBusy='';this.stopUpdateStatusPolling();this.loadManager();}},error:()=>{if(this.controlBusy==='update-all'){this.controlMessage='Update all was started, but its status is temporarily unavailable.';}}});}runControl(action:'restart'|'update'|'update-all'|'switch'):void{if(this.controlBusy||this.user?.role!=='admin')return;const labels={restart:'restart ComfyUI',update:'update ComfyUI', 'update-all':'update ComfyUI and all custom nodes',switch:`switch ComfyUI to ${this.selectedVersion}`};if(action==='switch'&&!this.selectedVersion)return;if(!window.confirm(`Are you sure you want to ${labels[action]}?`))return;this.controlBusy=action;this.controlMessage='';if(action==='update-all'){this.updateResult=null;this.updateProgress=null;}const request=action==='restart'?this.http.post('/api/comfy/restart',{}):action==='update'?this.http.post('/api/comfy/update',{}):action==='update-all'?this.http.post('/api/comfy/update-all',{}):this.http.post('/api/comfy/switch-version',{version:this.selectedVersion});request.subscribe({next:(x:any)=>{this.controlMessage=x?.message||'Request accepted.';if(action==='update-all'){this.controlMessage='Update all started. Checking progress…';this.startUpdateStatusPolling();}else{this.controlBusy='';}this.loadManager();},error:(e:any)=>{this.controlMessage=e?.error?.error||'Request failed.';this.controlBusy='';this.stopUpdateStatusPolling();}});}formatDuration(value:number|null):string{if(value===null)return 'Unavailable';const totalSeconds=Math.max(0,Math.round(value/1000));return `${Math.floor(totalSeconds/60)}m ${totalSeconds%60}s`;}formatClock(timestamp:number|null):string{if(!timestamp)return 'Unavailable';return new Date(timestamp).toLocaleTimeString([], {hour:'2-digit',minute:'2-digit',second:'2-digit'});}runningDuration():string{return this.formatDuration(this.runningJob?.queuedAt ? this.now-this.runningJob.queuedAt : this.runningJob?.durationMs ?? null);}formatBytes(value:number):string{if(!value)return '—';return `${(value/1073741824).toFixed(1)} GB`;}formatFileSize(value:number|null|undefined):string{if(!value)return '—';if(value<1024*1024)return `${(value/1024).toFixed(0)} KB`;return `${(value/1048576).toFixed(1)} MB`;}isVideo(asset:Asset):boolean{return asset.type==='video';}} +import { CommonModule } from '@angular/common'; +import { Component, OnDestroy, inject } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { AuthService, AuthUser } from '../auth.service'; + +interface Asset { filename: string; url: string; type: string; width?: number | null; height?: number | null; sizeBytes?: number | null; } +interface Model { nodeId: string; type: string; name: string; } +interface Progress { value: number; max: number; percent: number; } +interface Job { promptId: string; title: string; status: string; queuedAt: number | null; durationMs: number | null; outputCount: number; assets: Asset[]; models: Model[]; progress: Progress | null; } +interface Device { name: string; type: string; vramTotal: number; vramFree: number; } +interface ServerStatus { reachable: boolean; version: string; os: string; pythonVersion: string; devices: Device[]; queue: { running: number; pending: number; total: number }; } +interface ManagerInfo { managerVersion: string; currentVersion: string; versions: string[]; } +interface ModelStat { name: string; jobCount: number; } +interface ModelStats { totalJobs: number; totalModels: number; models: ModelStat[]; } +interface UpdateStatus { total_count: number; done_count: number; in_progress_count: number; is_processing: boolean; } +interface UpdateProgress { target: string; uiTarget: string; totalCount: number; doneCount: number; } +interface UpdateItem { id: string; title: string; url: string; error: string; } +interface UpdateResult { comfyui: string | null; updated: UpdateItem[]; skipped: UpdateItem[]; failed: UpdateItem[]; completedAt: number; } + +@Component({ standalone: true, imports: [CommonModule], templateUrl: './jobs.page.html', styleUrls: ['./jobs.page.scss'] }) +export class JobsPage implements OnDestroy { + jobs: Job[] = []; + runningJob: Job | null = null; + server: ServerStatus | null = null; + manager: ManagerInfo | null = null; + modelStats: ModelStats = { totalJobs: 0, totalModels: 0, models: [] }; + user: AuthUser | null = null; + selectedVersion = ''; + controlBusy = ''; + controlMessage = ''; + updateResult: UpdateResult | null = null; + updateProgress: UpdateProgress | null = null; + loading = true; + error = ''; + serverError = ''; + now = Date.now(); + private http = inject(HttpClient); + private auth = inject(AuthService); + private refreshTimer: ReturnType; + private elapsedTimer: ReturnType; + private updateStatusTimer: ReturnType | null = null; + + constructor() { + this.auth.user$.subscribe(user => this.user = user); + this.refreshTimer = setInterval(() => this.reload(false), 5000); + this.elapsedTimer = setInterval(() => this.now = Date.now(), 1000); + this.reload(true); + } + + ngOnDestroy(): void { clearInterval(this.refreshTimer); clearInterval(this.elapsedTimer); this.stopUpdateStatusPolling(); } + + reload(showLoading = true): void { + if (showLoading) this.loading = true; + this.error = ''; + this.loadServerStatus(); + this.loadManager(); + this.http.get<{ jobs: Job[]; running: Job | null }>('/api/jobs').subscribe({ + next: x => { this.jobs = x.jobs; this.runningJob = x.running; this.now = Date.now(); this.loadAssetMetadata(this.jobs); this.loadModelStats(); this.loading = false; }, + error: e => { this.error = e.status === 502 ? 'ComfyUI is unavailable right now.' : 'Unable to load jobs.'; this.loadModelStats(); this.loading = false; }, + }); + } + + loadModelStats(): void { + this.http.get('/api/model-stats').subscribe({ + next: x => this.modelStats = x, + error: () => this.modelStats = { totalJobs: 0, totalModels: 0, models: [] }, + }); + } + + loadAssetMetadata(jobs: Job[]): void { + for (const job of jobs) for (const asset of job.assets) this.http.get<{ width: number | null; height: number | null; sizeBytes: number | null }>(asset.url.replace('/api/assets?', '/api/assets/meta?')).subscribe({ next: meta => Object.assign(asset, meta) }); + } + + loadServerStatus(): void { + this.serverError = ''; + this.http.get<{ status: ServerStatus }>('/api/server-status').subscribe({ next: x => this.server = x.status, error: () => { this.server = null; this.serverError = 'Unable to read server status'; } }); + } + + loadManager(): void { + this.http.get('/api/comfy/manager').subscribe({ next: x => { this.manager = x; if (!this.selectedVersion || !x.versions.includes(this.selectedVersion)) this.selectedVersion = x.versions[0] || ''; }, error: () => this.manager = null }); + } + + startUpdateStatusPolling(): void { this.stopUpdateStatusPolling(); this.updateStatusTimer = setInterval(() => this.loadUpdateStatus(), 5000); } + stopUpdateStatusPolling(): void { if (this.updateStatusTimer) { clearInterval(this.updateStatusTimer); this.updateStatusTimer = null; } } + + loadUpdateStatus(): void { + this.http.get<{ status: UpdateStatus; progress: UpdateProgress | null; result: UpdateResult | null }>('/api/comfy/update-status').subscribe({ + next: x => { + if (this.controlBusy !== 'update-all') return; + if (x.progress) this.updateProgress = x.progress; + if (x.result) this.updateResult = x.result; + const status = x.status; + if (status.is_processing || status.total_count > status.done_count) { + const current = x.progress?.target ? ` Current task: ${x.progress.target}.` : ''; + this.controlMessage = `Update all in progress: ${status.done_count} of ${status.total_count} tasks complete.${current}`; + } else if (x.result) { + this.controlMessage = 'Update all completed. See the results below.'; + this.controlBusy = ''; + this.stopUpdateStatusPolling(); + this.loadManager(); + } + }, + error: () => { if (this.controlBusy === 'update-all') this.controlMessage = 'Update all was started, but its status is temporarily unavailable.'; }, + }); + } + + runControl(action: 'restart' | 'update' | 'update-all' | 'switch'): void { + if (this.controlBusy || this.user?.role !== 'admin') return; + const labels = { restart: 'restart ComfyUI', update: 'update ComfyUI', 'update-all': 'update ComfyUI and all custom nodes', switch: `switch ComfyUI to ${this.selectedVersion}` }; + if (action === 'switch' && !this.selectedVersion) return; + if (!window.confirm(`Are you sure you want to ${labels[action]}?`)) return; + this.controlBusy = action; + this.controlMessage = ''; + if (action === 'update-all') { this.updateResult = null; this.updateProgress = null; } + const request = action === 'restart' ? this.http.post('/api/comfy/restart', {}) : action === 'update' ? this.http.post('/api/comfy/update', {}) : action === 'update-all' ? this.http.post('/api/comfy/update-all', {}) : this.http.post('/api/comfy/switch-version', { version: this.selectedVersion }); + request.subscribe({ + next: (x: any) => { this.controlMessage = x?.message || 'Request accepted.'; if (action === 'update-all') { this.controlMessage = 'Update all started. Checking progress…'; this.startUpdateStatusPolling(); } else this.controlBusy = ''; this.loadManager(); }, + error: (e: any) => { this.controlMessage = e?.error?.error || 'Request failed.'; this.controlBusy = ''; this.stopUpdateStatusPolling(); }, + }); + } + + formatDuration(value: number | null): string { if (value === null) return 'Unavailable'; const totalSeconds = Math.max(0, Math.round(value / 1000)); return `${Math.floor(totalSeconds / 60)}m ${totalSeconds % 60}s`; } + formatClock(timestamp: number | null): string { if (!timestamp) return 'Unavailable'; return new Date(timestamp).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' }); } + runningDuration(): string { return this.formatDuration(this.runningJob?.queuedAt ? this.now - this.runningJob.queuedAt : this.runningJob?.durationMs ?? null); } + formatBytes(value: number): string { if (!value) return '—'; return `${(value / 1073741824).toFixed(1)} GB`; } + formatFileSize(value: number | null | undefined): string { if (!value) return '—'; if (value < 1024 * 1024) return `${(value / 1024).toFixed(0)} KB`; return `${(value / 1048576).toFixed(1)} MB`; } + isVideo(asset: Asset): boolean { return asset.type === 'video'; } +} diff --git a/job-store.js b/job-store.js new file mode 100644 index 0000000..90d2abd --- /dev/null +++ b/job-store.js @@ -0,0 +1,82 @@ +const fs = require('node:fs'); +const path = require('node:path'); + +function emptyState() { + return { version: 1, jobs: {} }; +} + +function readState(filePath) { + try { + const state = JSON.parse(fs.readFileSync(filePath, 'utf8')); + if (!state || typeof state !== 'object' || !state.jobs || typeof state.jobs !== 'object') return emptyState(); + return { version: 1, jobs: state.jobs }; + } catch { + return emptyState(); + } +} + +function writeState(filePath, state) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const temporaryPath = `${filePath}.tmp-${process.pid}`; + fs.writeFileSync(temporaryPath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); + fs.renameSync(temporaryPath, filePath); +} + +function normaliseJob(job) { + if (!job || !String(job.promptId || '').trim()) return null; + return { + promptId: String(job.promptId), + title: String(job.title || 'Untitled job'), + status: String(job.status || 'unknown'), + queuedAt: job.queuedAt ?? null, + completedAt: job.completedAt ?? null, + durationMs: job.durationMs ?? null, + outputCount: Number(job.outputCount) || 0, + models: Array.isArray(job.models) ? job.models.map((model) => ({ + nodeId: String(model?.nodeId || ''), + type: String(model?.type || 'model'), + name: String(model?.name || '').trim(), + })).filter((model) => model.name).filter((model, index, all) => all.findIndex((item) => item.name === model.name) === index) : [], + }; +} + +function createJobStore(filePath) { + const target = filePath || process.env.JOB_STORE_PATH || path.join(__dirname, 'data', 'job-history.json'); + + function sync(jobs) { + const state = readState(target); + const now = Date.now(); + let changed = false; + for (const candidate of jobs || []) { + const job = normaliseJob(candidate); + if (!job) continue; + const existing = state.jobs[job.promptId]; + const next = { ...(existing || {}), ...job, firstSeenAt: existing?.firstSeenAt || now }; + if (JSON.stringify(existing) !== JSON.stringify(next)) { + next.lastSeenAt = now; + state.jobs[job.promptId] = next; + changed = true; + } + } + if (changed) writeState(target, state); + return { changed, jobCount: Object.keys(state.jobs).length }; + } + + function stats() { + const state = readState(target); + const counts = new Map(); + for (const job of Object.values(state.jobs)) { + for (const model of job.models || []) { + const current = counts.get(model.name) || { name: model.name, jobCount: 0 }; + current.jobCount += 1; + counts.set(model.name, current); + } + } + const models = [...counts.values()].sort((a, b) => b.jobCount - a.jobCount || a.name.localeCompare(b.name)); + return { totalJobs: Object.keys(state.jobs).length, totalModels: models.length, models }; + } + + return { sync, stats, path: target }; +} + +module.exports = { createJobStore }; diff --git a/server.js b/server.js index d9606c9..2b99da7 100644 --- a/server.js +++ b/server.js @@ -3,6 +3,7 @@ const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); const sharp = require('sharp'); +const { createJobStore } = require('./job-store'); const app = express(); const port = Number(process.env.PORT || 7002); const dataDir = path.join(__dirname, 'data'); @@ -12,6 +13,7 @@ const indexHtml = path.join(browserDir, 'index.html'); const comfyBase = String(process.env.COMFYUI_URL || 'http://127.0.0.1:8188').replace(/\/$/, ''); const liveProgress = new Map(); const executionStarts = new Map(); +const jobStore = createJobStore(); let managerUpdateResult = null; let managerUpdateProgress = null; app.use(express.json({ limit: '1mb' })); @@ -103,12 +105,13 @@ function jobFromHistory(promptId, entry) { return { promptId, title: metadata.title || 'Untitled job', status: statusFromHistory(entry), queuedAt: startedAt, completedAt: statusTimestamp(entry, 'execution_success') || statusTimestamp(entry, 'execution_error'), durationMs: durationMs(entry, promptId), outputCount: assets.length, assets, models: modelsFromPrompt(entry), progress: progress ? { value: progress.value, max: progress.max, percent: Math.min(100, Math.round((progress.value / progress.max) * 100)) } : null }; } function jobFromQueue(item, historyEntry) { const [queueNumber, promptId, prompt, extraData] = item || []; const id = String(promptId || `queued-${queueNumber}`); return jobFromHistory(id, historyEntry || { prompt: [queueNumber, promptId, prompt || {}, extraData || {}], status: {}, outputs: {} }); } -async function listJobs() { const [history, queue] = await Promise.all([comfyJson('/history'), comfyJson('/queue')]); const jobs = Object.entries(history || {}).map(([promptId, entry]) => jobFromHistory(promptId, entry)).sort((a, b) => (b.completedAt || b.queuedAt || 0) - (a.completedAt || a.queuedAt || 0)); const runningItems = Array.isArray(queue?.queue_running) ? queue.queue_running : []; const running = runningItems.map((item) => jobFromQueue(item, history?.[String(item?.[1] || '')])).at(0) || null; return { jobs, running }; } +async function listJobs() { const [history, queue] = await Promise.all([comfyJson('/history'), comfyJson('/queue')]); const jobs = Object.entries(history || {}).map(([promptId, entry]) => jobFromHistory(promptId, entry)).sort((a, b) => (b.completedAt || b.queuedAt || 0) - (a.completedAt || a.queuedAt || 0)); const runningItems = Array.isArray(queue?.queue_running) ? queue.queue_running : []; const running = runningItems.map((item) => jobFromQueue(item, history?.[String(item?.[1] || '')])).at(0) || null; jobStore.sync(running ? [...jobs, running] : jobs); return { jobs, running }; } app.get('/api/health', (_req, res) => res.json({ ok: true, service: 'comfyui-portal-next' })); app.post('/api/login', (req, res) => { const user = users().find((item) => item.username === String(req.body?.username || '').trim()); if (!user || !user.active || hashPassword(req.body?.password, user.passwordSalt) !== user.passwordHash) return res.status(401).json({ error: 'Invalid credentials' }); const maxAge = req.body?.remember ? '; Max-Age=2592000' : ''; res.setHeader('Set-Cookie', `cp_auth=${encodeURIComponent(user.username)}; Path=/; HttpOnly; SameSite=Lax${maxAge}`); res.json({ user: { id: user.id, username: user.username, displayName: user.displayName, role: user.role } }); }); app.post('/api/logout', (_req, res) => { res.setHeader('Set-Cookie', 'cp_auth=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0'); res.json({ ok: true }); }); app.get('/api/me', (req, res) => res.json({ user: currentUser(req) })); app.get('/api/jobs', async (req, res) => { if (!requireUser(req, res)) return; try { res.json(await listJobs()); } catch (error) { res.status(502).json({ error: error.message || 'Unable to read ComfyUI jobs' }); } }); +app.get('/api/model-stats', (req, res) => { if (!requireUser(req, res)) return; res.json(jobStore.stats()); }); app.get('/api/server-status', async (req, res) => { if (!requireUser(req, res)) return; try { const [stats, queue] = await Promise.all([comfyJson('/system_stats'), comfyJson('/queue')]); res.json({ status: serverStatusFromStats(stats, queue) }); } catch (error) { res.status(502).json({ error: error.message || 'Unable to read ComfyUI server status' }); } }); app.get('/api/comfy/manager', async (req, res) => { if (!requireUser(req, res)) return; try { const [version, versions] = await Promise.all([comfyRequest('/manager/version'), comfyJson('/comfyui_manager/comfyui_versions')]); res.json({ managerVersion: String(version?.message || version || '').trim(), currentVersion: versions.current || '', versions: Array.isArray(versions.versions) ? versions.versions : [] }); } catch (error) { res.status(502).json({ error: error.message || 'Unable to read ComfyUI Manager information' }); } }); app.post('/api/comfy/restart', async (req, res) => { const user = requireUser(req, res); if (!user) return; if (user.role !== 'admin') return res.status(403).json({ error: 'Forbidden' }); try { await comfyRequest('/manager/reboot', 'POST', {}); res.json({ ok: true, message: 'ComfyUI restart requested.' }); } catch (error) { res.status(502).json({ error: error.message || 'Unable to restart ComfyUI' }); } }); @@ -123,4 +126,4 @@ app.use(express.static(browserDir, { index: false })); app.use((req, res, next) => { if (req.path.startsWith('/login') || req.path.startsWith('/api/')) return next(); if (!currentUser(req)) return res.redirect('/login'); next(); }); app.use((_req, res) => { if (!fs.existsSync(indexHtml)) return res.status(500).send('Build output missing.'); res.type('html').send(fs.readFileSync(indexHtml, 'utf8')); }); if (require.main === module) { startComfyProgressSocket(); app.listen(port, () => console.log(`ComfyUiPortal Next listening on ${port}`)); } -module.exports = { app, jobFromHistory, statusFromHistory }; +module.exports = { app, jobFromHistory, statusFromHistory, jobStore }; diff --git a/server.test.js b/server.test.js index 8c72ddb..b844664 100644 --- a/server.test.js +++ b/server.test.js @@ -3,6 +3,7 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const { app, jobFromHistory, statusFromHistory } = require('./server'); +const { createJobStore } = require('./job-store'); test('serves frontend assets before authentication redirects', async (t) => { const server = app.listen(0); @@ -27,3 +28,15 @@ test('maps completed ComfyUI history into a read-only job summary', () => { test('classifies failed history without exposing mutation paths', () => { assert.equal(statusFromHistory({ status: { messages: [['execution_error', {}]] } }), 'failed'); }); + +test('persists jobs across syncs and deduplicates model usage by prompt id', () => { + const directory = fs.mkdtempSync(path.join(__dirname, 'job-store-test-')); + const store = createJobStore(path.join(directory, 'jobs.json')); + const job = { promptId: 'prompt-1', title: 'First', status: 'completed', outputCount: 1, models: [{ nodeId: '1', type: 'Checkpoint', name: 'model-a.safetensors' }, { nodeId: '2', type: 'Lora', name: 'style.safetensors' }] }; + store.sync([job]); + store.sync([{ ...job, status: 'completed' }]); + store.sync([{ promptId: 'prompt-2', title: 'Second', status: 'completed', outputCount: 1, models: [{ nodeId: '1', type: 'Checkpoint', name: 'model-a.safetensors' }] }]); + assert.deepEqual(store.stats(), { totalJobs: 2, totalModels: 2, models: [{ name: 'model-a.safetensors', jobCount: 2 }, { name: 'style.safetensors', jobCount: 1 }] }); + assert.equal(JSON.parse(fs.readFileSync(store.path, 'utf8')).jobs['prompt-1'].promptId, 'prompt-1'); + fs.rmSync(directory, { recursive: true, force: true }); +});