-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathserver.mjs
More file actions
1271 lines (1199 loc) · 62.9 KB
/
Copy pathserver.mjs
File metadata and controls
1271 lines (1199 loc) · 62.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Franklin Canvas self-contained backend.
//
// Account API or wallet/x402 billing and image / music / video generation all
// run in this process via the @blockrun/llm SDK. Credentials stay server-side.
//
// Endpoints:
// GET /api/wallet?chain=base|solana — address, USDC balance, spend
// GET /api/wallet/transactions — per-call spend log
// POST /api/generate — { kind, prompt, model?, durationS?, lyrics?, instrumental?, imageUrl?, aspectRatio?, resolution?, generateAudio? }
// GET /api/generated/<file> — serves a generated file from ~/.franklin/web-jobs/
// GET /api/prompts — open prompt library index
// GET /api/prompts/detail?path=… — single prompt body + cover image
// GET /api/health
import http from 'node:http';
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import crypto from 'node:crypto';
import { spawn } from 'node:child_process';
import { Buffer } from 'node:buffer';
import {
LLMClient,
ImageClient,
MusicClient,
VideoClient,
SolanaLLMClient,
getOrCreateWallet,
getOrCreateSolanaWallet,
getCostLogSummary,
parsePaymentRequired,
extractPaymentDetails,
createPaymentPayload,
} from '@blockrun/llm';
import { runAgentChat, runBackendTool, describeMedia, summarizeConversation, listMemories, deleteMemory, CANVAS_TOOL_NAMES } from './agent-tools.mjs';
import { billingContext, isAccountMode, ACCOUNT_PORTAL, ACCOUNT_KEYS_URL, ACCOUNT_CREDITS_URL } from './account-auth.mjs';
const PORT = Number(process.env.PORT || 3100);
const apiUrl = process.env.BLOCKRUN_API_URL || undefined;
// Gateway origin for the manual x402 video submit+poll flow.
const GATEWAY = process.env.BLOCKRUN_API_URL || 'https://blockrun.ai/api';
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const JOBS_DIR = path.join(os.homedir(), '.franklin', 'web-jobs');
fs.mkdirSync(JOBS_DIR, { recursive: true });
// On-disk project files: each canvas (nodes+edges) is one JSON file on disk,
// so projects are portable / version-controllable / editable outside the browser.
const PROJECTS_DIR = path.join(os.homedir(), '.franklin', 'projects');
fs.mkdirSync(PROJECTS_DIR, { recursive: true });
// CORS — wide open in dev so any Vite port can talk to :3100. In production,
// set ALLOWED_ORIGINS to a comma-separated list of origins (or "*" if you
// really mean any). Anything not in the list is rejected.
const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS
? process.env.ALLOWED_ORIGINS.split(',').map((s) => s.trim()).filter(Boolean)
: null; // null => dev mode, allow any
function corsHeaders(req) {
const origin = req.headers.origin || '';
const allow = ALLOWED_ORIGINS === null
? '*'
: (ALLOWED_ORIGINS.includes('*') || ALLOWED_ORIGINS.includes(origin) ? origin : '');
return {
'Access-Control-Allow-Origin': allow,
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
'Access-Control-Allow-Headers': 'content-type',
'Vary': 'Origin',
};
}
function json(req, res, body, status = 200) {
const s = JSON.stringify(body);
res.writeHead(status, { 'Content-Type': 'application/json', ...corsHeaders(req) });
res.end(s);
}
function readBody(req) {
return new Promise((resolve, reject) => {
let data = '';
req.on('data', (c) => { data += c; });
req.on('end', () => resolve(data));
req.on('error', reject);
});
}
// Wallet helpers — Franklin-style. getOrCreateWallet() reads the SDK file
// (~/.blockrun/wallet) if it exists, otherwise mints a new EVM wallet on
// the spot and writes it to disk. Same file Franklin core uses, so the two
// products share one wallet per machine. New users skip the "go install
// Franklin first" step entirely; they just need to fund the auto-generated
// address with USDC on Base to start generating.
//
// `isNew` lets the UI tell the user "we just created this for you" the
// first time, so they know where to send funds.
function getWallet() {
try {
const w = getOrCreateWallet();
return { privateKey: w.privateKey, address: w.address, isNew: !!w.isNew };
} catch {
return { privateKey: null, address: '', isNew: false };
}
}
// Solana version of the same auto-create flow. Async because the underlying
// @solana/web3.js helpers are lazy-loaded.
async function getSolanaWallet() {
try {
const w = await getOrCreateSolanaWallet();
return { privateKey: w.privateKey, address: w.address, isNew: !!w.isNew };
} catch {
return { privateKey: null, address: '', isNew: false };
}
}
const MIME = {
'image/png': 'png',
'image/jpeg': 'jpg',
'image/webp': 'webp',
'video/mp4': 'mp4',
'video/webm': 'webm',
'audio/mpeg': 'mp3',
'audio/mp3': 'mp3',
};
// Download a remote URL to disk under JOBS_DIR/<base>.<ext>, streaming so a
// big video doesn't park in memory. Returns { ext }.
async function downloadTo(url, basePath, fallbackExt) {
const r = await fetch(url);
if (!r.ok) throw new Error(`download ${r.status}`);
const ct = (r.headers.get('content-type') || '').split(';')[0].trim();
const ext = MIME[ct] || fallbackExt || 'bin';
const buf = Buffer.from(await r.arrayBuffer());
fs.writeFileSync(`${basePath}.${ext}`, buf);
return { ext };
}
// @blockrun/llm 2.x only logs cost_log entries from LLMClient. Image / video /
// music spend is mirrored here so getCostLogSummary() sees every kind.
const SHARED_COST_LOG = path.join(os.homedir(), '.blockrun', 'cost_log.jsonl');
function appendCostLog({ endpoint, costUsd, model, wallet, kind }) {
try {
fs.mkdirSync(path.dirname(SHARED_COST_LOG), { recursive: true });
fs.appendFileSync(SHARED_COST_LOG, JSON.stringify({
ts: Date.now(),
endpoint,
cost_usd: costUsd,
model,
wallet,
client_kind: kind,
}) + '\n');
} catch { /* ignore */ }
}
function diffSpend(before, after) {
const b = before?.totalUsd ?? before?.usd ?? 0;
const a = after?.totalUsd ?? after?.usd ?? 0;
return Math.max(0, a - b);
}
function getBillingContext() {
const account = billingContext();
if (account.authMode === 'api-key') return account;
const wallet = getWallet();
if (!wallet.privateKey) throw new Error('No wallet found. Set SOLANA_WALLET_KEY or BASE_CHAIN_WALLET_KEY.');
return {
authMode: 'wallet',
...wallet,
clientOptions: { privateKey: wallet.privateKey, apiUrl },
};
}
async function generateImage(body, jobId) {
const ctx = getBillingContext();
const client = new ImageClient(ctx.clientOptions);
const opts = { model: body.model || 'google/nano-banana' };
// Map the node's aspect ratio to an output size, and pass quality through.
const IMG_SIZE = { '1:1': '1024x1024', '16:9': '1792x1024', '9:16': '1024x1792', '4:3': '1024x768', '3:4': '768x1024' };
if (body.aspectRatio && IMG_SIZE[body.aspectRatio]) opts.size = IMG_SIZE[body.aspectRatio];
if (body.quality === 'standard' || body.quality === 'hd') opts.quality = body.quality;
const before = ctx.authMode === 'wallet' ? client.getSpending?.() : undefined;
// Two reference images → multi-image fusion (the gateway's image2image `image`
// field accepts an array; the SDK forwards it verbatim). e.g. style from img1
// + subject from img2. One image → normal image-to-image. None → text-to-image.
const editImages = body.imageUrl2 ? [body.imageUrl, body.imageUrl2] : body.imageUrl;
const result = body.imageUrl
? await client.edit(body.prompt, editImages, opts)
: await client.generate(body.prompt, opts);
const after = ctx.authMode === 'wallet' ? client.getSpending?.() : undefined;
const remoteUrl = result?.data?.[0]?.url;
if (!remoteUrl) throw new Error('Image gateway returned no URL');
const { ext } = await downloadTo(remoteUrl, path.join(JOBS_DIR, jobId), 'png');
const costUsd = ctx.authMode === 'wallet' ? diffSpend(before, after) : null;
if (ctx.authMode === 'wallet') appendCostLog({ endpoint: body.imageUrl ? '/v1/images/edits' : '/v1/images/generations', costUsd, model: opts.model, wallet: ctx.address, kind: 'ImageClient' });
return { resultUrl: `/api/generated/${jobId}.${ext}`, costUsd };
}
async function generateMusic(body, jobId) {
const ctx = getBillingContext();
const client = new MusicClient(ctx.clientOptions);
const opts = { model: body.model || 'minimax/music-2.5+' };
if (body.durationS) opts.durationSeconds = body.durationS;
if (body.lyrics) opts.lyrics = body.lyrics;
if (typeof body.instrumental === 'boolean') opts.instrumental = body.instrumental;
const before = ctx.authMode === 'wallet' ? client.getSpending?.() : undefined;
const result = await client.generate(body.prompt, opts);
const after = ctx.authMode === 'wallet' ? client.getSpending?.() : undefined;
const remoteUrl = result?.data?.[0]?.url;
if (!remoteUrl) throw new Error('Music gateway returned no URL');
const { ext } = await downloadTo(remoteUrl, path.join(JOBS_DIR, jobId), 'mp3');
const costUsd = ctx.authMode === 'wallet' ? diffSpend(before, after) : null;
if (ctx.authMode === 'wallet') appendCostLog({ endpoint: '/v1/audio/generations', costUsd, model: opts.model, wallet: ctx.address, kind: 'MusicClient' });
return { resultUrl: `/api/generated/${jobId}.${ext}`, costUsd };
}
// Pull the x402 payment-required challenge out of a 402 response (header, or the
// JSON body for gateways that put it there). Returns a base64 challenge string.
async function extractChallengeHeader(response) {
let header = response.headers.get('payment-required');
if (!header) {
try {
const b = await response.clone().json();
if (b.x402 || b.accepts) header = Buffer.from(JSON.stringify(b)).toString('base64');
} catch { /* ignore */ }
}
return header || null;
}
// Sign a stored challenge into a FRESH PAYMENT-SIGNATURE header. Re-signing from
// the original (submit) challenge gives a new validity window each time, which is
// what fixes the long-video 402 loop: the gateway's "verification failed" 402
// (on an expired authorization) ships NO new challenge, so we must re-sign from
// the one we captured at submit time.
async function signChallenge(challengeHeader, endpoint, privateKey, address) {
if (!challengeHeader) return null;
const paymentRequired = parsePaymentRequired(challengeHeader);
const details = extractPaymentDetails(paymentRequired);
const payload = await createPaymentPayload(
privateKey, address, details.recipient, details.amount,
details.network || 'eip155:8453',
{
resourceUrl: details.resource?.url || endpoint,
resourceDescription: details.resource?.description || 'Franklin Canvas video',
maxTimeoutSeconds: details.maxTimeoutSeconds || 60,
extra: details.extra,
},
);
return { 'PAYMENT-SIGNATURE': payload };
}
// Sign an x402 payment-required response into a PAYMENT-SIGNATURE header.
async function signVideoPayment(response, endpoint, privateKey, address) {
return signChallenge(await extractChallengeHeader(response), endpoint, privateKey, address);
}
// Video uses async submit + poll. CRITICAL: the signed PAYMENT-SIGNATURE header
// from the 402 retry must be reused on EVERY poll GET — the gateway verifies
// identity on each poll and settles on the first completed response. (The SDK's
// VideoClient.generate auto-poll omits this header → "Poll failed: HTTP 402".)
async function generateVideo(body, jobId) {
const account = billingContext();
if (account.authMode === 'api-key') {
const client = new VideoClient(account.clientOptions);
const result = await client.generate(body.prompt, {
model: body.model || 'bytedance/seedance-2.0',
...(body.imageUrl ? { imageUrl: body.imageUrl } : {}),
...(body.imageUrl2 ? { lastFrameUrl: body.imageUrl2 } : {}),
...(Array.isArray(body.referenceImageUrls) && body.referenceImageUrls.length ? { referenceImageUrls: body.referenceImageUrls } : {}),
...(body.durationS ? { durationSeconds: body.durationS } : {}),
...(body.aspectRatio ? { aspectRatio: body.aspectRatio } : {}),
...(body.resolution ? { resolution: body.resolution } : {}),
...(typeof body.generateAudio === 'boolean' ? { generateAudio: body.generateAudio } : {}),
...(typeof body.seed === 'number' ? { seed: body.seed } : {}),
...(typeof body.watermark === 'boolean' ? { watermark: body.watermark } : {}),
...(typeof body.returnLastFrame === 'boolean' ? { returnLastFrame: body.returnLastFrame } : {}),
});
const remoteUrl = result?.data?.[0]?.url;
if (!remoteUrl) throw new Error('Video gateway returned no URL');
const { ext } = await downloadTo(remoteUrl, path.join(JOBS_DIR, jobId), 'mp4');
return { resultUrl: `/api/generated/${jobId}.${ext}`, costUsd: null };
}
const phaseT0 = Date.now();
const phase = {}; // submitMs / firstQueuedMs / firstProgressMs / completedMs / downloadMs
const { privateKey, address } = getWallet();
if (!privateKey) throw new Error('No wallet found. Run `franklin wallet init` or set BASE_CHAIN_WALLET_KEY.');
const model = body.model || 'bytedance/seedance-2.0';
const endpoint = `${GATEWAY}/v1/videos/generations`;
const reqBody = JSON.stringify({
model,
prompt: body.prompt,
...(body.imageUrl ? { image_url: body.imageUrl } : {}),
// Second image on a video job = the LAST frame (first-and-last-frame
// interpolation; Seedance only, gateway validates support).
...(body.imageUrl2 ? { last_frame_url: body.imageUrl2 } : {}),
// Omni / multi-reference images (Seedance 2.0) — character/style refs.
...(Array.isArray(body.referenceImageUrls) && body.referenceImageUrls.length
? { reference_image_urls: body.referenceImageUrls } : {}),
...(body.durationS ? { duration_seconds: body.durationS } : {}),
...(body.aspectRatio ? { aspect_ratio: body.aspectRatio } : {}),
...(body.resolution ? { resolution: body.resolution } : {}),
...(typeof body.generateAudio === 'boolean' ? { generate_audio: body.generateAudio } : {}),
});
const headers = { 'Content-Type': 'application/json', 'User-Agent': 'franklin-canvas' };
// Phase 1: submit (first POST → 402 → sign → retry with payment header).
let resp = await fetch(endpoint, { method: 'POST', headers, body: reqBody });
let paymentHeaders = null;
let submitChallenge = null; // kept so we can re-sign a fresh auth on each poll-402
if (resp.status === 402) {
submitChallenge = await extractChallengeHeader(resp);
paymentHeaders = await signChallenge(submitChallenge, endpoint, privateKey, address);
if (!paymentHeaders) throw new Error('Payment signing failed — check wallet balance.');
resp = await fetch(endpoint, { method: 'POST', headers: { ...headers, ...paymentHeaders }, body: reqBody });
}
if (!resp.ok) {
const t = await resp.text().catch(() => '');
throw new Error(`Video submit failed (${resp.status}): ${t.slice(0, 300)}`);
}
const submit = await resp.json();
if (!submit.poll_url || !paymentHeaders) {
throw new Error(`No poll_url returned: ${JSON.stringify(submit).slice(0, 200)}`);
}
const origin = new URL(GATEWAY).origin;
const pollUrl = submit.poll_url.startsWith('http') ? submit.poll_url : `${origin}${submit.poll_url}`;
// Phase 2: poll until completion. The gateway returns 402 on the poll while
// the job is still running (a settlement challenge that only resolves once
// the result is ready) — so we treat 402 as "in progress", re-sign, and keep
// polling. Settlement happens on the completed 200 response. (The SDK's
// built-in poll throws on this 402, which is why we poll manually.)
const startedAt = Date.now();
phase.submitMs = startedAt - phaseT0;
const deadline = startedAt + 20 * 60 * 1000; // Seedance cinematic/pro can run long
const logPhases = (note) => {
try {
const f = path.join(os.homedir(), '.franklin', 'video-debug.log');
fs.appendFileSync(f, `[${new Date().toISOString()}] ${model} i2v=${body.imageUrl ? 'Y' : 'N'} dur=${body.durationS ?? '-'} polls=${polls} :: submit=${(phase.submitMs ?? 0) / 1000 | 0}s queued@${phase.firstQueuedMs != null ? (phase.firstQueuedMs / 1000 | 0) + 's' : '-'} progress@${phase.firstProgressMs != null ? (phase.firstProgressMs / 1000 | 0) + 's' : '-'} completed@${phase.completedMs != null ? (phase.completedMs / 1000 | 0) + 's' : '-'} total=${(Date.now() - phaseT0) / 1000 | 0}s :: ${note}\n`);
} catch { /* ignore */ }
};
let remoteUrl;
let polls = 0;
let consec402 = 0; // consecutive failed-verification polls
let last402 = '';
while (Date.now() < deadline) {
await sleep(5000);
const pr = await fetch(pollUrl, { headers: { ...headers, ...paymentHeaders } });
let pj = {};
if (pr.status === 200 || pr.status === 202) { pj = await pr.json().catch(() => ({})); }
if (pj.status === 'queued' && phase.firstQueuedMs == null) phase.firstQueuedMs = Date.now() - phaseT0;
if (pj.status === 'in_progress' && phase.firstProgressMs == null) phase.firstProgressMs = Date.now() - phaseT0;
if (++polls % 3 === 0 || pr.status !== 202) {
console.log(`[video poll #${polls}] http=${pr.status} status=${pj.status ?? '-'} t=${Math.round((Date.now() - startedAt) / 1000)}s`);
}
if (pr.status === 200 || pr.status === 202) {
consec402 = 0;
if (pj.status === 'completed' && pj.data?.[0]?.url) { remoteUrl = pj.data[0].url; phase.completedMs = Date.now() - phaseT0; break; }
if (pj.status === 'failed') { logPhases('upstream-failed'); throw new Error(`Video failed upstream: ${JSON.stringify(pj.error || '').slice(0, 200)}`); }
// queued / in_progress → keep polling
} else if (pr.status === 402) {
consec402++;
last402 = (await pr.text().catch(() => '')).slice(0, 300);
// The poll's "verification failed" 402 ships NO fresh challenge, so re-sign
// from the SUBMIT challenge — that gives a new validity window each time, so
// a long-running job's authorization never lapses into a permanent 402 loop.
const re = await signChallenge(submitChallenge, endpoint, privateKey, address);
if (re) paymentHeaders = re;
// Fast-fail: if verification still fails after re-signing repeatedly (~1min),
// bail with the gateway's reason instead of spinning to the 20-min deadline.
if (consec402 >= 12) { logPhases(`402-LOOP ${last402}`); throw new Error(`Payment kept failing verification on poll (402 ×${consec402}); no payment taken. Gateway: ${last402}`); }
} else if (pr.status === 429 || pr.status >= 500) {
// transient → keep polling
} else {
const t = await pr.text().catch(() => '');
throw new Error(`Poll failed (${pr.status}): ${t.slice(0, 200)}`);
}
}
if (!remoteUrl) { logPhases('TIMEOUT/no-completion'); throw new Error('Video generation timed out (no completion within 20min). No payment settled.'); }
const dlStart = Date.now();
const { ext } = await downloadTo(remoteUrl, path.join(JOBS_DIR, jobId), 'mp4');
phase.downloadMs = Date.now() - dlStart;
logPhases(`OK download=${(phase.downloadMs / 1000) | 0}s`);
const costUsd = body.durationS ? body.durationS * 0.2 : 0; // estimate for the spend log
appendCostLog({ endpoint: '/v1/videos/generations', costUsd, model, wallet: address, kind: 'VideoClient' });
return { resultUrl: `/api/generated/${jobId}.${ext}`, costUsd };
}
// ── Prompt library ──
// Sourced from BlockRun's Prompt-Case-Hub, which aggregates several public prompt
// repos into ~848 cases in one unified format (each case = one markdown file with
// YAML front-matter + a fenced ```prompt body; see the hub's FORMAT.md).
//
// The catalog (titles + metadata) lives in cases/index.json, fetched once. The
// full prompt body for a case is fetched on demand (when the user clicks "Use")
// so we never pull 848 files up front.
const CASE_LIB_BASE = 'https://raw.githubusercontent.com/BlockRunAI/Prompt-Case-Hub/main';
const PROMPT_TTL_MS = 6 * 60 * 60 * 1000; // 6h
let promptCache = { at: 0, items: [] };
// Map a Prompt-Case-Hub index.json entry → the PromptItem shape the frontend
// expects. The index is denormalized (modality/workflow/reference_images/model/
// preview) so we can build the whole catalog without fetching any case body.
function indexEntryToItem(c) {
const tags = Array.isArray(c.tags) ? c.tags : [];
return {
id: c.id,
title: c.title,
titleCn: '',
category: tags[0] || c.workflow,
workflow: c.workflow,
model: c.model || '',
tags,
prompt: '', // filled on demand via /api/prompts/detail
image: c.preview || '',
path: c.file, // e.g. "cases/awesome-gpt-image-2-4.md"
needsRef: (c.reference_images || 0) > 0,
source: 'prompt-case-hub',
};
}
async function getPromptLibrary() {
if (promptCache.items.length && Date.now() - promptCache.at < PROMPT_TTL_MS) {
return promptCache.items;
}
const r = await fetch(`${CASE_LIB_BASE}/cases/index.json`, { headers: { 'user-agent': 'franklin-canvas' } });
if (!r.ok) return promptCache.items;
const data = await r.json();
const items = (data.cases || []).map(indexEntryToItem);
if (items.length) promptCache = { at: Date.now(), items };
return items;
}
// Replace {argument name="..." default="VALUE"} (quotes may be JSON-escaped
// as \") with VALUE; drop any argument tag that has no default.
function resolveArguments(text) {
let out = text;
// with default
out = out.replace(/\{\s*argument\b[^}]*?default=\\?"((?:[^"\\]|\\.)*?)\\?"[^}]*?\}/g, (_, v) => v.replace(/\\"/g, '"'));
// leftover argument tags without a default → remove
out = out.replace(/\{\s*argument\b[^}]*?\}/g, '');
return out;
}
// Fetch + parse a single unified-format case file: pull the fenced `prompt`
// body and a preview image. Files live under cases/ in Prompt-Case-Hub.
async function getPromptDetail(relPath) {
if (!relPath || relPath.includes('..') || !relPath.startsWith('cases/') || !relPath.endsWith('.md')) {
throw new Error('bad path');
}
const r = await fetch(`${CASE_LIB_BASE}/${relPath}`, { headers: { 'user-agent': 'franklin-canvas' } });
if (!r.ok) throw new Error(`case ${r.status}`);
const raw = await r.text();
// Front-matter: title + preview image.
let title = '';
let image = '';
const fm = raw.match(/^---\n([\s\S]*?)\n---\n/);
if (fm) {
const t = fm[1].match(/^title:\s*"?(.+?)"?\s*$/m);
if (t) title = t[1];
const pv = fm[1].match(/^preview:\s*"?(https?:\/\/[^"\s]+)"?/m);
if (pv) image = pv[1];
}
if (!image) {
const imgM = raw.match(/!\[[^\]]*\]\((https?:\/\/[^)]+)\)/);
if (imgM) image = imgM[1];
}
// Prompt: the fenced ```prompt block. The fence can be 3+ backticks (longer
// when the prompt itself contains ```), so capture the opening run and match
// the same run as the close.
const pm = raw.match(/^(`{3,})prompt[ \t]*\n([\s\S]*?)\n\1[ \t]*$/m);
let prompt = pm ? pm[2] : '';
// Resolve any residual {argument name="x" default="y"} → "y" (cases are
// pre-resolved at migration, but keep this defensive for new imports).
prompt = resolveArguments(prompt).trim();
return { title, prompt, image };
}
// ── Comparison stitch (ffmpeg) ──
// Composite N model videos into one grid MP4 with per-cell labels. Label PNGs
// are rendered by the browser (proper fonts, no backend font deps) and passed
// in as data: URLs. Layouts: 2→1×2, 3→1×3, 4→2×2, 5→2×3 (last cell black).
// Cell at 960×540 keeps each clip near its source 720p (2×2 → 1080p output)
// instead of the old 640×360 quarter-res cells.
const CMP_CELL_W = 960;
const CMP_CELL_H = 540;
const CMP_LAYOUTS = { 2: [2, 1], 3: [3, 1], 4: [2, 2], 5: [3, 2] };
function runFfmpeg(args) {
return new Promise((resolve, reject) => {
const ff = spawn('ffmpeg', args);
let err = '';
ff.stderr.on('data', (d) => { err += d.toString(); });
ff.on('error', reject);
ff.on('close', (code) => (code === 0 ? resolve() : reject(new Error('ffmpeg failed: ' + err.slice(-500)))));
});
}
// Map a /api/generated/<file> URL to its local path; remote URLs return null
// (the caller downloads them into the temp dir).
function localGeneratedPath(url) {
if (typeof url !== 'string') throw new Error('bad video url');
if (url.startsWith('/api/generated/')) {
const f = path.basename(url.slice('/api/generated/'.length).split('?')[0]);
const fp = path.join(JOBS_DIR, f);
if (!fs.existsSync(fp)) throw new Error('source video not found');
return fp;
}
return null;
}
function probeDuration(file) {
return new Promise((resolve) => {
const pp = spawn('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nw=1:nk=1', file]);
let out = '';
pp.stdout.on('data', (d) => { out += d.toString(); });
pp.on('close', () => resolve(parseFloat(out) || 0));
pp.on('error', () => resolve(0));
});
}
function probeHasAudio(file) {
return new Promise((resolve) => {
const pp = spawn('ffprobe', ['-v', 'error', '-select_streams', 'a', '-show_entries', 'stream=index', '-of', 'csv=p=0', file]);
let out = '';
pp.stdout.on('data', (d) => { out += d.toString(); });
pp.on('close', () => resolve(out.trim().length > 0));
pp.on('error', () => resolve(false));
});
}
function probeDimensions(file) {
return new Promise((resolve) => {
const pp = spawn('ffprobe', ['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height', '-of', 'csv=p=0', file]);
let out = '';
pp.stdout.on('data', (d) => { out += d.toString(); });
pp.on('close', () => { const [w, h] = out.trim().split(',').map(Number); resolve({ w: w || 0, h: h || 0 }); });
pp.on('error', () => resolve({ w: 0, h: 0 }));
});
}
// mode 'grid' (default): all cells play simultaneously.
// mode 'sequence': cells play one at a time in order — the active cell plays
// while the others hold a frozen frame (first frame before their turn, last
// frame after) — the classic side-by-side model-comparison reel.
async function stitchComparison(items, mode = 'grid', orientation = 'landscape', labelPos = { x: 0.02, y: 0.03 }) {
const n = items.length;
if (n < 2 || n > 5) throw new Error('need 2–5 videos');
// landscape → grid (2×2 etc); portrait → single column stacked top-to-bottom
// for a TikTok-style reel.
const portrait = orientation === 'portrait';
const cols = portrait ? 1 : CMP_LAYOUTS[n][0];
const rows = portrait ? n : CMP_LAYOUTS[n][1];
// Portrait fits the stack into a 9:16 phone frame (720×1280): each clip gets
// a 1280/N-tall band (letterboxed) so the whole thing stays phone-shaped
// instead of growing endlessly long. Landscape uses the 640×360 grid cell.
// Dimensions must be even for yuv420p/libx264.
// Portrait fills a 1080×1920 phone frame; each clip gets a 1920/N band.
const cellW = portrait ? 1080 : CMP_CELL_W;
const cellH = portrait ? (Math.round(1920 / n) - (Math.round(1920 / n) % 2)) : CMP_CELL_H;
const jobId = `cmp_${crypto.randomUUID()}`;
const tmp = path.join(JOBS_DIR, jobId + '_tmp');
fs.mkdirSync(tmp, { recursive: true });
try {
// Resolve videos to local files (download remotes); write label PNGs.
const videoPaths = [];
const labelPaths = [];
for (let i = 0; i < n; i++) {
const it = items[i] || {};
let vp = localGeneratedPath(it.url);
if (!vp) { const { ext } = await downloadTo(it.url, path.join(tmp, `v${i}`), 'mp4'); vp = path.join(tmp, `v${i}.${ext}`); }
videoPaths.push(vp);
const m = /^data:image\/png;base64,(.+)$/s.exec(it.labelPng || '');
if (m) { const lp = path.join(tmp, `l${i}.png`); fs.writeFileSync(lp, Buffer.from(m[1], 'base64')); labelPaths.push(lp); }
else labelPaths.push(null);
}
// Sequence mode needs a fixed per-cell slot length = the longest clip.
let slot = 0;
if (mode === 'sequence') {
const durs = await Promise.all(videoPaths.map(probeDuration));
slot = Math.max(2, Math.round(Math.max(...durs, 0) * 100) / 100) || 5;
}
// Inputs: videos first (0..n-1), then any label PNGs. Sequence caps each
// video to one slot with -t so cells stay aligned.
const inputs = [];
let nInputs = 0;
for (const v of videoPaths) {
if (mode === 'sequence') inputs.push('-t', String(slot));
inputs.push('-i', v); nInputs++;
}
const labelIdx = [];
for (let i = 0; i < n; i++) {
if (labelPaths[i]) { labelIdx[i] = nInputs; inputs.push('-i', labelPaths[i]); nInputs++; }
else labelIdx[i] = -1;
}
// Watermark position as a fraction of the free space in each cell: (0,0) =
// top-left, (1,1) = bottom-right. ffmpeg's overlay (W-w)*px keeps the label
// fully inside the cell at every position. Matches the draggable preview.
const px = Math.min(1, Math.max(0, Number(labelPos?.x) || 0));
const py = Math.min(1, Math.max(0, Number(labelPos?.y) || 0));
const overlayXY = `x='(W-w)*${px.toFixed(4)}':y='(H-h)*${py.toFixed(4)}'`;
const fit = `scale=${cellW}:${cellH}:force_original_aspect_ratio=decrease,pad=${cellW}:${cellH}:(ow-iw)/2:(oh-ih)/2,setsar=1`;
// Per-cell: fit into the cell (+ sequence timing), then overlay its label.
let fc = '';
for (let i = 0; i < n; i++) {
if (mode === 'sequence') {
const start = (i * slot).toFixed(3);
const stop = ((n - 1 - i) * slot).toFixed(3);
fc += `[${i}:v]${fit},tpad=stop_mode=clone:stop_duration=${slot},trim=0:${slot},setpts=PTS-STARTPTS,tpad=start_duration=${start}:start_mode=clone:stop_duration=${stop}:stop_mode=clone[s${i}];`;
} else {
fc += `[${i}:v]${fit}[s${i}];`;
}
fc += labelIdx[i] >= 0 ? `[s${i}][${labelIdx[i]}:v]overlay=${overlayXY}[c${i}];` : `[s${i}]null[c${i}];`;
}
// Black fillers for any empty cells in the grid.
const total = cols * rows;
for (let i = n; i < total; i++) fc += `color=c=#101012:s=${cellW}x${cellH}:r=24[c${i}];`;
// Build rows, then stack rows. A single-column (portrait) layout needs no
// hstack — each cell is its own row.
const rowLabels = [];
for (let r = 0; r < rows; r++) {
if (cols === 1) { rowLabels.push(`c${r}`); continue; }
const cells = [];
for (let c = 0; c < cols; c++) cells.push(`[c${r * cols + c}]`);
fc += `${cells.join('')}hstack=inputs=${cols}[row${r}];`;
rowLabels.push(`row${r}`);
}
let outLabel = rowLabels[0];
if (rows > 1) { fc += `${rowLabels.map((l) => `[${l}]`).join('')}vstack=inputs=${rows}[out];`; outLabel = 'out'; }
// Sequence mode: lay each clip's audio into its own time slot so you hear
// the active model during its turn. Clips without an audio track are simply
// skipped from the mix.
const audioArgs = [];
if (mode === 'sequence') {
const hasAudio = await Promise.all(videoPaths.map(probeHasAudio));
const withAudio = [];
for (let i = 0; i < n; i++) {
if (!hasAudio[i]) continue;
fc += `[${i}:a]atrim=0:${slot},asetpts=PTS-STARTPTS,adelay=${Math.round(i * slot * 1000)}:all=1,apad=whole_dur=${(n * slot).toFixed(3)}[a${i}];`;
withAudio.push(i);
}
if (withAudio.length === 1) {
fc += `[a${withAudio[0]}]anull[aout];`;
audioArgs.push('-map', '[aout]', '-c:a', 'aac');
} else if (withAudio.length > 1) {
fc += `${withAudio.map((i) => `[a${i}]`).join('')}amix=inputs=${withAudio.length}:normalize=0:duration=longest[aout];`;
audioArgs.push('-map', '[aout]', '-c:a', 'aac');
} else {
audioArgs.push('-an');
}
} else {
audioArgs.push('-an');
}
const outPath = path.join(JOBS_DIR, `${jobId}.mp4`);
await runFfmpeg(['-y', ...inputs, '-filter_complex', fc, '-map', `[${outLabel}]`, ...audioArgs, '-c:v', 'libx264', '-preset', 'medium', '-crf', '19', '-movflags', '+faststart', '-pix_fmt', 'yuv420p', outPath]);
return { resultUrl: `/api/generated/${jobId}.mp4` };
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
}
// Concatenate clips end-to-end into ONE continuous full-frame film (vs the grid
// comparison above). Each clip is scaled+padded into a common frame (taken from
// the first clip, capped at 1280 wide, even dims) so mismatched sizes/orientations
// still join cleanly. Audio is concatenated when EVERY clip has a track; otherwise
// the film is silent (concat needs matching stream counts).
async function concatFilms(items, music) {
// Each item is either a bare url/string, or { url, inS?, durationS? } where
// inS/durationS trim the clip (seconds). Keep the trims aligned with urls.
const list = (Array.isArray(items) ? items : [])
.map((it) => (typeof it === 'string' ? { url: it } : it))
.filter((it) => it && it.url);
const urls = list.map((it) => it.url);
const n = urls.length;
if (n < 2) throw new Error('need at least 2 clips to assemble a film');
// Optional music lane: one or more audio clips (trimmable) laid back-to-back
// and mixed under the whole film as a soundtrack.
const musicList = (Array.isArray(music) ? music : music ? [music] : [])
.map((it) => (typeof it === 'string' ? { url: it } : it))
.filter((it) => it && it.url);
const jobId = `film_${crypto.randomUUID()}`;
const tmp = path.join(JOBS_DIR, jobId + '_tmp');
fs.mkdirSync(tmp, { recursive: true });
try {
const videoPaths = [];
for (let i = 0; i < n; i++) {
let vp = localGeneratedPath(urls[i]);
if (!vp) { const { ext } = await downloadTo(urls[i], path.join(tmp, `v${i}`), 'mp4'); vp = path.join(tmp, `v${i}.${ext}`); }
videoPaths.push(vp);
}
const musicPaths = [];
for (let i = 0; i < musicList.length; i++) {
let mp = localGeneratedPath(musicList[i].url);
if (!mp) { const { ext } = await downloadTo(musicList[i].url, path.join(tmp, `m${i}`), 'mp3'); mp = path.join(tmp, `m${i}.${ext}`); }
musicPaths.push(mp);
}
// Frame size from the first clip (even, ≤1280 wide).
const dim = await probeDimensions(videoPaths[0]);
let W = Math.min(1280, dim.w || 1280); W -= W % 2;
let H = dim.w ? Math.round(W * (dim.h / dim.w)) : 720; H -= H % 2;
const hasAudio = await Promise.all(videoPaths.map(probeHasAudio));
const allAudio = hasAudio.every(Boolean);
// Per-clip trim → ffmpeg trim/atrim filters. A clip with no trim plays whole.
const trimOf = (it) => {
const inS = Math.max(0, Number(it?.inS) || 0);
const durS = Number(it?.durationS);
const dur = Number.isFinite(durS) && durS > 0 ? durS : null;
return { inS, dur };
};
// Film length = sum of each video clip's effective (trimmed) duration —
// probe the untrimmed ones so the soundtrack can be fit to the total.
let filmDur = 0;
for (let i = 0; i < n; i++) {
const { inS, dur } = trimOf(list[i]);
if (dur != null) filmDur += dur;
else { const full = await probeDuration(videoPaths[i]).catch(() => 0); filmDur += Math.max(0, (full || 0) - inS); }
}
const inputs = [];
for (const v of videoPaths) inputs.push('-i', v);
for (const m of musicPaths) inputs.push('-i', m);
const musicBase = n; // music input indices start after the video inputs
const fit = `scale=${W}:${H}:force_original_aspect_ratio=decrease,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=30,format=yuv420p`;
let fc = '';
const parts = [];
for (let i = 0; i < n; i++) {
const { inS, dur } = trimOf(list[i]);
const vtrim = (inS > 0 || dur != null)
? `trim=start=${inS}${dur != null ? `:duration=${dur}` : ''},setpts=PTS-STARTPTS,`
: '';
fc += `[${i}:v]${vtrim}${fit}[v${i}];`;
if (allAudio) {
const atrim = (inS > 0 || dur != null)
? `atrim=start=${inS}${dur != null ? `:duration=${dur}` : ''},asetpts=PTS-STARTPTS,`
: '';
fc += `[${i}:a]${atrim}aresample=async=1:first_pts=0[a${i}];`;
parts.push(`[v${i}][a${i}]`);
} else parts.push(`[v${i}]`);
}
const hasMusic = musicPaths.length > 0;
fc += `${parts.join('')}concat=n=${n}:v=1:a=${allAudio ? 1 : 0}[outv]${allAudio ? '[filmA]' : ''};`;
// Build the soundtrack: concat the music clips, then fit to the film length
// (pad if short, trim if long) so it spans the whole cut.
let outAudioLabel = null;
if (hasMusic) {
const mParts = [];
for (let i = 0; i < musicPaths.length; i++) {
const { inS, dur } = trimOf(musicList[i]);
const at = (inS > 0 || dur != null)
? `atrim=start=${inS}${dur != null ? `:duration=${dur}` : ''},`
: '';
fc += `[${musicBase + i}:a]${at}asetpts=PTS-STARTPTS,aresample=async=1:first_pts=0[m${i}];`;
mParts.push(`[m${i}]`);
}
if (musicPaths.length > 1) fc += `${mParts.join('')}concat=n=${musicPaths.length}:v=0:a=1[mcat];`;
else fc += `${mParts[0]}anull[mcat];`;
fc += `[mcat]apad,atrim=duration=${filmDur.toFixed(3)},asetpts=PTS-STARTPTS[music];`;
if (allAudio) {
// Duck the soundtrack under the clips' own audio, then mix.
fc += `[music]volume=0.65[musicv];[filmA][musicv]amix=inputs=2:duration=first:dropout_transition=0[outa]`;
outAudioLabel = '[outa]';
} else outAudioLabel = '[music]';
} else if (allAudio) {
outAudioLabel = '[filmA]';
}
// Trim trailing ';' when no extra audio graph was appended.
fc = fc.replace(/;$/, '');
const map = ['-map', '[outv]'];
if (outAudioLabel) map.push('-map', outAudioLabel, '-c:a', 'aac');
else map.push('-an');
const outPath = path.join(JOBS_DIR, `${jobId}.mp4`);
await runFfmpeg(['-y', ...inputs, '-filter_complex', fc, ...map, '-c:v', 'libx264', '-preset', 'medium', '-crf', '20', '-movflags', '+faststart', '-pix_fmt', 'yuv420p', outPath]);
return { resultUrl: `/api/generated/${jobId}.mp4` };
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
}
// ── Agent: workflow planner ──
// A lightweight planner (not a full tool-calling loop): given the user's idea,
// the LLM returns a JSON media-workflow plan that the frontend then builds and
// runs node-by-node on the canvas (with a per-step cost confirm). Tools are the
// canvas node types — image / video / music generation — chained by `from`.
const AGENT_PLAN_MODEL = 'anthropic/claude-sonnet-4.6';
// Text models the agent planner may use (mirror TEXT_MODELS in nodes.tsx).
const AGENT_TEXT_MODELS = new Set([
'anthropic/claude-haiku-4.5', 'anthropic/claude-sonnet-4.6', 'anthropic/claude-opus-4.7',
'anthropic/claude-opus-4.8', 'openai/gpt-5.5', 'google/gemini-3.1-pro',
'google/gemini-2.5-flash', 'deepseek/deepseek-v4-pro',
]);
// Mirror the frontend catalogs (src/canvas/nodes.tsx) so the planner only
// picks model ids the gateway actually serves.
const IMAGE_MODEL_IDS = 'google/nano-banana, google/nano-banana-pro, openai/gpt-image-1, openai/gpt-image-2, xai/grok-imagine-image, zai/cogview-4';
const VIDEO_MODEL_IDS = 'xai/grok-imagine-video, bytedance/seedance-1.5-pro, azure/sora-2, bytedance/seedance-2.0-fast, bytedance/seedance-2.0';
const MUSIC_MODEL_IDS = 'minimax/music-2.5+';
const AGENT_SYSTEM = `You are a media-workflow planner inside a node-based AI studio. The user describes something they want to create (usually a short video). You design a small workflow of generation steps that the app will build visually on an infinite canvas and run one by one.
Available step types (these are the canvas node types / tools):
- "imagegen": generate an image from a text prompt (or from a previous image when "from" is set → image edit).
- "videogen": generate a video. With "from" set to an image step, it animates that image (image→video); without "from", it's text→video.
- "musicgen": generate a background music/audio clip from a text prompt.
Available model ids (pick the most fitting; omit to use a sensible default):
- image: ${IMAGE_MODEL_IDS}
- video: ${VIDEO_MODEL_IDS}
- music: ${MUSIC_MODEL_IDS}
Chaining: each step has a unique short "id" (s1, s2, …). Set "from" to a prior step's id to feed that step's OUTPUT as the input of this step (e.g. imagegen → videogen animates the image). A common pattern: s1 imagegen (establish the look) → s2 videogen from s1 (animate it) → optionally s3 musicgen for a soundtrack.
Rules:
- Keep it to 2–4 steps. Prefer one clear chain.
- Write rich, specific prompts (lighting, motion, style, mood) — they drive real generations.
- For video/music steps include "durationS" (3–10).
- Respond with ONLY a JSON object, no markdown fences, of the shape:
{"message":"<one or two friendly sentences explaining the plan>","steps":[{"id":"s1","type":"imagegen","title":"<short label>","prompt":"<detailed prompt>","model":"<id or omit>","from":null,"durationS":null}]}`;
function extractJsonObject(text) {
let t = (text || '').trim();
// strip ```json … ``` fences if present
const fence = t.match(/```(?:json)?\s*([\s\S]*?)```/i);
if (fence) t = fence[1].trim();
try { return JSON.parse(t); } catch { /* fall through to brace scan */ }
const start = t.indexOf('{');
const end = t.lastIndexOf('}');
if (start >= 0 && end > start) return JSON.parse(t.slice(start, end + 1));
throw new Error('no JSON in model output');
}
async function planWorkflow(prompt, history, model) {
const ctx = getBillingContext();
const client = new LLMClient(ctx.clientOptions);
const planModel = typeof model === 'string' && AGENT_TEXT_MODELS.has(model) ? model : AGENT_PLAN_MODEL;
const messages = [{ role: 'system', content: AGENT_SYSTEM }];
for (const h of (Array.isArray(history) ? history : []).slice(-6)) {
if (h && (h.role === 'user' || h.role === 'assistant') && typeof h.content === 'string') {
messages.push({ role: h.role, content: h.content });
}
}
messages.push({ role: 'user', content: String(prompt) });
const resp = await client.chatCompletion(planModel, messages, { maxTokens: 1400, temperature: 0.5 });
const text = resp.choices?.[0]?.message?.content || '';
const plan = extractJsonObject(text);
// Sanitize steps.
const okTypes = new Set(['imagegen', 'videogen', 'musicgen']);
const steps = (Array.isArray(plan.steps) ? plan.steps : [])
.filter((s) => s && okTypes.has(s.type) && typeof s.prompt === 'string' && s.prompt.trim())
.slice(0, 5)
.map((s, i) => ({
id: typeof s.id === 'string' && s.id ? s.id : `s${i + 1}`,
type: s.type,
title: typeof s.title === 'string' ? s.title.slice(0, 60) : '',
prompt: s.prompt.trim(),
model: typeof s.model === 'string' ? s.model : undefined,
from: typeof s.from === 'string' ? s.from : null,
durationS: Number.isFinite(s.durationS) ? Math.max(3, Math.min(10, s.durationS)) : undefined,
}));
// Drop dangling "from" references.
const ids = new Set(steps.map((s) => s.id));
for (const s of steps) if (s.from && !ids.has(s.from)) s.from = null;
return { message: typeof plan.message === 'string' ? plan.message : 'Here is a workflow for that.', steps };
}
// ── HTTP routing ───────────────────────────────────────────────────────
const server = http.createServer(async (req, res) => {
if (req.method === 'OPTIONS') { res.writeHead(204, corsHeaders(req)); res.end(); return; }
const p = (req.url || '').split('?')[0];
try {
if (p === '/api/wallet' && req.method === 'GET') {
if (isAccountMode()) {
// Validate configuration without returning or logging the credential.
billingContext();
return json(req, res, {
authMode: 'api-key', address: '', balanceUsdc: null,
recentSpendUsd: null, totalSpendUsd: null,
network: 'Account API', chain: 'account', isNew: false,
spendByCategory: [], portalUrl: ACCOUNT_PORTAL,
keysUrl: ACCOUNT_KEYS_URL, creditsUrl: ACCOUNT_CREDITS_URL,
});
}
// Optional ?chain=solana|base — defaults to Solana. Spend history is
// shared across chains (it lives in the BlockRun cost log) and isn't
// chain-tagged, so both branches return the same recent/total/byModel
// figures — only the wallet address + on-chain balance differ.
const url = new URL(req.url, 'http://localhost');
const chain = (url.searchParams.get('chain') || 'solana').toLowerCase() === 'base' ? 'base' : 'solana';
try {
const wallet = chain === 'solana' ? await getSolanaWallet() : getWallet();
const { address, privateKey, isNew } = wallet;
let balanceUsdc = 0;
let recentSpendUsd = 0; // true rolling 24h
let totalSpendUsd = 0; // all-time
let spendByCategory = [];
try {
// Compute spend directly from the shared cost log so "24h" is really
// 24h (getCostLogSummary returns an all-time total). Timestamps are
// seconds (SDK) or ms (us); normalize to ms.
if (fs.existsSync(SHARED_COST_LOG)) {
const now = Date.now();
const byModel = new Map();
for (const line of fs.readFileSync(SHARED_COST_LOG, 'utf8').split('\n')) {
if (!line.trim()) continue;
let e; try { e = JSON.parse(line); } catch { continue; }
const cost = Number(e.cost_usd) || 0;
let ts = Number(e.ts) || 0;
if (ts > 0 && ts < 1e12) ts *= 1000;
totalSpendUsd += cost;
if (ts && now - ts <= 24 * 60 * 60 * 1000) recentSpendUsd += cost;
const m = e.model || 'unknown';
byModel.set(m, (byModel.get(m) || 0) + cost);
}
spendByCategory = [...byModel.entries()]
.map(([category, usd]) => ({ category, usd }))
.sort((a, b) => b.usd - a.usd);
}
} catch { /* ignore */ }
try {
if (privateKey) {
// getBalance() lives on both LLMClient (Base) and SolanaLLMClient
// (Solana) and resolves to a USDC float.
const c = chain === 'solana'
? new SolanaLLMClient({ privateKey })
: new LLMClient({ privateKey, apiUrl });
const bal = await c.getBalance();
if (typeof bal === 'number') balanceUsdc = bal;
}
} catch { /* ignore — show 0 if balance lookup fails */ }
const network = chain === 'solana' ? 'Solana' : 'Base';
return json(req, res, { address, balanceUsdc, recentSpendUsd, totalSpendUsd, network, chain, isNew, spendByCategory });
} catch (err) {
const network = chain === 'solana' ? 'Solana' : 'Base';
return json(req, res, { address: '', balanceUsdc: 0, recentSpendUsd: 0, totalSpendUsd: 0, network, chain, isNew: false, spendByCategory: [], error: String(err) });
}
}
if (p === '/api/wallet/transactions' && req.method === 'GET') {
if (isAccountMode()) return json(req, res, []);
try {
const logPath = SHARED_COST_LOG;
if (!fs.existsSync(logPath)) return json(req, res, []);
const lines = fs.readFileSync(logPath, 'utf8').trim().split('\n').filter(Boolean);
const txs = lines
.map((l) => { try { return JSON.parse(l); } catch { return null; } })
.filter(Boolean)
// The cost log is SHARED across all BlockRun tools. Show generations
// (image/video/music) AND language-model calls (/v1/messages, /v1/chat).
.filter((e) => /\/(images|videos|audio)\/|\/(messages|chat)/.test(e.endpoint || ''))
.map((e, i) => {
// Normalize timestamps: SDK logs seconds (float), we log ms. Anything
// below 1e12 is seconds → ×1000.
const rawTs = Number(e.ts) || 0;
const tsMs = rawTs > 0 ? (rawTs < 1e12 ? Math.round(rawTs * 1000) : rawTs) : Date.now();
const ep = e.endpoint || '';
const kind = ep.includes('/videos/') ? 'Video'
: ep.includes('/images/') ? 'Image'
: ep.includes('/audio/') ? 'Music'
: /\/(messages|chat)/.test(ep) ? 'Text' : 'Generation';
return {
id: `${rawTs}-${i}`,
ts: tsMs,
type: 'spend',
amountUsd: e.cost_usd ?? 0,
description: `${kind} · ${e.model || ''}`.trim().replace(/·\s*$/, '').trim(),
model: e.model,
};
})
.sort((a, b) => b.ts - a.ts)
.slice(0, 100);
return json(req, res, txs);
} catch {
return json(req, res, []);
}
}
if (p === '/api/generate' && req.method === 'POST') {
const raw = await readBody(req);
const body = JSON.parse(raw);
if (!body.prompt) return json(req, res, { ok: false, error: 'prompt required' }, 400);
if (!['image', 'video', 'music'].includes(body.kind)) {
return json(req, res, { ok: false, error: 'kind must be image|video|music' }, 400);
}
const jobId = `${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
const t0 = Date.now();
try {
const result = body.kind === 'image' ? await generateImage(body, jobId)
: body.kind === 'music' ? await generateMusic(body, jobId)
: await generateVideo(body, jobId);
const ms = Date.now() - t0;
const cost = result.costUsd != null ? `$${result.costUsd.toFixed(4)}` : '?';
console.log(`[generate] ${body.kind} ${body.model || 'default'} ok ${ms}ms ${cost}`);
return json(req, res, { ok: true, ...result });
} catch (err) {
const ms = Date.now() - t0;
console.warn(`[generate] ${body.kind} ${body.model || 'default'} FAIL ${ms}ms: ${err.message || err}`);