Repository navigation
Expand file tree
/
Copy pathpublish-content.ts
More file actions
1939 lines (1791 loc) · 85.3 KB
/
Copy pathpublish-content.ts
File metadata and controls
1939 lines (1791 loc) · 85.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import { readFileSync, readdirSync, statSync, existsSync, appendFileSync } from 'node:fs';
import { spawnSync as _spawnSync } from 'node:child_process';
import { join } from 'node:path';
import { createHash } from 'node:crypto';
import { gzipSync } from 'node:zlib';
import { userInfo, hostname } from 'node:os';
import { parse as parseYaml } from 'yaml';
import { parseChannel, type Channel } from './fetch-tutorials.js';
import { beginSession, appendBatch, commitSession, abortSession, fetchRemoteHashes, fetchRemoteSourceHashes, renderConceptsPhase, renderTopicsPhase, renderChannelsPhase } from './lib/publish-client.js';
import { computePipelineToken } from './lib/pipeline-token.js';
import { withRetry, formatErrorChain } from './lib/publish-retry.js';
import { chunk, runConcurrent } from './lib/publish-batcher.js';
import { collectCodeCheckSpecs, publishCodeCheckSpecs } from './lib/publish-codecheck.js';
import { collectAssertSpecs, publishAssertSpecs } from './lib/publish-asserts.js';
import { publishValidateAnswerSpecs } from './lib/publish-validate-answer.js';
import { publishChallengeAnswers } from './lib/publish-challenge-answers.js';
import { publishContributors } from './publish/publish-contributors.js';
import { publishValidationRules } from './publish/publish-validation-rules.js';
import { computeOrphans, enforceCap, formatStepSummary } from './lib/purge-orphans.js';
import { discoverPageFiles, discoverAuthorPages, discoverAdvocatePages } from '../srv/lib/page-key-map.js';
export type { Channel };
/**
* Coerce arbitrary values to a trimmed non-empty string, or null. Used by
* frontmatter extractors that may receive missing keys, numbers, or empties.
*/
const trim = (v: unknown): string | null =>
typeof v === 'string' && v.trim().length > 0 ? v.trim() : null;
export interface PublishConfig {
baseUrl: string;
apiKey: string | undefined;
sourceDir: string;
force: boolean;
}
export function resolvePublishConfig({ channel }: { channel: Channel }): PublishConfig {
if (channel === 'qa') {
return {
baseUrl: process.env.CAP_QA_BASE_URL ?? 'http://localhost:4005',
apiKey: process.env.CONTENT_API_KEY_QA,
sourceDir: 'hugo/public-qa',
// Delta by default (mirrors prod), so a commit-triggered QA rebuild that
// sets PUBLISH_SLUG publishes only the changed slug — a sub-second commit
// instead of a ~35s full-catalog force write that intermittently 502/503s
// the single srv-qa instance. `force-publish=true` (workflow input → --force)
// still does a full re-seed. Safe now that QA has Option B fully active
// (mutable ContentCurrent, O(changed) publish, server carries unchanged slugs).
force: process.argv.includes('--force'),
};
}
return {
baseUrl: process.env.CAP_BASE_URL ?? 'http://localhost:4004',
apiKey: process.env.CONTENT_API_KEY,
sourceDir: 'hugo/public',
force: process.argv.includes('--force'),
};
}
// --- Build validation ---
const DEV_ARTIFACT_PATTERNS = [
{ pattern: /data-cap-base="http:\/\/localhost/, label: 'data-cap-base pointing to localhost' },
{ pattern: /livereload\.js/, label: 'Hugo livereload script injection' },
{ pattern: /<script>document\.write.*livereload/s, label: 'Hugo dev server livereload' },
];
export function validateProductionBuild(tutorials: Map<string, string>, sampleSize = 5): string[] {
const slugs = [...tutorials.keys()];
const sample = slugs.slice(0, Math.min(sampleSize, slugs.length));
const violations: string[] = [];
for (const slug of sample) {
const content = readFileSync(tutorials.get(slug)!, 'utf-8');
for (const { pattern, label } of DEV_ARTIFACT_PATTERNS) {
if (pattern.test(content)) {
violations.push(`${slug}: ${label}`);
}
}
}
return violations;
}
// --- Pure functions (exported for testing) ---
export function discoverTutorials(hugoDir: string): Map<string, string> {
const tutorialsDir = join(hugoDir, 'tutorials');
const result = new Map<string, string>();
let entries: string[];
try {
entries = readdirSync(tutorialsDir);
} catch {
return result;
}
for (const entry of entries) {
const indexPath = join(tutorialsDir, entry, 'index.html');
try {
const stat = statSync(indexPath);
if (stat.isFile()) {
// Normalize the slug KEY to lowercase (issue #2714): some source folders
// ship mixed-case (e.g. "...-RAP-App"), but Hugo serves lowercase URLs and
// the server stores/serves lowercase slugs. Lowercasing here — the single
// upstream chokepoint feeding every downstream payload — prevents a
// mixed-case Tutorials.slug row from ever being minted again. The PATH
// keeps the real on-disk (mixed-case) directory so the file read works.
result.set(entry.toLowerCase(), indexPath);
}
} catch {
// not a tutorial directory
}
}
return result;
}
// Concept slug predicate — used to skip Tutorials-only metadata extraction
// for concept landing pages (metadata, bodyText, branchSpecs, source markdown
// are all keyed off Tutorials.slug; concept-* keys would orphan in those
// tables).
export function isConceptSlug(slug: string): boolean {
return typeof slug === 'string' && slug.startsWith('concept-');
}
// #1659 Phase C — author page predicate. Author pages (`author-<login>`) are
// pre-rendered Hugo HTML like concept/page BLOBs; they must be excluded from
// Tutorials-only metadata/bodyText/branchSpec/source extraction (same reason as
// concept slugs) so they never orphan rows in those tutorial-keyed tables.
export function isAuthorSlug(slug: string): boolean {
return typeof slug === 'string' && slug.startsWith('author-');
}
// #1659 Phase C.2a — per-advocate detail page predicate (same rationale as
// concept/author slugs: exclude from Tutorials-only extraction).
export function isAdvocateSlug(slug: string): boolean {
return typeof slug === 'string' && slug.startsWith('advocate-');
}
// A slug points at a runtime-SSR'd catalog page (groups/missions) since PR
// #115 (#91). Such slugs must NEVER reach the publish endpoint; the server
// rejects them too, but we strip locally as well so payload size and CI
// logs stay clean. See issue #114.
export function isCatalogSlug(slug: string): boolean {
return slug.startsWith('group-') || slug.startsWith('mission-');
}
// Strips catalog slugs from the discovery map in place, returning the
// removed slugs in deterministic (sorted) order so callers can log them.
export function stripCatalogSlugs(tutorials: Map<string, string>): string[] {
const dropped: string[] = [];
for (const slug of tutorials.keys()) {
if (isCatalogSlug(slug)) {
dropped.push(slug);
tutorials.delete(slug);
}
}
return dropped.sort();
}
export function computeLocalHashes(tutorials: Map<string, string>): Map<string, string> {
const hashes = new Map<string, string>();
for (const [slug, filePath] of tutorials) {
const content = readFileSync(filePath);
const hash = createHash('sha256').update(content).digest('hex');
hashes.set(slug, hash);
}
return hashes;
}
export function computeDiff(
local: Map<string, string>,
remote: Record<string, string>
): string[] {
const changed: string[] = [];
for (const [slug, localHash] of local) {
if (remote[slug] !== localHash) {
changed.push(slug);
}
}
return changed;
}
export function buildPayload(
slugs: string[],
tutorials: Map<string, string>
): Record<string, string> {
const payload: Record<string, string> = {};
for (const slug of slugs) {
// Tripwire (issue #2714): slugs are lowercased at discoverTutorials, the
// upstream chokepoint. If a mixed-case slug reaches here, some code path
// reintroduced case — fail loud rather than silently publish a mixed-case
// key that mints a non-canonical Tutorials.slug row. Belt-and-suspenders
// behind the discovery-time normalization.
if (slug !== slug.toLowerCase()) {
throw new Error(
`buildPayload: non-canonical (mixed-case) slug "${slug}" — slugs must be lowercase before publish (issue #2714)`
);
}
const filePath = tutorials.get(slug);
if (!filePath) continue;
const content = readFileSync(filePath);
const compressed = gzipSync(content);
payload[slug] = compressed.toString('base64');
}
return payload;
}
/**
* Build the source-markdown side of the publish payload (PR #591). For each
* slug, reads `<cacheDir>/<slug>.md` (the raw upstream tutorial markdown that
* `fetch-tutorials.ts` populated), gzips it, base64-encodes, and stores under
* the same slug key. Slugs whose source file is missing are silently skipped
* — the server stores null sourceContent/sourceHash for those rows, and
* `/content/source-hashes` simply omits them. This is the right behavior for
* special slugs (`__shell__`, `__nav__`, `__404__`) that have no upstream
* markdown.
*
* Returns the gzipped-base64 map AND a parallel map of pre-gzip SHA-256
* hashes; the server independently re-hashes after gunzip and rejects on
* mismatch (defense against in-flight corruption), but exposing the hash
* locally lets the CLI auto-verify against `/content/source-hashes`
* without round-tripping through gzip.
*/
/**
* #672 short-circuit support: compute pre-gzip SHA-256 of each slug's source
* markdown WITHOUT building the gzip payload. Used in delta mode to drop slugs
* whose source bytes already match the server, before paying for buildPayload's
* Hugo-output re-read + gzip.
*
* Slugs whose source file is missing (special slugs: __shell__, __nav__, __404__)
* are silently skipped.
*/
export function computeLocalSourceHashes(
slugs: string[],
cacheDir: string
): Map<string, string> {
const sourceHashes = new Map<string, string>();
for (const slug of slugs) {
const mdPath = join(cacheDir, `${slug}.md`);
if (!existsSync(mdPath)) continue;
const content = readFileSync(mdPath);
const hash = createHash('sha256').update(content).digest('hex');
sourceHashes.set(slug, hash);
}
return sourceHashes;
}
export function buildSourcePayload(
slugs: string[],
cacheDir: string
): { sources: Record<string, string>; sourceHashes: Map<string, string> } {
const sources: Record<string, string> = {};
const sourceHashes = computeLocalSourceHashes(slugs, cacheDir);
for (const slug of slugs) {
const mdPath = join(cacheDir, `${slug}.md`);
if (!existsSync(mdPath)) continue;
const content = readFileSync(mdPath);
const compressed = gzipSync(content);
sources[slug] = compressed.toString('base64');
}
return { sources, sourceHashes };
}
/**
* #2245: Build the per-slug git commit SHA map for the append payload.
* For each slug, reads `<cacheDir>/<slug-lowercase>.commit-sha` (written by
* fetch-tutorials.ts alongside the main .md cache file). Slugs with no sidecar
* are silently skipped — the server stores null sourceCommit for those rows.
*
* The key in the returned map is the ORIGINAL-CASE slug (matching how `sources`
* and `files` are keyed), so the server's `sourceCommits[slug]` lookup lands on
* the correct entry. The file lookup uses `slug.toLowerCase()` to match the
* lowercase-canonical filename written by fetch-tutorials.
*/
export function buildSourceCommitsPayload(
slugs: string[],
cacheDir: string,
): Record<string, string> {
const result: Record<string, string> = {};
for (const slug of slugs) {
const shaPath = join(cacheDir, `${slug.toLowerCase()}.commit-sha`);
if (!existsSync(shaPath)) continue;
const sha = readFileSync(shaPath, 'utf-8').trim();
if (sha) result[slug] = sha;
}
return result;
}
/**
* Parallel to buildSourceCommitsPayload: reads `<slug>.source-repo` sidecars
* (owner/name of the GitHub repo the source came from) written by
* fetch-tutorials. Keyed by ORIGINAL-CASE slug so the server's
* sourceRepos[slug] lookup lands on the correct entry; the file lookup uses
* slug.toLowerCase() to match the lowercase-canonical sidecar filename.
* Feeds the public-namespace -Contribution publish guard.
*/
export function buildSourceReposPayload(
slugs: string[],
cacheDir: string,
): Record<string, string> {
const result: Record<string, string> = {};
for (const slug of slugs) {
const repoPath = join(cacheDir, `${slug.toLowerCase()}.source-repo`);
if (!existsSync(repoPath)) continue;
const repo = readFileSync(repoPath, 'utf-8').trim();
if (repo) result[slug] = repo;
}
return result;
}
const TUTORIAL_MAIN_RE = /<main\b[^>]*class\s*=\s*["']?[^"'>]*\btutorial-main\b[^"'>]*["']?[^>]*>([\s\S]*?)<\/main>/i;
const BODY_RE = /<body\b[^>]*>([\s\S]*?)<\/body>/i;
const STRIP_BLOCKS_RE = /<(script|style|nav|footer|aside)\b[^>]*>[\s\S]*?<\/\1>/gi;
const TAG_RE = /<[^>]+>/g;
const WHITESPACE_RE = /\s+/g;
const ENTITY_MAP: Record<string, string> = {
'&': '&', '<': '<', '>': '>', '"': '"',
''': "'", ''': "'", ' ': ' ',
};
export function extractBodyText(html: string): string {
const mainMatch = html.match(TUTORIAL_MAIN_RE);
let zone = mainMatch ? mainMatch[1] : (html.match(BODY_RE)?.[1] ?? html);
zone = zone.replace(STRIP_BLOCKS_RE, ' ');
zone = zone.replace(TAG_RE, ' ');
zone = zone.replace(/&[a-z#0-9]+;/gi, m => ENTITY_MAP[m.toLowerCase()] ?? ' ');
zone = zone.replace(WHITESPACE_RE, ' ').trim();
return zone;
}
export function extractAllBodyTexts(
tutorials: Map<string, string>,
slugs: string[]
): Record<string, string> {
const result: Record<string, string> = {};
for (const slug of slugs) {
const filePath = tutorials.get(slug);
if (!filePath) continue;
const html = readFileSync(filePath, 'utf-8');
const text = extractBodyText(html);
if (text) result[slug] = text;
}
return result;
}
// --- Branch spec extraction (issue #172 PR 3) ---
// Reads branchPoints / skipPoints out of each tutorial's parsed Hugo
// frontmatter so they ride along in the publish payload and land in the
// BranchSpecs sidecar entity. Tutorials with no branches/skips are skipped
// entirely (no row written) — the decide handler treats absence as "linear".
export interface BranchSpec {
branchPoints: Array<{
id: string;
parentStepNumber: number;
groupKey: string;
branches: Array<{
key: string;
label: string;
condition: string | null;
embeddingHint: string | null;
}>;
}>;
skipPoints: Array<{
stepNumber: number;
skipIf: string;
skipLabel?: string;
skipReason?: string;
}>;
}
export function extractAllBranchSpecs(
contentDir: string,
slugs: string[]
): Record<string, BranchSpec> {
const out: Record<string, BranchSpec> = {};
for (const slug of slugs) {
const fmPath = join(contentDir, `${slug}.md`);
if (!existsSync(fmPath)) continue;
const raw = readFileSync(fmPath, 'utf-8');
const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
if (!fmMatch) continue;
let fm: any;
try { fm = parseYaml(fmMatch[1]); } catch { continue; }
const branchPoints: BranchSpec['branchPoints'] = [];
const skipPoints: BranchSpec['skipPoints'] = [];
for (const step of (fm.steps ?? [])) {
if (step.branchPointId && Array.isArray(step.branches)) {
branchPoints.push({
id: step.branchPointId,
parentStepNumber: step.number,
groupKey: step.branchGroup,
branches: step.branches.map((b: any) => ({
key: b.key,
label: b.label,
condition: b.condition ?? null,
embeddingHint: b.embeddingHint ?? null,
})),
});
}
if (step.skipIf) {
skipPoints.push({
stepNumber: step.number,
skipIf: step.skipIf,
skipLabel: step.skipLabel,
skipReason: step.skipReason,
});
}
}
if (branchPoints.length || skipPoints.length) {
out[slug] = { branchPoints, skipPoints };
}
}
return out;
}
// --- Metadata extraction ---
export interface StepMeta {
number: number;
title: string;
}
export interface TutorialMeta {
slug: string;
title: string;
description: string;
time: number | null;
level: string | null;
primaryTag: string | null;
stepCount: number;
steps: StepMeta[];
lastUpdated: string | null;
primaryContributorEmail: string | null;
primaryContributorLogin: string | null;
frontmatterGithubLogin: string | null;
frontmatterAuthorName: string | null;
}
export function extractMetadata(
contentDir: string,
slugs: string[]
): Record<string, TutorialMeta> {
const result: Record<string, TutorialMeta> = {};
for (const slug of slugs) {
const mdPath = join(contentDir, `${slug}.md`);
if (!existsSync(mdPath)) continue;
const raw = readFileSync(mdPath, 'utf-8');
const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
if (!fmMatch) continue;
let fm: any;
try {
fm = parseYaml(fmMatch[1]);
} catch {
continue;
}
const steps: StepMeta[] = Array.isArray(fm.steps)
? fm.steps.map((s: any) => ({ number: s.number ?? 0, title: s.title ?? '' }))
: [];
const contributors = Array.isArray(fm.contributors) ? fm.contributors : [];
const primary = contributors.length > 0 ? contributors[0] : null;
const primaryContributorEmail = primary ? trim((primary as any).email) : null;
const primaryContributorLogin = primary ? trim((primary as any).login) : null;
const frontmatterGithubLogin = trim(fm.githubLogin);
const frontmatterAuthorName = trim(fm.author);
result[slug] = {
slug,
title: fm.title ?? slug,
description: fm.description ?? '',
time: typeof fm.time === 'number' ? fm.time : null,
level: fm.level ?? null,
primaryTag: fm.primaryTag ?? null,
stepCount: fm.stepCount ?? steps.length,
steps,
lastUpdated: trim(fm.lastUpdated),
primaryContributorEmail,
primaryContributorLogin,
frontmatterGithubLogin,
frontmatterAuthorName,
};
}
return result;
}
// --- CLI ---
/** Append a markdown block to $GITHUB_STEP_SUMMARY if set. No-op locally. */
function writeStepSummary(markdown: string) {
const target = process.env.GITHUB_STEP_SUMMARY;
if (!target) return;
try {
appendFileSync(target, markdown + '\n');
} catch (err) {
console.error(`[purge-orphans] Failed to append $GITHUB_STEP_SUMMARY: ${formatErrorChain(err)}`);
}
}
export function validateFlagCombo(flags: { force: boolean; heal: boolean; verifyOnly: boolean; purgeOrphans?: boolean; unpublishSlugs?: boolean }) {
const modes = [
flags.force && 'force',
flags.heal && 'heal',
flags.verifyOnly && 'verify-only',
flags.purgeOrphans && 'purge-orphans',
flags.unpublishSlugs && 'unpublish-slugs'
].filter(Boolean);
if (modes.length > 1) {
throw new Error(`Flags ${modes.join(', ')} are mutually exclusive`);
}
}
/**
* #2349 — parse + validate the `--unpublish-slugs <csv>` argument into a clean
* slug list. Unlike `--purge-orphans` (which COMPUTES the orphan set by diffing
* a full local cache against the server), this mode is handed an EXPLICIT list
* of slugs already known to be deleted upstream — the source-repo dispatch
* computed them from `git diff --diff-filter=D`. So there is no cache to read
* and no discovery to trust; the list IS the input.
*
* Validation mirrors the dispatch/receiver slug charset contract (lowercase
* canonical [a-z0-9-]): split on comma, trim, drop empties, lowercase, and
* reject any token with a non-slug character so a malformed dispatch payload
* can never reach the soft-delete endpoint. De-dupes while preserving order.
*
* Pure: no I/O. Throws on an empty result (an unpublish call with zero valid
* slugs is a caller bug — fail loud rather than POST an empty batch).
*/
export function parseUnpublishSlugs(csv: string): string[] {
const seen = new Set<string>();
const out: string[] = [];
for (const raw of (csv ?? '').split(',')) {
const s = raw.trim().toLowerCase();
if (!s) continue;
if (!/^[a-z0-9-]+$/.test(s)) {
throw new Error(`--unpublish-slugs: invalid slug "${raw.trim()}" (must be lowercase [a-z0-9-])`);
}
if (seen.has(s)) continue;
seen.add(s);
out.push(s);
}
if (out.length === 0) {
throw new Error('--unpublish-slugs: no valid slugs after parsing');
}
return out;
}
export type PublishMode = 'force' | 'heal' | 'delta';
/**
* #672 operator override. Resolve the set of slugs the operator has explicitly
* approved to commit even if the server's no-revert guard flags them as reverts
* (source-byte churn re-reads current upstream as an abandoned-history match).
*
* Intentionally per-slug: `--allow-revert` MUST be paired with `--slug` so an
* operator can only override the single tutorial they named — never blanket-
* override every revert in a full-catalog publish. Returns [] when the flag is
* off (guard fully active, default behavior).
*/
export function resolveAllowRevertSlugs(opts: { allowRevert: boolean; slug: string }): string[] {
if (!opts.allowRevert) return [];
const s = (opts.slug ?? '').trim();
if (!s) {
throw new Error('--allow-revert requires --slug (per-slug override only; refusing a blanket revert override)');
}
return [s];
}
export function computePublishPlan(opts: {
local: Map<string, string>;
remote: Record<string, string>;
mode: PublishMode;
/**
* #2727 render-pipeline token. When provided and the local token differs from
* the server's active token (or the server has none yet), the render pipeline
* (parsers / Hugo templates / CSS) changed since the active publish, so a
* delta plan treats EVERY local slug as changed — a parser/template/CSS-only
* fix produces identical source markdown and (until `hugo/public` is rebuilt)
* cannot be distinguished per-slug, so the pipeline token forces the republish.
* Omit both to preserve pre-#2727 HTML-diff behavior. Ignored in force/heal
* mode, which already select every slug.
*/
localToken?: string;
serverToken?: string;
}): { targetSlugs: string[]; pipelineChanged?: boolean } {
if (opts.mode === 'force') return { targetSlugs: [...opts.local.keys()] };
// Pipeline-token override: a changed render pipeline invalidates all renders.
const pipelineChanged =
opts.localToken !== undefined && opts.localToken !== opts.serverToken;
if (pipelineChanged) {
return { targetSlugs: [...opts.local.keys()], pipelineChanged: true };
}
return { targetSlugs: computeDiff(opts.local, opts.remote), pipelineChanged: false };
}
/**
* #1668 — split the post-commit verify diff into genuine store/verify
* mismatches vs. slugs that differ ONLY because the #672 no-revert guard
* deliberately kept prior content (so the local revert bytes never landed).
*
* The auto-verify step compares local hashes against the server AFTER commit.
* A slug the guard rejected still carries its prior (non-revert) content on the
* server, so its local hash won't match — it lands in `verifyDiff`. Lumping it
* into "Verify FAILED" turns a benign, expected outcome into an alarming one
* (cost real triage time — see the issue). This partition lets the caller
* report rejected reverts as expected and reserve "Verify FAILED" (exit 2) for
* true mismatches.
*
* Pure function: order-preserving over `verifyDiff`. Rejected-revert slugs that
* don't appear in `verifyDiff` (their bytes happened to match) are ignored, not
* invented into either list.
*/
export function partitionVerifyMismatches(opts: {
verifyDiff: string[];
rejectedReverts?: string[];
}): { genuine: string[]; expectedReverts: string[] } {
const rejected = new Set(opts.rejectedReverts ?? []);
const genuine: string[] = [];
const expectedReverts: string[] = [];
for (const slug of opts.verifyDiff) {
(rejected.has(slug) ? expectedReverts : genuine).push(slug);
}
return { genuine, expectedReverts };
}
/**
* #672 short-circuit: drop slugs from `targetSlugs` whose state is already in
* sync with the server. A slug is in sync when BOTH:
* - source-markdown hash matches the server's stored sourceHash, AND
* - rendered-HTML hash matches the server's stored contentHash.
*
* The double-hash check is critical. The original #672 design (PR #675) only
* compared source-markdown — which works for the regression mode it was built
* to catch (stale workstation cache re-uploads old bytes) but silently hides
* the much more common case where Hugo's templates change without any source-
* markdown change. In that case every rendered HTML differs from the server
* but every source-md matches, so a source-only short-circuit drops every
* slug from the publish payload, the server carries forward stale HTML, and
* post-publish auto-verify fails (rebuild-content workflow run 28304515829,
* the first run after #675 merged, hit exactly this).
*
* Pure function: no I/O. The caller is responsible for fetching the server
* source-hash map and computing the local source-hash map.
*
* @param targetSlugs slugs that `computePublishPlan` selected for upload
* @param localSource slug → SHA-256 of source markdown (from cache)
* @param serverSource slug → SHA-256 of source markdown (from /content/source-hashes)
* @param localHtml slug → SHA-256 of rendered HTML (from hugo/public)
* @param serverHtml slug → SHA-256 of rendered HTML (from /content/hashes)
* @returns the filtered target slug list (a slug is kept unless BOTH hashes match)
*/
export function applySourceHashShortCircuit(opts: {
targetSlugs: string[];
localSource: Map<string, string>;
serverSource: Record<string, string>;
localHtml: Map<string, string>;
serverHtml: Record<string, string>;
}): string[] {
return opts.targetSlugs.filter((slug) => {
const localSrc = opts.localSource.get(slug);
const serverSrc = opts.serverSource[slug];
const localHtml = opts.localHtml.get(slug);
const serverHtml = opts.serverHtml[slug];
// Drop only when BOTH source-md AND rendered-HTML match the server.
// Missing local hash (special slug) or missing server hash (new slug) → keep.
const sourceInSync = !!(localSrc && serverSrc && localSrc === serverSrc);
const htmlInSync = !!(localHtml && serverHtml && localHtml === serverHtml);
return !(sourceInSync && htmlInSync);
});
}
interface PublishOptions {
hugoDir: string;
baseUrl: string;
apiKey: string;
trigger: string;
hugoVersion: string;
initiator: string;
dryRun: boolean;
force: boolean;
heal: boolean;
verifyOnly: boolean;
verbose: boolean;
concurrency: number;
batchSize: number;
purgeOrphans: boolean;
purgeCapAbs: number;
unpublishSlugs: string;
slug: string;
}
function parseArgs(argv: string[]): PublishOptions {
const get = (flag: string, fallback: string): string => {
const idx = argv.indexOf(flag);
return idx !== -1 && idx + 1 < argv.length ? argv[idx + 1] : fallback;
};
const has = (flag: string): boolean => argv.includes(flag);
return {
hugoDir: get('--hugo-dir', 'hugo/public'),
baseUrl: get('--base-url', process.env.CAP_BASE_URL || 'http://localhost:4004'),
apiKey: get('--api-key', process.env.CONTENT_API_KEY || ''),
trigger: get('--trigger', `manual@${process.env.GITHUB_SHA?.slice(0, 7) || 'local'}`),
hugoVersion: get('--hugo-version', ''),
initiator: get(
'--initiator',
process.env.INITIATOR
|| `${userInfo().username || 'unknown'}@${hostname()}`
),
dryRun: has('--dry-run'),
force: has('--force'),
heal: has('--heal'),
verifyOnly: has('--verify-only'),
verbose: has('--verbose'),
allowRevert: has('--allow-revert'),
concurrency: parseInt(get('--concurrency', '6'), 10),
batchSize: parseInt(get('--batch-size', '50'), 10),
purgeOrphans: has('--purge-orphans'),
purgeCapAbs: parseInt(get('--purge-cap-abs', process.env.PURGE_CAP_ABS ?? '50'), 10),
// #2349 — targeted unpublish. Comma-separated slugs known-deleted upstream
// (source-repo dispatch computed them via `git diff --diff-filter=D`). POSTs
// them straight to /content/orphan-purge (soft-delete) — no cache read, no
// orphan computation. Env fallback so the workflow can pass the dispatch's
// deleted_slugs through without an extra flag. Empty = mode off.
unpublishSlugs: get('--unpublish-slugs', process.env.UNPUBLISH_SLUGS ?? ''),
// #1278 — single-tutorial fast path. When set, only this slug is hashed +
// published; the server carries forward all other slugs from the prior
// ACTIVE version (normal delta behavior), so the catalog stays complete.
// Empty string = whole-catalog delta (default). Also accepts env fallback
// so the workflow can pass TUTORIAL_SLUG through without an extra flag.
slug: get('--slug', process.env.PUBLISH_SLUG ?? ''),
// Multi-slug variant of --slug for the N-tutorial repository_dispatch path.
// Comma-separated; scopes the publish to exactly these slugs (O(N), not
// O(catalog)). Empty = fall back to --slug / whole-catalog delta. Env
// fallback (PUBLISH_SLUGS) lets the workflow pass dispatch_slugs through.
slugs: get('--slugs', process.env.PUBLISH_SLUGS ?? ''),
};
}
function pickEntries<T>(src: Record<string, T>, keys: string[]): Record<string, T> {
const out: Record<string, T> = {};
for (const k of keys) if (k in src) out[k] = src[k];
return out;
}
/**
* #1278 — single-tutorial fast path. Reduce the discovered tutorials map to
* the one requested slug (in place), so downstream hashing/payload only
* touches that slug. Throws (caller exits non-zero) if the slug isn't present
* in the Hugo output — a typo'd/deleted slug must fail loudly, not silently
* publish nothing. No-op when `slug` is empty (whole-catalog delta, default).
* Returns true if a filter was applied.
*
* Pure over the Map + a throwing contract, so it's unit-testable without the
* filesystem/network.
*/
export function filterToSlug(tutorials: Map<string, string>, slug: string): boolean {
if (!slug) return false;
if (!tutorials.has(slug)) {
throw new Error(
`--slug "${slug}" not found (available count=${tutorials.size}). ` +
`Check the slug is lowercase-canonical and its page exists.`
);
}
const only = tutorials.get(slug)!;
tutorials.clear();
tutorials.set(slug, only);
return true;
}
/**
* Multi-slug variant of filterToSlug: reduce the discovered map to exactly the
* requested slugs. Used by the N-tutorial repository_dispatch path (a push that
* changed several tutorials) so the publish stays O(changed slugs) instead of
* demoting to a whole-catalog delta. The server carries forward every other
* slug at commit (same as any delta), so the catalog stays complete.
*
* Blank/whitespace entries are ignored and the list is de-duplicated. If ANY
* requested slug is absent it throws (loud fail) rather than silently
* publishing a subset — a typo in one of N slugs must not quietly drop it.
* Pure over the Map + a throwing contract, so it's unit-testable without the
* filesystem/network. Returns false (no-op) on an empty list.
*/
export function filterToSlugs(tutorials: Map<string, string>, slugs: string[]): boolean {
const wanted = [...new Set((slugs ?? []).map((s) => (s ?? '').trim()).filter(Boolean))];
if (wanted.length === 0) return false;
const missing = wanted.filter((s) => !tutorials.has(s));
if (missing.length) {
throw new Error(
`--slugs not found: ${missing.join(', ')} (available count=${tutorials.size}). ` +
`Check each slug is lowercase-canonical and its page exists.`
);
}
const kept = new Map(wanted.map((s) => [s, tutorials.get(s)!] as const));
tutorials.clear();
for (const [k, v] of kept) tutorials.set(k, v);
return true;
}
// [#2885] Approuter asset gate — mirrors the check-approuter-assets.cjs step
// that rebuild-content.yml already runs. Reuses the .cjs as a subprocess to
// avoid re-implementing probe/inconclusive logic.
/** Decide whether to run the approuter asset gate based on env vars. */
export function approuterGateDecision(env: { APPROUTER_URL?: string; SKIP_APPROUTER_GATE?: string }): { run: boolean; reason: string } {
if (!env.APPROUTER_URL) return { run: false, reason: 'no-approuter' };
if (env.SKIP_APPROUTER_GATE) return { run: false, reason: 'skip-flag' };
return { run: true, reason: 'ok' };
}
/** Build the argv array for the check-approuter-assets.cjs subprocess. */
export function buildApprouterGateArgs(opts: { hugoDir: string; slugs: string }, approuterUrl: string): string[] {
const args = [
'scripts/check-approuter-assets.cjs',
'--approuter-url', approuterUrl,
'--hugo-dir', opts.hugoDir,
'--check-islands',
];
if (opts.slugs.trim()) args.push('--slugs', opts.slugs.trim());
return args;
}
/** Run the approuter asset gate. Returns 0 when skipped or passed; non-zero on gate failure. */
export function runApprouterGate(
opts: { hugoDir: string; slug: string; slugs: string },
env: NodeJS.ProcessEnv,
spawn: typeof _spawnSync = _spawnSync,
): number {
const decision = approuterGateDecision(env);
if (!decision.run) {
if (decision.reason === 'skip-flag') {
console.warn('[publish-content] SKIP_APPROUTER_GATE=1 — skipping approuter asset gate (bypass mode).');
} else {
console.warn('[publish-content] APPROUTER_URL not set — skipping approuter asset gate (local dev mode).');
}
return 0;
}
const args = buildApprouterGateArgs(opts, env.APPROUTER_URL!);
const result = spawn('node', args, {
stdio: 'inherit',
env: { ...env, PUBLISH_SLUG: opts.slug },
});
return result.status ?? 0;
}
async function collectSidecars(hugoDir: string, payload: Record<string, string>, log: (s: string) => void, channel: Channel = "prod"): Promise<string[]> {
const keys: string[] = [];
const navJsonPath = join(hugoDir, 'tutorials', '_nav.json');
if (existsSync(navJsonPath)) {
const navContent = readFileSync(navJsonPath);
const navData = JSON.parse(navContent.toString('utf-8'));
const allNavTutorials = navData.tutorials ?? navData;
payload['__nav__'] = gzipSync(Buffer.from(JSON.stringify({ tutorials: allNavTutorials }))).toString('base64');
keys.push('__nav__');
log(`Included nav metadata for ${allNavTutorials.length} tutorials`);
}
const notFoundPath = join(hugoDir, '404.html');
if (existsSync(notFoundPath)) {
const notFoundContent = readFileSync(notFoundPath);
payload['__404__'] = gzipSync(notFoundContent).toString('base64');
keys.push('__404__');
}
// The chrome shell wraps tutorial content with site-wide UI (header, nav,
// footer, Joule FAB, etc.). On the QA channel, hugo.qa.toml strips all that
// chrome AND the QA srv at /tutorials-qa/ stands alone for author preview —
// the __shell__ slug is unused server-side for QA reads. Skipping avoids
// a hard failure when the QA Hugo build legitimately doesn't emit
// _shell/index.html. Issue: rebuild-content-qa.yml run #27885050471.
if (channel === 'qa') {
log('Skipping __shell__ sidecar (QA channel — no chrome wrapping)');
return keys;
}
const shellPath = join(hugoDir, '_shell', 'index.html');
if (!existsSync(shellPath)) {
throw new Error(`[publish-content] _shell/index.html missing — Hugo build did not emit chrome shell. Path: ${shellPath}`);
}
const shellRaw = readFileSync(shellPath, 'utf-8');
const mainMatch = shellRaw.match(/<main\b[^>]*>[\s\S]*?<\/main>/);
if (!mainMatch) throw new Error('[publish-content] _shell/index.html does not contain <main>...</main>');
const shellHtml = shellRaw.replace(mainMatch[0], '<!-- MAIN -->');
if (shellHtml.length < 1000) throw new Error(`[publish-content] chrome shell suspiciously small (${shellHtml.length} bytes)`);
payload['__shell__'] = gzipSync(Buffer.from(shellHtml, 'utf-8')).toString('base64');
keys.push('__shell__');
return keys;
}
async function main() {
const channel = parseChannel(process.argv);
const opts = parseArgs(process.argv.slice(2));
if (channel === 'qa') {
const cfg = resolvePublishConfig({ channel });
opts.hugoDir = cfg.sourceDir;
opts.baseUrl = cfg.baseUrl;
opts.apiKey = cfg.apiKey ?? '';
opts.force = cfg.force;
}
// --verify-only is a read-only call against the public /content/source-hashes
// endpoint (PR #591; pre-PR #591 it used /content/hashes — also public-read).
// No auth needed, so the API key check is skipped. Without this exemption,
// the daily content-drift-check workflow could never succeed (it sets
// CONTENT_API_KEY="" deliberately).
if (!opts.apiKey && !opts.verifyOnly) {
const envHint = channel === 'qa' ? 'CONTENT_API_KEY_QA' : 'CONTENT_API_KEY';
console.error(`Error: No API key. Set ${envHint} env var or pass --api-key`);
process.exit(1);
}
const log = opts.verbose ? console.log : () => {};
// --- verify-only short-circuit ---
//
// PR #591 design: --verify-only compares source-markdown hashes, NOT
// rendered-HTML hashes. The rendered HTML is volatile-by-design relative to
// upstream (recs rail + CAP-fed breadcrumbs + Shiki vs Chroma + ...), and
// every previous attempt to normalize those volatile regions out of the hash
// (#589, #590) failed to scale because there are too many. The source
// markdown changes only when an author edits the upstream tutorial — a
// monotonic, meaningful signal.
//
// The drift workflow now runs ONLY `npm run fetch-tutorials` before
// --verify-only — no Hugo build, no Shiki, no Vue. Fast (~1-2 min) and
// accurate.
//
// Runs BEFORE `discoverTutorials(opts.hugoDir)` so the drift workflow
// doesn't need to build Hugo at all. Pre-PR #591 the verify-only path
// ran AFTER the Hugo-discovery step and would error out with 'No
// tutorials found. Did you run the Hugo build?' on a workflow that
// skipped Hugo — caught the morning of 2026-06-24 by run 28104752827.
if (opts.verifyOnly) {
const cacheDir = channel === 'qa'
? join(process.cwd(), '.tutorial-cache-qa')
: join(process.cwd(), '.tutorial-cache');
// 1) Discover slugs from the cache by listing *.md files (NOT from Hugo
// output, which the workflow no longer builds).
let mdFiles: string[];
try { mdFiles = readdirSync(cacheDir).filter(f => f.endsWith('.md') && !f.startsWith('_')); }
catch (err) {
console.error(`Verify failed: cannot read tutorial-cache dir ${cacheDir}: ${formatErrorChain(err)}`);
process.exit(1);
}
const localHashes = new Map<string, string>();
for (const f of mdFiles) {
const slug = f.replace(/\.md$/, '');
const buf = readFileSync(join(cacheDir, f));
localHashes.set(slug, createHash('sha256').update(buf).digest('hex'));
}
log(`Hashed ${localHashes.size} source markdown files in ${cacheDir}`);
// 2) Fetch server's source-hash map.
let remote: Record<string, string>;
try {
remote = await fetchRemoteSourceHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey });
} catch (err) {
console.error('Verify failed: cannot reach /content/source-hashes:', formatErrorChain(err));
process.exit(1);
}
const serverSlugCount = Object.keys(remote).length;
log(`Fetched ${serverSlugCount} source hashes from ${opts.baseUrl}/content/source-hashes`);
// 3) Diff. Only count slugs the server actually KNOWS about — if the
// server returned an empty/partial map (e.g. server pre-dates PR
// #591 and never wrote source hashes), warn and exit 0. Otherwise
// a fresh deploy would always look like 1400 slugs of "drift".
if (serverSlugCount === 0) {
console.warn('');
console.warn('Verify SKIPPED: server returned 0 source hashes.');
console.warn(' This is expected immediately after PR #591 deploys, before');
console.warn(' a new publish-content run has populated sourceHash. Re-run');
console.warn(' this verify after `npm run publish-content` completes.');
process.exit(0);
}
const drifted: string[] = [];
const missingLocal: string[] = []; // present on server, missing in local cache
const missingServer: string[] = []; // present locally, no hash on server
for (const [slug, localHash] of localHashes) {
const serverHash = remote[slug];
if (!serverHash) {
missingServer.push(slug);