Skip to content
CI

docs: update architecture documentation (#14691) #34604

Sign in to view logs

docs: update architecture documentation (#14691)

docs: update architecture documentation (#14691) #34604

Workflow file for this run

.github/workflows/ci.yml at 230cbb5
name: CI
on:
push:
branches: [main]
pull_request:
paths:
- '**.go'
- 'pkg/workflow/**'
- 'actions/**'
- '.github/workflows/ci.yml'
- '.github/workflows/**/*.md'
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-test
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Display Go environment
run: |
echo "Go environment:"
go env | grep -E "GOPROXY|GOSUMDB|GOMODCACHE|GOPRIVATE"
echo ""
echo "Module cache location: $(go env GOMODCACHE)"
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
# Use -x for verbose output to see what's being downloaded
if go mod download -x; then
echo "✅ Successfully downloaded Go modules"
break
else
EXIT_CODE=$?
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts (exit code: $EXIT_CODE)"
echo "This indicates that proxy.golang.org is unreachable or returning errors"
echo ""
echo "Diagnostic information:"
echo "- GOPROXY: $(go env GOPROXY)"
echo "- GOSUMDB: $(go env GOSUMDB)"
echo "- Network connectivity: checking proxy.golang.org..."
if curl -s -I --connect-timeout 5 https://proxy.golang.org/golang.org/@v/list >/dev/null 2>&1; then
echo " ✓ proxy.golang.org is reachable"
else
echo " ✗ proxy.golang.org is NOT reachable"
fi
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Pre-flight check - Validate test dependencies
run: |
echo "Validating that test dependencies are available..."
echo "This ensures go test can compile test packages without network access."
echo ""
# List all test dependencies to ensure they're in the cache
# This will fail fast if any dependencies are missing
echo "Checking test dependencies for all packages..."
if go list -test -deps ./... >/dev/null 2>&1; then
echo "✅ All test dependencies are available"
else
echo "❌ Failed to resolve test dependencies"
echo ""
echo "Attempting to show which dependencies are missing:"
go list -test -deps ./... 2>&1 || true
exit 1
fi
echo ""
echo "Module cache statistics:"
echo "- Cache directory: $(go env GOMODCACHE)"
if [ -d "$(go env GOMODCACHE)" ]; then
echo "- Cache size: $(du -sh $(go env GOMODCACHE) 2>/dev/null | cut -f1 || echo 'unknown')"
echo "- Number of cached modules: $(find $(go env GOMODCACHE) -name "go.mod" 2>/dev/null | wc -l || echo 'unknown')"
fi
- name: Run unit tests with coverage
id: run-unit-tests
run: |
set -o pipefail
# Run tests with JSON output for artifacts, but also show failures
go test -v -parallel=8 -timeout=3m -run='^Test' -tags '!integration' -coverprofile=coverage.out -json ./... | tee test-result-unit.json
# Check if tests failed by looking at JSON output
if grep -q '"Action":"fail"' test-result-unit.json; then
echo "❌ Tests failed - see output above"
exit 1
fi
# Generate coverage HTML report
go tool cover -html=coverage.out -o coverage.html
- name: Report test failures
if: failure() && steps.run-unit-tests.outcome == 'failure'
run: |
echo "## 🔍 Unit Test Failure Analysis" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Analyzing unit test results" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
# Run the failure report script
if ./scripts/report-test-failures.sh test-result-unit.json | tee /tmp/failure-report.txt; then
echo "No failures detected in JSON output (unexpected - tests failed but no failure records found)" >> $GITHUB_STEP_SUMMARY
else
# Script found failures - add to summary
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
cat /tmp/failure-report.txt >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
fi
# Coverage reports for recent builds only - 7 days is sufficient for debugging recent changes
- name: Upload coverage report
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: coverage-report
path: coverage.html
retention-days: 7
- name: Upload unit test results
if: always() # Upload even if tests fail so canary_go can track coverage
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: test-result-unit
path: test-result-unit.json
retention-days: 14
integration:
runs-on: ubuntu-latest
permissions:
contents: read
strategy:
fail-fast: false
matrix:
test-group:
- name: "CLI Compile & Poutine"
packages: "./pkg/cli"
pattern: "^TestCompile[^W]|TestPoutine" # Exclude TestCompileWorkflows to avoid duplicates
- name: "CLI MCP Playwright"
packages: "./pkg/cli"
pattern: "TestMCPInspectPlaywright"
- name: "CLI MCP Gateway"
packages: "./pkg/cli"
pattern: "TestMCPGateway"
- name: "CLI MCP Other"
packages: "./pkg/cli"
pattern: "TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig"
- name: "CLI Logs & Firewall"
packages: "./pkg/cli"
pattern: "TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow"
- name: "CLI Progress Flag" # Isolate slow test (~65s for TestProgressFlagSignature)
packages: "./pkg/cli"
pattern: "TestProgressFlagSignature"
- name: "CLI HTTP MCP Connect" # Isolate slow HTTP MCP connection tests (~43s)
packages: "./pkg/cli"
pattern: "TestConnectHTTPMCPServer"
- name: "CLI Compile Workflows" # Isolate slow workflow compilation test
packages: "./pkg/cli"
pattern: "TestCompileWorkflows_EmptyMarkdown"
- name: "CLI Security Tools" # Group security tool compilation tests
packages: "./pkg/cli"
pattern: "TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor"
- name: "CLI Add & List Commands"
packages: "./pkg/cli"
pattern: "^TestAdd|^TestList"
- name: "CLI Update Command"
packages: "./pkg/cli"
pattern: "^TestUpdate"
- name: "CLI Audit & Inspect"
packages: "./pkg/cli"
pattern: "^TestAudit|^TestInspect"
- name: "CLI Docker Build" # Isolate slow Docker build test (~38s)
packages: "./pkg/cli"
pattern: "TestDockerBuild"
- name: "CLI Completion & Other" # Remaining catch-all (reduced from original)
packages: "./pkg/cli"
pattern: "" # Catch-all for tests not matched by other CLI patterns
skip_pattern: "^TestCompile[^W]|TestPoutine|TestMCPInspectPlaywright|TestMCPGateway|TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow|TestProgressFlagSignature|TestConnectHTTPMCPServer|TestCompileWorkflows_EmptyMarkdown|TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor|^TestAdd|^TestList|^TestUpdate|^TestAudit|^TestInspect|TestDockerBuild"
- name: "Workflow Compiler"
packages: "./pkg/workflow"
pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse"
- name: "Workflow Tools & MCP"
packages: "./pkg/workflow"
pattern: "TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall"
- name: "Workflow Validation"
packages: "./pkg/workflow"
pattern: "TestValidat|TestLock|TestError|TestWarning"
- name: "Workflow Safe Outputs"
packages: "./pkg/workflow"
pattern: "SafeOutputs|CreatePullRequest|OutputLabel|HasSafeOutputs"
- name: "Workflow GitHub & Git"
packages: "./pkg/workflow"
pattern: "GitHub|Git|PushToPullRequest|BuildFromAllowed"
- name: "Workflow Rendering & Bundling"
packages: "./pkg/workflow"
pattern: "Render|Bundle|Script|WritePromptText"
- name: "Workflow Cache"
packages: "./pkg/workflow"
pattern: "^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache"
- name: "Workflow Actions & Containers"
packages: "./pkg/workflow"
pattern: "^TestAction[^P]|Container"
- name: "Workflow Dependabot & Security"
packages: "./pkg/workflow"
pattern: "Dependabot|Security|PII"
- name: "CMD Tests" # All cmd/gh-aw integration tests
packages: "./cmd/gh-aw"
pattern: ""
skip_pattern: "" # No other groups cover cmd tests
- name: "Parser Remote Fetch & Cache"
packages: "./pkg/parser"
pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache"
- name: "Parser Location & Validation"
packages: "./pkg/parser"
pattern: "" # Catch-all for tests not matched by other Parser patterns
skip_pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache"
- name: "Workflow Permissions"
packages: "./pkg/workflow"
pattern: "TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow"
- name: "Workflow Misc Part 1" # Split large catch-all into two balanced groups
packages: "./pkg/workflow"
pattern: "TestAgent|TestCopilot|TestCustom|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider"
- name: "Workflow String & Sanitization"
packages: "./pkg/workflow"
pattern: "String|Sanitize|Normalize|Trim|Clean|Format"
- name: "Workflow Runtime & Setup"
packages: "./pkg/workflow"
pattern: "Runtime|Setup|Install|Download|Version|Binary"
- name: "Workflow Misc Part 2" # Remaining workflow tests
packages: "./pkg/workflow"
pattern: ""
skip_pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse|TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall|TestValidat|TestLock|TestError|TestWarning|SafeOutputs|CreatePullRequest|OutputLabel|HasSafeOutputs|GitHub|Git|PushToPullRequest|BuildFromAllowed|Render|Bundle|Script|WritePromptText|^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache|^TestAction[^P]|Container|Dependabot|Security|PII|TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestAgent|TestCopilot|TestCustom|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|String|Sanitize|Normalize|Trim|Clean|Format|Runtime|Setup|Install|Download|Version|Binary"
concurrency:
group: ci-${{ github.ref }}-integration-${{ matrix.test-group.name }}
cancel-in-progress: true
name: "Integration: ${{ matrix.test-group.name }}"
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Display Go environment
run: |
echo "Go environment:"
go env | grep -E "GOPROXY|GOSUMDB|GOMODCACHE|GOPRIVATE"
echo ""
echo "Module cache location: $(go env GOMODCACHE)"
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
# Use -x for verbose output to see what's being downloaded
if go mod download -x; then
echo "✅ Successfully downloaded Go modules"
break
else
EXIT_CODE=$?
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts (exit code: $EXIT_CODE)"
echo "This indicates that proxy.golang.org is unreachable or returning errors"
echo ""
echo "Diagnostic information:"
echo "- GOPROXY: $(go env GOPROXY)"
echo "- GOSUMDB: $(go env GOSUMDB)"
echo "- Network connectivity: checking proxy.golang.org..."
if curl -s -I --connect-timeout 5 https://proxy.golang.org/golang.org/@v/list >/dev/null 2>&1; then
echo " ✓ proxy.golang.org is reachable"
else
echo " ✗ proxy.golang.org is NOT reachable"
fi
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Pre-flight check - Validate test dependencies
run: |
echo "Validating that test dependencies are available..."
echo "This ensures go test can compile test packages without network access."
echo ""
# List all test dependencies to ensure they're in the cache
# This will fail fast if any dependencies are missing
echo "Checking test dependencies for ${{ matrix.test-group.packages }}..."
if go list -test -deps ${{ matrix.test-group.packages }} >/dev/null 2>&1; then
echo "✅ All test dependencies are available"
else
echo "❌ Failed to resolve test dependencies"
echo ""
echo "Attempting to show which dependencies are missing:"
go list -test -deps ${{ matrix.test-group.packages }} 2>&1 || true
exit 1
fi
echo ""
echo "Module cache statistics:"
echo "- Cache directory: $(go env GOMODCACHE)"
if [ -d "$(go env GOMODCACHE)" ]; then
echo "- Cache size: $(du -sh $(go env GOMODCACHE) 2>/dev/null | cut -f1 || echo 'unknown')"
echo "- Number of cached modules: $(find $(go env GOMODCACHE) -name "go.mod" 2>/dev/null | wc -l || echo 'unknown')"
fi
- name: Build gh-aw binary for integration tests
run: make build
- name: Run integration tests - ${{ matrix.test-group.name }}
id: run-tests
run: |
set -o pipefail
# Sanitize the test group name for use in filename
SAFE_NAME=$(echo "${{ matrix.test-group.name }}" | sed 's/[^a-zA-Z0-9]/-/g' | sed 's/--*/-/g')
if [ -z "${{ matrix.test-group.pattern }}" ]; then
# Catch-all group: run with -skip to exclude tests matched by other groups
if [ -n "${{ matrix.test-group.skip_pattern || '' }}" ]; then
go test -v -parallel=8 -timeout=10m -tags 'integration' -skip '${{ matrix.test-group.skip_pattern }}' -json ${{ matrix.test-group.packages }} | tee "test-result-integration-${SAFE_NAME}.json"
else
go test -v -parallel=8 -timeout=10m -tags 'integration' -json ${{ matrix.test-group.packages }} | tee "test-result-integration-${SAFE_NAME}.json"
fi
else
go test -v -parallel=8 -timeout=10m -tags 'integration' -run '${{ matrix.test-group.pattern }}' -json ${{ matrix.test-group.packages }} | tee "test-result-integration-${SAFE_NAME}.json"
fi
- name: Report test failures
if: failure() && steps.run-tests.outcome == 'failure'
run: |
# Sanitize the test group name to match the file created in the previous step
SAFE_NAME=$(echo "${{ matrix.test-group.name }}" | sed 's/[^a-zA-Z0-9]/-/g' | sed 's/--*/-/g')
echo "## 🔍 Test Failure Analysis" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Analyzing test results for: **${{ matrix.test-group.name }}**" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
# Run the failure report script
if ./scripts/report-test-failures.sh "test-result-integration-${SAFE_NAME}.json" | tee /tmp/failure-report.txt; then
echo "No failures detected in JSON output (unexpected - tests failed but no failure records found)" >> $GITHUB_STEP_SUMMARY
else
# Script found failures - add to summary
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
cat /tmp/failure-report.txt >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
fi
- name: Upload integration test results
if: always() # Upload even if tests fail so canary_go can track coverage
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: test-result-integration-${{ matrix.test-group.name }}
path: test-result-integration-*.json
retention-days: 14
canary_go:
runs-on: ubuntu-latest
needs: [integration] # test dependency removed - download-artifact fetches by name, not job dependency
if: always() # Run even if some tests fail to report coverage
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: List all tests in codebase
run: |
set -euo pipefail
echo "Extracting all test function names from source files..."
./scripts/list-all-tests.sh > all-tests.txt
echo "Found $(wc -l < all-tests.txt) tests in codebase"
- name: Download all test result artifacts
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
with:
path: test-results
pattern: test-result-*
merge-multiple: false
- name: List downloaded artifacts
run: |
set -euo pipefail
echo "Downloaded test result artifacts:"
find test-results -type f -name "*.json" | sort
echo ""
echo "Total JSON files: $(find test-results -type f -name "*.json" | wc -l)"
- name: Extract executed tests from artifacts
run: |
set -euo pipefail
echo "Extracting test names from JSON artifacts..."
./scripts/extract-executed-tests.sh test-results > executed-tests.txt
echo "Found $(wc -l < executed-tests.txt) executed tests"
- name: Compare test coverage
run: |
./scripts/compare-test-coverage.sh all-tests.txt executed-tests.txt
- name: Upload test coverage report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: test-coverage-analysis
path: |
all-tests.txt
executed-tests.txt
retention-days: 14
update:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-update
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Build gh-aw binary
run: make build
- name: Test update command (dry-run)
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "Testing update command to ensure it runs without issues..."
# Run update with verbose flag to check for updates without making changes
# The command checks for gh-aw updates, action updates, and workflow updates
./gh-aw update --verbose --no-actions
echo "✅ Update command executed successfully" >> $GITHUB_STEP_SUMMARY
build:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-build
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Node.js
id: setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6
with:
node-version: "24"
cache: npm
cache-dependency-path: actions/setup/js/package-lock.json
- name: Report Node cache status
run: |
if [ "${{ steps.setup-node.outputs.cache-hit }}" == "true" ]; then
echo "✅ Node cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Node cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: npm ci
run: npm ci
working-directory: ./actions/setup/js
- name: Build code
run: make build
- name: Upload Linux binary
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: gh-aw-linux-amd64
path: gh-aw
retention-days: 14
- name: Rebuild lock files
run: make recompile
env:
GH_TOKEN: ${{ github.token }}
validate-yaml:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Check for ANSI escape sequences in YAML files
run: |
echo "🔍 Scanning YAML workflow files for ANSI escape sequences..."
# Find all YAML files in .github/workflows directory
YAML_FILES=$(find .github/workflows -type f \( -name "*.yml" -o -name "*.yaml" \) | sort)
# Track if any ANSI codes are found
FOUND_ANSI=0
# Check each file for ANSI escape sequences
for file in $YAML_FILES; do
# Use grep to find ANSI escape sequences (ESC [ ... letter)
# The pattern matches: \x1b followed by [ followed by optional digits/semicolons followed by a letter
if grep -P '\x1b\[[0-9;]*[a-zA-Z]' "$file" > /dev/null 2>&1; then
echo "❌ ERROR: Found ANSI escape sequences in: $file"
echo ""
echo "Lines with ANSI codes:"
grep -n -P '\x1b\[[0-9;]*[a-zA-Z]' "$file" || true
echo ""
FOUND_ANSI=1
fi
done
if [ $FOUND_ANSI -eq 1 ]; then
echo ""
echo "💡 ANSI escape sequences detected in YAML files!"
echo ""
echo "These are terminal color codes that break YAML parsing."
echo "Common causes:"
echo " - Copy-pasting from colored terminal output"
echo " - Text editors preserving ANSI codes"
echo " - Scripts generating colored output"
echo ""
echo "To fix:"
echo " 1. Remove the ANSI codes from the affected files"
echo " 2. Run 'make recompile' to regenerate workflow files"
echo " 3. Use '--no-color' flags when capturing command output"
echo ""
exit 1
fi
echo "✅ No ANSI escape sequences found in YAML files"
js:
runs-on: ubuntu-latest
needs: validate-yaml
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-js
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Node.js
id: setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6
with:
node-version: "24"
cache: npm
cache-dependency-path: actions/setup/js/package-lock.json
- name: Report Node cache status
run: |
if [ "${{ steps.setup-node.outputs.cache-hit }}" == "true" ]; then
echo "✅ Node cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Node cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Install npm dependencies
run: cd actions/setup/js && npm ci
- name: Setup prompt templates for tests
run: |
mkdir -p /opt/gh-aw/prompts
cp actions/setup/md/*.md /opt/gh-aw/prompts/
- name: Run tests
run: cd actions/setup/js && npm test
js-integration-live-api:
runs-on: ubuntu-latest
needs: validate-yaml
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-js-integration-live-api
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Node.js
id: setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6
with:
node-version: "24"
cache: npm
cache-dependency-path: actions/setup/js/package-lock.json
- name: Report Node cache status
run: |
if [ "${{ steps.setup-node.outputs.cache-hit }}" == "true" ]; then
echo "✅ Node cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Node cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Install npm dependencies
run: cd actions/setup/js && npm ci
- name: Run live GitHub API integration test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "## 🔍 Live GitHub API Integration Test" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ -z "$GITHUB_TOKEN" ]; then
echo "⚠️ GITHUB_TOKEN not available - test will be skipped" >> $GITHUB_STEP_SUMMARY
echo "ℹ️ This is expected in forks or when secrets are not available" >> $GITHUB_STEP_SUMMARY
cd actions/setup/js && npm test -- frontmatter_hash_github_api.test.cjs
else
echo "✅ GITHUB_TOKEN available - running live API test" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
cd actions/setup/js && npm test -- frontmatter_hash_github_api.test.cjs
echo "" >> $GITHUB_STEP_SUMMARY
echo "✨ Live API test completed successfully" >> $GITHUB_STEP_SUMMARY
fi
bench:
# Only run benchmarks on main branch for performance tracking
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-bench
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Run benchmarks
run: make bench
- name: Display benchmark summary
run: |
echo "## 📊 Benchmark Results" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
# Show compiler benchmarks from the results
grep "BenchmarkCompile" bench_results.txt | head -20 >> $GITHUB_STEP_SUMMARY || echo "No benchmark results found" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "📁 Full results saved to artifact: benchmark-results" >> $GITHUB_STEP_SUMMARY
# Benchmark results for performance trend analysis - 14 days allows comparison across multiple runs
- name: Save benchmark results
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: benchmark-results
path: bench_results.txt
if-no-files-found: ignore
retention-days: 14
lint-go:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-lint-go
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
with:
fetch-depth: 0 # Fetch all history for incremental linting
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
# Go formatting check (fast, no deps needed)
- name: Check Go formatting
run: |
unformatted=$(go fmt ./...)
if [ -n "$unformatted" ]; then
echo "❌ Code is not formatted. Run 'make fmt' to fix." >> $GITHUB_STEP_SUMMARY
echo "Unformatted files:" >> $GITHUB_STEP_SUMMARY
echo "$unformatted" >> $GITHUB_STEP_SUMMARY
echo ""
echo "To fix this locally, run:"
echo " make fmt"
echo ""
echo "Or format individual files with:"
echo " go fmt ./path/to/file.go"
exit 1
fi
echo "✅ Go formatting check passed" >> $GITHUB_STEP_SUMMARY
# Install only golangci-lint (the only tool needed for linting)
# Other tools (actionlint, gosec, gopls, govulncheck) are not used in this job
- name: Install golangci-lint
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.7.2
# Run golangci-lint via Makefile for consistency
# Uses incremental linting on PRs for faster CI (50-75% speedup)
# Performance optimizations in .golangci.yml:
# - timeout: 5m prevents hanging
# - modules-download-mode: readonly uses cached modules only
- name: Run golangci-lint
run: |
export PATH="$PATH:$(go env GOPATH)/bin"
if [ "${{ github.event_name }}" = "pull_request" ]; then
# Incremental linting on PRs - only check changed files
# This provides 50-75% faster linting on typical PRs
BASE_REF="origin/${{ github.base_ref }}"
if git rev-parse --verify "$BASE_REF" >/dev/null 2>&1; then
echo "Using incremental lint against $BASE_REF"
make golint-incremental BASE_REF="$BASE_REF"
else
echo "⚠️ Base ref $BASE_REF not found, falling back to full lint"
make golint
fi
else
# Full scan on main branch to ensure comprehensive coverage
make golint
fi
# Error message linting (requires Go only)
- name: Lint error messages
run: make lint-errors
lint-js:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-lint-js
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Node.js
id: setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6
with:
node-version: "24"
cache: npm
cache-dependency-path: actions/setup/js/package-lock.json
- name: Report Node cache status
run: |
if [ "${{ steps.setup-node.outputs.cache-hit }}" == "true" ]; then
echo "✅ Node cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Node cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Install npm dependencies
run: cd actions/setup/js && npm ci
# JavaScript and JSON formatting checks
- name: Lint JavaScript files
run: make lint-cjs
- name: Check JSON formatting
run: make fmt-check-json
audit:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-audit
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Build gh-aw binary
run: make build
- name: Run dependency audit (human-readable)
run: |
echo "## Dependency Health Audit" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
./gh-aw update --audit 2>&1 | tee audit_output.txt || true
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
head -100 audit_output.txt >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
- name: Run dependency audit (JSON)
id: audit_json
run: |
# Run audit with JSON output for agent-friendly parsing
./gh-aw update --audit --json > audit.json 2>&1
# Display summary in GitHub Actions
echo "✅ Dependency audit completed" >> $GITHUB_STEP_SUMMARY
# Extract key metrics
TOTAL_DEPS=$(jq '.summary.total_dependencies' audit.json)
OUTDATED=$(jq '.summary.outdated_count' audit.json)
SECURITY=$(jq '.summary.security_advisories' audit.json)
V0_PERCENT=$(jq '.summary.v0_percentage' audit.json)
echo "📊 **Audit Results:**" >> $GITHUB_STEP_SUMMARY
echo "- Total dependencies: $TOTAL_DEPS" >> $GITHUB_STEP_SUMMARY
echo "- Outdated: $OUTDATED" >> $GITHUB_STEP_SUMMARY
echo "- Security advisories: $SECURITY" >> $GITHUB_STEP_SUMMARY
echo "- v0.x exposure: ${V0_PERCENT}%" >> $GITHUB_STEP_SUMMARY
- name: Upload audit results
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: dependency-audit
path: |
audit.json
audit_output.txt
retention-days: 30
actions-build:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-actions-build
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Build actions
run: make actions-build
- name: Validate actions
run: make actions-validate
fuzz:
# Only run fuzz tests on main branch (10s is insufficient for PRs)
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-fuzz
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Run fuzz tests
run: |
set -o pipefail
# Create directory for fuzz results
mkdir -p fuzz-results
# Helper function to run fuzz test and handle context deadline
# Go fuzz tests can exit with status 1 and "context deadline exceeded" when
# they reach the -fuzztime limit. We treat this as expected success to allow
# all fuzz targets to run instead of stopping at the first timeout.
run_fuzz_test() {
local fuzz_name=$1
local package=$2
local output_file="fuzz-results/${fuzz_name}.txt"
echo "Running ${fuzz_name}..."
if go test -run='^$' -fuzz="${fuzz_name}" -fuzztime=10s "${package}" 2>&1 | tee "${output_file}"; then
echo "✅ ${fuzz_name} completed successfully"
return 0
else
# Check if the failure was due to context deadline (expected)
if grep -q "context deadline exceeded" "${output_file}"; then
echo "✅ ${fuzz_name} completed (context deadline reached as expected)"
return 0
else
echo "❌ ${fuzz_name} failed with unexpected error"
return 1
fi
fi
}
# Run fuzz tests and capture output
run_fuzz_test "FuzzParseFrontmatter" "./pkg/parser/"
run_fuzz_test "FuzzScheduleParser" "./pkg/parser/"
run_fuzz_test "FuzzExpressionParser" "./pkg/workflow/"
run_fuzz_test "FuzzMentionsFiltering" "./pkg/workflow/"
run_fuzz_test "FuzzSanitizeOutput" "./pkg/workflow/"
run_fuzz_test "FuzzSanitizeIncomingText" "./pkg/workflow/"
run_fuzz_test "FuzzSanitizeLabelContent" "./pkg/workflow/"
run_fuzz_test "FuzzWrapExpressionsInTemplateConditionals" "./pkg/workflow/"
run_fuzz_test "FuzzYAMLParsing" "./pkg/workflow/"
run_fuzz_test "FuzzTemplateRendering" "./pkg/workflow/"
run_fuzz_test "FuzzInputValidation" "./pkg/workflow/"
run_fuzz_test "FuzzNetworkPermissions" "./pkg/workflow/"
run_fuzz_test "FuzzSafeJobConfig" "./pkg/workflow/"
# Copy fuzz corpus data (testdata/fuzz directories)
echo "Copying fuzz corpus data..."
find ./pkg -path "*/testdata/fuzz" -type d | while read -r dir; do
pkg_name=$(echo "$dir" | sed 's|^\./pkg/||' | sed 's|/testdata/fuzz$||')
echo "Copying corpus from $dir to fuzz-results/corpus/$pkg_name/"
mkdir -p "fuzz-results/corpus/$pkg_name"
cp -r "$dir"/* "fuzz-results/corpus/$pkg_name/" 2>/dev/null || echo "No corpus data in $dir"
done
- name: Upload fuzz test results
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: fuzz-results
path: fuzz-results/
retention-days: 14
security:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-security
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Run security regression tests
run: make test-security
security-scan:
# Only run security scans on main branch to reduce PR overhead
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 10 # Prevent jobs from hanging indefinitely
permissions:
contents: read
strategy:
fail-fast: false
matrix:
tool:
- name: zizmor
flag: --zizmor
- name: actionlint
flag: --actionlint
- name: poutine
flag: --poutine
concurrency:
group: ci-${{ github.ref }}-security-scan-${{ matrix.tool.name }}
cancel-in-progress: true
name: "Security Scan: ${{ matrix.tool.name }}"
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Build gh-aw
run: make build
- name: Run ${{ matrix.tool.name }} security scan on poem workflow
run: ./gh-aw compile poem-bot ${{ matrix.tool.flag }} --verbose
logs-token-check:
if: false
runs-on: ubuntu-latest
permissions:
contents: read
actions: read
concurrency:
group: ci-${{ github.ref }}-logs-token-check
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Build gh-aw
run: make build
- name: Run logs command with JSON output
id: logs_check
run: |
set -e # Fail on first error
# Run the logs command and capture only stdout (JSON output)
# stderr is not redirected, so warning messages go to console
./gh-aw logs smoke-copilot -c 2 --json --verbose > logs_output.json
# Display the output for debugging
echo "Logs command output:"
cat logs_output.json
# Check if the JSON structure is valid
echo "## Validating JSON Structure"
# Check if token count is found in the JSON output
if jq -e '.summary.total_tokens' logs_output.json > /dev/null 2>&1; then
TOKEN_COUNT=$(jq '.summary.total_tokens' logs_output.json)
echo "✅ Token count found: $TOKEN_COUNT"
# Validate that token count is greater than 0
if [ "$TOKEN_COUNT" -gt 0 ]; then
echo "✅ Token count is greater than 0: $TOKEN_COUNT"
echo "token_count=$TOKEN_COUNT" >> $GITHUB_OUTPUT
else
echo "❌ Token count is 0 - expected tokens to be parsed from logs"
exit 1
fi
else
echo "❌ Token count not found in JSON output"
exit 1
fi
# Check if runs array exists (even if empty)
if jq -e '.runs' logs_output.json > /dev/null 2>&1; then
RUNS_COUNT=$(jq '.runs | length' logs_output.json)
echo "✅ Runs array found: $RUNS_COUNT runs"
else
echo "❌ Runs array not found in JSON output"
exit 1
fi
# If there are runs, validate that key fields are resolved
if [ "$RUNS_COUNT" -gt 0 ]; then
# Check if agent (engine_id) field exists in first run
if jq -e '.runs[0] | has("agent")' logs_output.json > /dev/null 2>&1; then
AGENT=$(jq -r '.runs[0].agent // "null"' logs_output.json)
echo "✅ Agent field found in run: $AGENT"
else
echo "❌ Agent field not found in run data"
exit 1
fi
# Check if workflow_path field exists in first run
if jq -e '.runs[0] | has("workflow_path")' logs_output.json > /dev/null 2>&1; then
WORKFLOW_PATH=$(jq -r '.runs[0].workflow_path // "null"' logs_output.json)
echo "✅ Workflow path field found in run: $WORKFLOW_PATH"
else
echo "❌ Workflow path field not found in run data"
exit 1
fi
# Check if workflow_name is present
if jq -e '.runs[0].workflow_name' logs_output.json > /dev/null 2>&1; then
WORKFLOW_NAME=$(jq -r '.runs[0].workflow_name' logs_output.json)
echo "✅ Workflow name found in run: $WORKFLOW_NAME"
else
echo "❌ Workflow name not found in run data"
exit 1
fi
else
echo "ℹ️ No runs found to validate (this is ok)"
fi
echo "✅ All JSON structure validations passed"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
mcp-server-compile-test:
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-mcp-server-compile-test
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Build gh-aw binary
run: make build
- name: Create test workflow with error
run: |
mkdir -p .github/workflows
cat > .github/workflows/test-invalid.md << 'EOF'
---
on: push
engine: copilot
invalid_field: this will cause an error
---
# Test Invalid Workflow
This workflow has an invalid field that will cause a compilation error.
EOF
- name: Test MCP server compile tool
run: |
# Create a test script using the MCP Go SDK
cat > test_mcp_compile.go << 'GOEOF'
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"time"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
func main() {
// Create MCP client
client := mcp.NewClient(&mcp.Implementation{
Name: "ci-test-client",
Version: "1.0.0",
}, nil)
// Start the MCP server as a subprocess with absolute path
binaryPath := "./gh-aw"
serverCmd := exec.Command(binaryPath, "mcp-server", "--cmd", binaryPath)
transport := &mcp.CommandTransport{Command: serverCmd}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Connect to the server
session, err := client.Connect(ctx, transport, nil)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to connect to MCP server: %v\n", err)
os.Exit(1)
}
defer session.Close()
fmt.Println("✅ Successfully connected to MCP server")
// Call the compile tool with the invalid workflow
params := &mcp.CallToolParams{
Name: "compile",
Arguments: map[string]any{
"workflows": []string{"test-invalid.md"},
},
}
result, err := session.CallTool(ctx, params)
if err != nil {
fmt.Fprintf(os.Stderr, "MCP tool call returned error (this is expected): %v\n", err)
// Check if the error contains expected error information
fmt.Println("✅ Compile tool correctly returned an error")
os.Exit(0)
}
// Get the result content
if len(result.Content) == 0 {
fmt.Fprintln(os.Stderr, "❌ Expected non-empty result from compile tool")
os.Exit(1)
}
textContent, ok := result.Content[0].(*mcp.TextContent)
if !ok {
fmt.Fprintln(os.Stderr, "❌ Expected text content from compile tool")
os.Exit(1)
}
fmt.Printf("Compile tool output:\n%s\n", textContent.Text)
// Parse the JSON output to check for errors
var compileResults []map[string]any
if err := json.Unmarshal([]byte(textContent.Text), &compileResults); err != nil {
fmt.Fprintf(os.Stderr, "Failed to parse JSON output: %v\n", err)
os.Exit(1)
}
// Check if the workflow is marked as invalid
if len(compileResults) == 0 {
fmt.Fprintln(os.Stderr, "❌ Expected at least one workflow result")
os.Exit(1)
}
result0 := compileResults[0]
valid, ok := result0["valid"].(bool)
if !ok {
fmt.Fprintln(os.Stderr, "❌ Expected 'valid' field in result")
os.Exit(1)
}
if valid {
fmt.Fprintln(os.Stderr, "❌ Expected workflow to be invalid")
os.Exit(1)
}
// Check that errors field exists and has at least one error
errors, ok := result0["errors"].([]any)
if !ok || len(errors) == 0 {
fmt.Fprintln(os.Stderr, "❌ Expected errors array with at least one error")
os.Exit(1)
}
fmt.Println("✅ Compile tool correctly reported validation errors:")
errorsJSON, _ := json.MarshalIndent(errors, " ", " ")
fmt.Printf(" %s\n", string(errorsJSON))
os.Exit(0)
}
GOEOF
# Run the test
go run test_mcp_compile.go
- name: Report test results
if: always()
run: |
echo "## MCP Server Compile Tool Test" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "This test verifies that:" >> $GITHUB_STEP_SUMMARY
echo "1. The gh-aw MCP server can be started successfully" >> $GITHUB_STEP_SUMMARY
echo "2. The compile tool can be invoked through the MCP server" >> $GITHUB_STEP_SUMMARY
echo "3. The compile tool correctly detects and reports validation errors" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "✅ Test completed successfully" >> $GITHUB_STEP_SUMMARY
cross-platform-build:
name: Build & Test on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
permissions:
contents: read
strategy:
fail-fast: false
matrix:
os:
- macos-latest
- windows-latest
concurrency:
group: ci-${{ github.ref }}-cross-platform-${{ matrix.os }}
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
shell: bash
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
shell: bash
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Build gh-aw binary
run: make build
- name: Create test workflow
shell: bash
run: |
mkdir -p .github/workflows
cat > .github/workflows/test-cross-platform.md << 'EOF'
---
on: push
engine: copilot
---
# Test Workflow for Cross-Platform CI
This is a simple test workflow to verify the compile command works correctly.
## Task
Echo hello world.
EOF
- name: Test compile command
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "Testing compile command on ${{ matrix.os }}..."
# Determine binary name based on OS
if [[ "$RUNNER_OS" == "Windows" ]]; then
BINARY="./gh-aw.exe"
else
BINARY="./gh-aw"
fi
# Verify binary exists
if [ ! -f "$BINARY" ]; then
echo "❌ Binary not found: $BINARY"
ls -la
exit 1
fi
# Run compile command
"$BINARY" compile test-cross-platform --verbose
# Check if lock file was generated
if [ -f ".github/workflows/test-cross-platform.lock.yml" ]; then
echo "✅ Compile succeeded - lock file generated"
else
echo "❌ Compile failed - no lock file generated"
exit 1
fi
echo "## Cross-Platform Test Results" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "✅ Successfully compiled workflow on ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**Platform:** ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY
echo "**Binary:** $BINARY" >> $GITHUB_STEP_SUMMARY
echo "**Go version:** $(go version)" >> $GITHUB_STEP_SUMMARY
- name: Clean up test files
if: always()
shell: bash
run: |
rm -f .github/workflows/test-cross-platform.md
rm -f .github/workflows/test-cross-platform.lock.yml
alpine-container-test:
name: Alpine Container Test
runs-on: ubuntu-latest
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}-alpine-container
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
- name: Set up Go
id: setup-go
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6
with:
go-version-file: go.mod
cache: true
- name: Report Go cache status
run: |
if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then
echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY
else
echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY
fi
- name: Download dependencies with retry
run: |
set -e
MAX_RETRIES=3
RETRY_DELAY=5
for i in $(seq 1 $MAX_RETRIES); do
echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..."
if go mod download; then
echo "✅ Successfully downloaded Go modules"
break
else
if [ $i -eq $MAX_RETRIES ]; then
echo "❌ Failed to download Go modules after $MAX_RETRIES attempts"
echo "This may indicate that proxy.golang.org is unreachable"
echo "Please check network connectivity or consider vendoring dependencies"
exit 1
fi
echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..."
sleep $RETRY_DELAY
fi
done
- name: Verify dependencies
run: go mod verify
- name: Build Linux binary for Alpine
run: make build-linux
- name: Build Alpine Docker image
run: |
echo "Building Alpine Docker image..."
docker build -t gh-aw-alpine:test \
--build-arg BINARY=gh-aw-linux-amd64 \
-f Dockerfile .
echo "✅ Alpine Docker image built successfully"
- name: Test Docker image basic commands
run: |
echo "Testing Docker image basic commands..."
docker run --rm gh-aw-alpine:test --version
docker run --rm gh-aw-alpine:test --help
echo "✅ Basic commands work"
- name: Create test workflow in container
run: |
echo "Creating test workflow file..."
mkdir -p test-workspace/.github/workflows
cat > test-workspace/.github/workflows/test-alpine.md << 'EOF'
---
on: push
engine: copilot
---
# Test Workflow for Alpine Container
This is a simple test workflow to verify the compile command works correctly in Alpine container.
## Task
Echo hello from Alpine container.
EOF
echo "✅ Test workflow created"
- name: Run compile through Alpine container
run: |
echo "Running compile command through Alpine container..."
docker run --rm \
-v "$(pwd)/test-workspace:/workspace" \
-w /workspace \
gh-aw-alpine:test compile test-alpine --verbose
echo "✅ Compile command executed"
- name: Verify lock file generation
run: |
echo "Verifying lock file was generated..."
if [ -f "test-workspace/.github/workflows/test-alpine.lock.yml" ]; then
echo "✅ Lock file generated successfully"
echo ""
echo "Lock file contents:"
head -20 test-workspace/.github/workflows/test-alpine.lock.yml
else
echo "❌ Lock file not found"
ls -la test-workspace/.github/workflows/
exit 1
fi
- name: Generate test summary
if: always()
run: |
echo "## Alpine Container Test Results" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "This test verifies that:" >> $GITHUB_STEP_SUMMARY
echo "1. The Alpine Docker image can be built successfully" >> $GITHUB_STEP_SUMMARY
echo "2. The gh-aw binary works correctly in Alpine Linux" >> $GITHUB_STEP_SUMMARY
echo "3. The compile command can process workflows in the container" >> $GITHUB_STEP_SUMMARY
echo "4. Lock files are generated correctly" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ -f "test-workspace/.github/workflows/test-alpine.lock.yml" ]; then
echo "✅ All tests passed successfully" >> $GITHUB_STEP_SUMMARY
else
echo "❌ Lock file generation failed" >> $GITHUB_STEP_SUMMARY
fi
- name: Clean up test files
if: always()
run: |
rm -rf test-workspace