diff --git a/.gitea/workflows-archive/ci-cd.yml b/.gitea/workflows-archive/ci-cd.yml new file mode 100644 index 0000000..32dd0c8 --- /dev/null +++ b/.gitea/workflows-archive/ci-cd.yml @@ -0,0 +1,195 @@ +name: CI/CD Pipeline + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + +env: + DOCKER_REGISTRY: docker.io + IMAGE_NAME: basil + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: basil + POSTGRES_PASSWORD: basil + POSTGRES_DB: basil_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm install + + - name: Build shared package + run: | + cd packages/shared + npm run build + + - name: Generate Prisma Client + run: | + cd packages/api + npm run prisma:generate + + - name: Run database migrations + run: | + cd packages/api + npm run prisma:migrate + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil_test?schema=public + + - name: Run unit tests - API + run: | + cd packages/api + npm run test + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil_test?schema=public + NODE_ENV: test + + - name: Run unit tests - Web + run: | + cd packages/web + npm run test + + - name: Run unit tests - Shared + run: | + cd packages/shared + npm run test + + - name: Install Playwright browsers + run: npx playwright install --with-deps + + - name: Build application for E2E tests + run: npm run build + + - name: Run E2E tests + run: npm run test:e2e + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil_test?schema=public + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results + path: | + packages/*/coverage/ + playwright-report/ + retention-days: 30 + + build-and-push: + name: Build and Push Docker Images + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata for API + id: meta-api + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_REGISTRY }}/${{ secrets.DOCKER_USERNAME }}/${{ env.IMAGE_NAME }}-api + tags: | + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Extract metadata for Web + id: meta-web + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_REGISTRY }}/${{ secrets.DOCKER_USERNAME }}/${{ env.IMAGE_NAME }}-web + tags: | + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push API image + uses: docker/build-push-action@v5 + with: + context: . + file: packages/api/Dockerfile + push: true + tags: ${{ steps.meta-api.outputs.tags }} + labels: ${{ steps.meta-api.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push Web image + uses: docker/build-push-action@v5 + with: + context: . + file: packages/web/Dockerfile + push: true + tags: ${{ steps.meta-web.outputs.tags }} + labels: ${{ steps.meta-web.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Trigger deployment webhook + if: success() + run: | + curl -X POST ${{ secrets.DEPLOY_WEBHOOK_URL }} \ + -H "Content-Type: application/json" \ + -d '{"branch": "main", "commit": "${{ github.sha }}", "message": "${{ github.event.head_commit.message }}"}' + + lint: + name: Code Quality + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm install + + - name: Run linter + run: npm run lint diff --git a/.gitea/workflows-archive/ci.yml b/.gitea/workflows-archive/ci.yml new file mode 100644 index 0000000..9cfd701 --- /dev/null +++ b/.gitea/workflows-archive/ci.yml @@ -0,0 +1,183 @@ +name: CI Pipeline + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + +jobs: + lint: + name: Lint Code + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run linters + run: npm run lint + + test-api: + name: Test API Package + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: basil + POSTGRES_PASSWORD: basil + POSTGRES_DB: basil_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run API tests + working-directory: packages/api + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil_test + NODE_ENV: test + run: npm run test + + - name: Upload API test coverage + uses: actions/upload-artifact@v4 + if: always() + with: + name: api-coverage + path: packages/api/coverage/ + retention-days: 7 + + test-web: + name: Test Web Package + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run web tests + working-directory: packages/web + run: npm run test + + - name: Upload web test coverage + uses: actions/upload-artifact@v4 + if: always() + with: + name: web-coverage + path: packages/web/coverage/ + retention-days: 7 + + test-shared: + name: Test Shared Package + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run shared package tests + working-directory: packages/shared + run: npm run test + + - name: Upload shared test coverage + uses: actions/upload-artifact@v4 + if: always() + with: + name: shared-coverage + path: packages/shared/coverage/ + retention-days: 7 + + build: + name: Build All Packages + runs-on: ubuntu-latest + needs: [lint, test-api, test-web, test-shared] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build all packages + run: npm run build + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + packages/api/dist/ + packages/web/dist/ + packages/shared/dist/ + retention-days: 7 + + coverage-report: + name: Generate Coverage Report + runs-on: ubuntu-latest + needs: [test-api, test-web, test-shared] + if: always() + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all coverage artifacts + uses: actions/download-artifact@v4 + with: + path: coverage-artifacts + + - name: Display coverage summary + run: | + echo "## Test Coverage Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Coverage reports have been generated for all packages." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- API Package Coverage" >> $GITHUB_STEP_SUMMARY + echo "- Web Package Coverage" >> $GITHUB_STEP_SUMMARY + echo "- Shared Package Coverage" >> $GITHUB_STEP_SUMMARY diff --git a/.gitea/workflows-archive/docker.yml b/.gitea/workflows-archive/docker.yml new file mode 100644 index 0000000..1f75cdd --- /dev/null +++ b/.gitea/workflows-archive/docker.yml @@ -0,0 +1,146 @@ +name: Docker Build & Deploy + +on: + push: + branches: [ main, master ] + tags: + - 'v*' + pull_request: + branches: [ main, master ] + +jobs: + build-and-test: + name: Build Docker Images + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build API image + uses: docker/build-push-action@v5 + with: + context: . + file: ./packages/api/Dockerfile + push: false + tags: basil-api:test + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build Web image + uses: docker/build-push-action@v5 + with: + context: . + file: ./packages/web/Dockerfile + push: false + tags: basil-web:test + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Test Docker Compose + run: | + docker-compose -f docker-compose.yml config + echo "✅ Docker Compose configuration is valid" + + push-images: + name: Push Docker Images + runs-on: ubuntu-latest + needs: build-and-test + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ secrets.DOCKER_REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata + id: meta + run: | + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + else + VERSION=latest + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + + - name: Build and push API image + uses: docker/build-push-action@v5 + with: + context: . + file: ./packages/api/Dockerfile + push: true + tags: | + ${{ secrets.DOCKER_REGISTRY }}/basil-api:${{ steps.meta.outputs.version }} + ${{ secrets.DOCKER_REGISTRY }}/basil-api:latest + labels: | + org.opencontainers.image.created=${{ steps.meta.outputs.date }} + org.opencontainers.image.version=${{ steps.meta.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push Web image + uses: docker/build-push-action@v5 + with: + context: . + file: ./packages/web/Dockerfile + push: true + tags: | + ${{ secrets.DOCKER_REGISTRY }}/basil-web:${{ steps.meta.outputs.version }} + ${{ secrets.DOCKER_REGISTRY }}/basil-web:latest + labels: | + org.opencontainers.image.created=${{ steps.meta.outputs.date }} + org.opencontainers.image.version=${{ steps.meta.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Image digest + run: echo "Images have been built and pushed successfully" + + deploy-staging: + name: Deploy to Staging + runs-on: ubuntu-latest + needs: push-images + if: github.ref == 'refs/heads/develop' + environment: + name: staging + url: https://staging.basil.example.com + steps: + - name: Deploy to staging + run: | + echo "Deploying to staging environment..." + echo "This is a placeholder for actual deployment steps." + echo "Examples: SSH to server, run docker-compose pull, restart services, etc." + + deploy-production: + name: Deploy to Production + runs-on: ubuntu-latest + needs: push-images + if: startsWith(github.ref, 'refs/tags/v') + environment: + name: production + url: https://basil.example.com + steps: + - name: Deploy to production + run: | + echo "Deploying to production environment..." + echo "This is a placeholder for actual deployment steps." + echo "Examples: SSH to server, run docker-compose pull, restart services, etc." + + - name: Create deployment summary + run: | + echo "# 🚀 Deployment Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Version**: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY + echo "**Environment**: Production" >> $GITHUB_STEP_SUMMARY + echo "**Status**: Deployed Successfully ✅" >> $GITHUB_STEP_SUMMARY diff --git a/.gitea/workflows-archive/e2e.yml b/.gitea/workflows-archive/e2e.yml new file mode 100644 index 0000000..8ec80aa --- /dev/null +++ b/.gitea/workflows-archive/e2e.yml @@ -0,0 +1,148 @@ +name: E2E Tests + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + schedule: + # Run E2E tests nightly at 2 AM UTC + - cron: '0 2 * * *' + +jobs: + e2e-tests: + name: End-to-End Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: basil + POSTGRES_PASSWORD: basil + POSTGRES_DB: basil + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Install Playwright browsers + run: npx playwright install --with-deps + + - name: Build packages + run: npm run build + + - name: Run database migrations + working-directory: packages/api + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil + run: npm run prisma:migrate + + - name: Start application + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil + PORT: 3001 + NODE_ENV: test + run: | + npm run dev & + sleep 10 + + - name: Run E2E tests + run: npm run test:e2e + + - name: Upload Playwright report + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: playwright-report/ + retention-days: 14 + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-results + path: test-results/ + retention-days: 7 + + e2e-mobile: + name: E2E Tests (Mobile) + runs-on: ubuntu-latest + timeout-minutes: 30 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: basil + POSTGRES_PASSWORD: basil + POSTGRES_DB: basil + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Install Playwright browsers + run: npx playwright install --with-deps + + - name: Build packages + run: npm run build + + - name: Run database migrations + working-directory: packages/api + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil + run: npm run prisma:migrate + + - name: Start application + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil + PORT: 3001 + NODE_ENV: test + run: | + npm run dev & + sleep 10 + + - name: Run E2E tests on mobile + run: npx playwright test --project="Mobile Chrome" --project="Mobile Safari" + + - name: Upload mobile test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-mobile-results + path: test-results/ + retention-days: 7 diff --git a/.gitea/workflows-archive/security.yml b/.gitea/workflows-archive/security.yml new file mode 100644 index 0000000..9301849 --- /dev/null +++ b/.gitea/workflows-archive/security.yml @@ -0,0 +1,146 @@ +name: Security Scanning + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master ] + schedule: + # Run security scans weekly on Monday at 9 AM UTC + - cron: '0 9 * * 1' + +jobs: + dependency-audit: + name: NPM Audit + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run npm audit + run: npm audit --audit-level=moderate + continue-on-error: true + + - name: Run npm audit in API package + working-directory: packages/api + run: npm audit --audit-level=moderate + continue-on-error: true + + - name: Run npm audit in Web package + working-directory: packages/web + run: npm audit --audit-level=moderate + continue-on-error: true + + - name: Generate audit report + if: always() + run: | + echo "## Security Audit Report" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "NPM audit has been completed for all packages." >> $GITHUB_STEP_SUMMARY + echo "Review the logs above for any vulnerabilities." >> $GITHUB_STEP_SUMMARY + + dependency-check: + name: Dependency License Check + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Check for outdated dependencies + run: npm outdated || true + + - name: List all dependencies + run: | + echo "## Dependency List" >> $GITHUB_STEP_SUMMARY + npm list --all || true + + code-scanning: + name: Code Quality Scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run ESLint with security rules + run: npm run lint + continue-on-error: true + + - name: Check for hardcoded secrets (basic) + run: | + echo "Scanning for potential secrets..." + if grep -r -i -E "(password|secret|api[_-]?key|token|credential)" --include="*.ts" --include="*.js" --exclude-dir=node_modules --exclude-dir=dist . | grep -v "process.env" | grep -v "// "; then + echo "⚠️ Warning: Potential hardcoded secrets found!" + echo "Review the results above and ensure no sensitive data is committed." + else + echo "✅ No obvious hardcoded secrets detected." + fi + + docker-security: + name: Docker Image Security + runs-on: ubuntu-latest + if: github.event_name == 'push' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Build Docker images + run: docker-compose build + + - name: Scan Docker images for vulnerabilities + run: | + echo "## Docker Security Scan" >> $GITHUB_STEP_SUMMARY + echo "Docker images have been built successfully." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Consider using tools like Trivy or Snyk for comprehensive vulnerability scanning." >> $GITHUB_STEP_SUMMARY + + security-summary: + name: Security Summary + runs-on: ubuntu-latest + needs: [dependency-audit, dependency-check, code-scanning] + if: always() + steps: + - name: Generate security summary + run: | + echo "# 🔒 Security Scan Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "All security scans have been completed." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Scans Performed:" >> $GITHUB_STEP_SUMMARY + echo "- ✅ NPM Dependency Audit" >> $GITHUB_STEP_SUMMARY + echo "- ✅ Dependency License Check" >> $GITHUB_STEP_SUMMARY + echo "- ✅ Code Quality Scanning" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Review individual job logs for detailed results." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Recommended Additional Tools:" >> $GITHUB_STEP_SUMMARY + echo "- **Snyk**: For advanced vulnerability scanning" >> $GITHUB_STEP_SUMMARY + echo "- **Trivy**: For Docker image scanning" >> $GITHUB_STEP_SUMMARY + echo "- **SonarQube**: For code quality and security analysis" >> $GITHUB_STEP_SUMMARY + echo "- **Dependabot**: For automated dependency updates" >> $GITHUB_STEP_SUMMARY diff --git a/.gitea/workflows/README.md b/.gitea/workflows/README.md new file mode 100644 index 0000000..af9b221 --- /dev/null +++ b/.gitea/workflows/README.md @@ -0,0 +1,356 @@ +# Basil CI/CD Pipeline Documentation + +## Overview + +The Basil CI/CD pipeline is a comprehensive workflow that automates testing, building, and deployment of the Basil recipe management application. It consolidates all quality checks, security scanning, and deployment automation into a single workflow file. + +## Workflow Triggers + +The pipeline is triggered on: + +- **Push to main or develop branches**: Full pipeline with deployment (main only) +- **Pull requests to main or develop**: Tests and builds only, no deployment +- **Tagged releases** (v*): Full pipeline with semantic version tagging + +## Pipeline Stages + +### Stage 1: Parallel Quality Checks (~8 minutes) + +All these jobs run in parallel for maximum efficiency: + +- **lint**: ESLint code quality checks +- **test-api**: API unit tests with PostgreSQL service +- **test-web**: React web application unit tests +- **test-shared**: Shared package unit tests +- **security-scan**: NPM audit, secret scanning, dependency checks + +### Stage 2: Build Verification (~7 minutes) + +- **build**: Compiles all packages (depends on Stage 1 passing) + +### Stage 3: E2E Testing (~15 minutes) + +- **e2e-tests**: Playwright end-to-end tests (depends on build) + +### Stage 4: Docker & Deployment (~11 minutes, main branch only) + +- **docker-build-and-push**: Builds and pushes Docker images to Harbor +- **trigger-deployment**: Calls webhook to trigger server-side deployment + +## Total Pipeline Duration + +- **Pull Request**: ~30 minutes (Stages 1-3) +- **Main Branch Deploy**: ~41 minutes (All stages) + +## Required Gitea Secrets + +Configure these in your Gitea repository settings (Settings → Secrets): + +| Secret Name | Description | Example | +|-------------|-------------|---------| +| `HARBOR_REGISTRY` | Harbor registry URL | `harbor.pkartchner.com` | +| `HARBOR_USERNAME` | Harbor robot account username | `robot$basil+basil-cicd` | +| `HARBOR_PASSWORD` | Harbor robot account token | `ErJh8ze6VvZDnviVwc97Jevf6CrdzRBu` | +| `HARBOR_PROJECT` | Harbor project name | `basil` | +| `WEBHOOK_URL` | Deployment webhook endpoint | `http://localhost:9000/hooks/basil-deploy` | +| `WEBHOOK_SECRET` | Webhook authentication secret | `4cd30192f203f5ea905...` | + +## Image Naming Convention + +### Tags + +The workflow creates multiple tags for each image: + +- `latest`: Latest build from main branch +- `main-{short-sha}`: Specific commit (e.g., `main-abc1234`) +- `v{version}`: Semantic version tags (for tagged releases) + +### Image Names + +``` +harbor.pkartchner.com/basil/basil-api:latest +harbor.pkartchner.com/basil/basil-api:main-abc1234 +harbor.pkartchner.com/basil/basil-web:latest +harbor.pkartchner.com/basil/basil-web:main-abc1234 +``` + +## Deployment Process + +### Automated Deployment (Main Branch) + +1. Developer pushes to `main` branch +2. Pipeline runs all tests and builds +3. Docker images built and pushed to Harbor +4. Webhook triggered with deployment payload +5. Server receives webhook and runs deployment script +6. Script pulls images from Harbor +7. Docker Compose restarts containers with new images +8. Health checks verify successful deployment + +### Manual Deployment + +If you need to deploy manually or rollback: + +```bash +cd /srv/docker-compose/basil + +# Deploy latest +export IMAGE_TAG=latest +./scripts/deploy.sh + +# Deploy specific version +export IMAGE_TAG=main-abc1234 +./scripts/deploy.sh +``` + +## Security Features + +### Security Gates + +- **NPM Audit**: Checks for vulnerable dependencies (HIGH/CRITICAL) +- **Secret Scanning**: Detects hardcoded credentials in code +- **Trivy Image Scanning**: Scans Docker images for vulnerabilities +- **Dependency Checking**: Reports outdated packages + +### Fail-Fast Behavior + +- All tests must pass before Docker build starts +- Health checks must pass before deployment completes +- Any security scan failure stops the pipeline + +## Caching Strategy + +The workflow uses GitHub Actions cache to speed up builds: + +- **NPM Dependencies**: Cached between runs +- **Docker Layers**: Cached using GitHub Actions cache backend +- **Playwright Browsers**: Cached for E2E tests + +## Artifacts + +The workflow uploads artifacts that are retained for 7-14 days: + +- **Test Coverage**: Unit test coverage reports for all packages +- **Playwright Reports**: E2E test results and screenshots +- **Build Artifacts**: Compiled JavaScript/TypeScript output + +## Monitoring + +### View Workflow Runs + +1. Go to your Gitea repository +2. Click the "Actions" tab +3. Select a workflow run to see detailed logs + +### Check Deployment Status + +```bash +# Webhook service logs +journalctl -u basil-webhook -f + +# Deployment script logs +tail -f /srv/docker-compose/basil/deploy.log + +# Container status +docker ps | grep basil + +# Application health +curl https://basil.pkartchner.com/health +``` + +## Rollback Procedures + +### Scenario 1: Bad Deployment + +Deploy a previous working version: + +```bash +cd /srv/docker-compose/basil +export IMAGE_TAG=main-abc1234 # Previous working SHA +./scripts/deploy.sh +``` + +### Scenario 2: Rollback Workflow Changes + +Restore previous workflows: + +```bash +cd /srv/docker-compose/basil +rm -rf .gitea/workflows/ +mv .gitea/workflows-archive/ .gitea/workflows/ +git add .gitea/workflows/ +git commit -m "rollback: restore previous workflows" +git push origin main +``` + +### Scenario 3: Emergency Stop + +Stop containers immediately: + +```bash +cd /srv/docker-compose/basil +docker-compose down +``` + +## Troubleshooting + +### Common Issues + +**Issue: Workflow fails at Docker login** +- Solution: Verify Harbor secrets are configured correctly +- Check: Harbor service is running and accessible + +**Issue: Image push fails** +- Solution: Verify robot account has push permissions +- Check: Harbor disk space is sufficient + +**Issue: Webhook not triggered** +- Solution: Verify webhook URL and secret are correct +- Check: Webhook service is running (`systemctl status basil-webhook`) + +**Issue: Deployment health check fails** +- Solution: Check container logs (`docker logs basil-api`) +- Check: Database migrations completed successfully +- Rollback: Previous containers remain running on health check failure + +**Issue: Tests are flaky** +- Solution: Review test logs in artifacts +- Check: Database service health in workflow +- Consider: Increasing timeouts in playwright.config.ts + +## Local Development + +### Test Workflow Locally + +You can test parts of the workflow locally: + +```bash +# Run all tests +npm run test + +# Run E2E tests +npm run test:e2e + +# Run linting +npm run lint + +# Build all packages +npm run build + +# Build Docker images +docker-compose build + +# Test Harbor login +echo "ErJh8ze6VvZDnviVwc97Jevf6CrdzRBu" | \ + docker login harbor.pkartchner.com \ + -u "robot\$basil+basil-cicd" \ + --password-stdin +``` + +## Maintenance + +### Weekly Tasks + +- Review security scan results in workflow logs +- Check Harbor UI for vulnerability scan results +- Monitor workflow execution times +- Review and clean up old image tags in Harbor + +### Monthly Tasks + +- Rotate Harbor robot account credentials +- Update base Docker images if needed +- Review and optimize caching strategy +- Update dependencies (npm update) + +### Quarterly Tasks + +- Review and update Playwright browser versions +- Audit and remove unused workflow artifacts +- Performance testing and optimization +- Documentation updates + +## Performance Optimization + +Current optimization techniques: + +- **Parallel job execution**: Stage 1 jobs run concurrently +- **NPM caching**: Dependencies cached across runs +- **Docker layer caching**: Reuses unchanged layers +- **Selective deployment**: Only main branch triggers Docker build + +Future optimization opportunities: + +- Build matrix for multiple Node versions +- Split E2E tests into parallel shards +- Implement build artifact reuse +- Add conditional job skipping (skip tests if only docs changed) + +## Support + +For issues or questions: + +- Check workflow logs in Gitea Actions tab +- Review deployment logs: `/srv/docker-compose/basil/deploy.log` +- Check this documentation +- Review archived workflows in `.gitea/workflows-archive/` for comparison + +## Architecture Diagram + +``` +┌─────────────────────────────────────────────────────┐ +│ Developer Push to main │ +└─────────────────┬───────────────────────────────────┘ + │ + v +┌─────────────────────────────────────────────────────┐ +│ Gitea Actions Workflow (main.yml) │ +├─────────────────────────────────────────────────────┤ +│ Stage 1 (Parallel): │ +│ ├─ lint │ +│ ├─ test-api │ +│ ├─ test-web │ +│ ├─ test-shared │ +│ └─ security-scan │ +│ │ +│ Stage 2: build │ +│ │ +│ Stage 3: e2e-tests │ +│ │ +│ Stage 4 (main only): │ +│ ├─ docker-build-and-push → Harbor Registry │ +│ └─ trigger-deployment → Webhook │ +└─────────────────┬───────────────────────────────────┘ + │ + v +┌─────────────────────────────────────────────────────┐ +│ Server (localhost) │ +├─────────────────────────────────────────────────────┤ +│ Webhook Service (port 9000) │ +│ │ │ +│ v │ +│ Deploy Script (/srv/.../scripts/deploy.sh) │ +│ ├─ Login to Harbor │ +│ ├─ Create pre-deployment backup │ +│ ├─ Pull new images from Harbor │ +│ ├─ Update docker-compose.override.yml │ +│ ├─ Restart containers │ +│ ├─ Health checks │ +│ └─ Cleanup old images │ +└─────────────────┬───────────────────────────────────┘ + │ + v +┌─────────────────────────────────────────────────────┐ +│ Basil Application Running │ +│ https://basil.pkartchner.com │ +└─────────────────────────────────────────────────────┘ +``` + +## Version History + +- **v1.0** (2026-01-14): Initial consolidated workflow with Harbor integration + - Merged 5 separate workflows into single main.yml + - Added Harbor registry support + - Implemented webhook-based deployment + - Added comprehensive security scanning + - Optimized with parallel job execution diff --git a/.gitea/workflows/main.yml b/.gitea/workflows/main.yml new file mode 100644 index 0000000..4f5d6ff --- /dev/null +++ b/.gitea/workflows/main.yml @@ -0,0 +1,417 @@ +name: Basil CI/CD Pipeline + +on: + push: + branches: + - main + - develop + tags: + - 'v*' + pull_request: + branches: + - main + - develop + +env: + NODE_VERSION: '20' + HARBOR_REGISTRY: harbor.pkartchner.com + HARBOR_PROJECT: basil + +jobs: + # ============================================================================ + # STAGE 1: PARALLEL QUALITY CHECKS + # ============================================================================ + + lint: + name: Code Linting + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run linter + run: npm run lint + + test-api: + name: API Tests + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: basil + POSTGRES_PASSWORD: basil + POSTGRES_DB: basil_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build shared package + run: cd packages/shared && npm run build + + - name: Generate Prisma Client + run: cd packages/api && npm run prisma:generate + + - name: Run database migrations + run: cd packages/api && npm run prisma:migrate + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil_test?schema=public + + - name: Run API tests + run: cd packages/api && npm run test + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil_test?schema=public + NODE_ENV: test + + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: api-coverage + path: packages/api/coverage/ + retention-days: 14 + + test-web: + name: Web Tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build shared package + run: cd packages/shared && npm run build + + - name: Run Web tests + run: cd packages/web && npm run test + + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: web-coverage + path: packages/web/coverage/ + retention-days: 14 + + test-shared: + name: Shared Package Tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run Shared tests + run: cd packages/shared && npm run test + + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: shared-coverage + path: packages/shared/coverage/ + retention-days: 14 + + security-scan: + name: Security Scanning + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: NPM Audit + run: | + echo "Running npm audit..." + npm audit --audit-level=high || true + cd packages/api && npm audit --audit-level=high || true + cd ../web && npm audit --audit-level=high || true + continue-on-error: true + + - name: Secret Scanning + run: | + echo "Scanning for hardcoded secrets..." + if grep -r -E "(password|secret|api[_-]?key|token)\s*=\s*['\"][^'\"]+['\"]" \ + --include="*.ts" --include="*.js" \ + --exclude-dir=node_modules --exclude-dir=dist .; then + echo "⚠️ Potential hardcoded secrets found!" + exit 1 + fi + echo "✓ No hardcoded secrets detected" + + - name: Check outdated dependencies + run: | + echo "Checking for outdated dependencies..." + npm outdated || true + continue-on-error: true + + # ============================================================================ + # STAGE 2: BUILD VERIFICATION + # ============================================================================ + + build: + name: Build All Packages + runs-on: ubuntu-latest + needs: [lint, test-api, test-web, test-shared, security-scan] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build all packages + run: npm run build + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + packages/api/dist/ + packages/web/dist/ + packages/shared/dist/ + retention-days: 7 + + # ============================================================================ + # STAGE 3: E2E TESTING + # ============================================================================ + + e2e-tests: + name: E2E Tests + runs-on: ubuntu-latest + needs: build + timeout-minutes: 30 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: basil + POSTGRES_PASSWORD: basil + POSTGRES_DB: basil + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Install Playwright browsers + run: npx playwright install --with-deps + + - name: Build application + run: npm run build + + - name: Run database migrations + run: cd packages/api && npm run prisma:migrate + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil?schema=public + + - name: Run E2E tests + run: npm run test:e2e + env: + DATABASE_URL: postgresql://basil:basil@localhost:5432/basil?schema=public + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report/ + retention-days: 14 + + # ============================================================================ + # STAGE 4: DOCKER BUILD & DEPLOYMENT (main branch only) + # ============================================================================ + + docker-build-and-push: + name: Build & Push Docker Images + runs-on: ubuntu-latest + needs: e2e-tests + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + outputs: + image_tag: ${{ steps.meta.outputs.tag }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Harbor + uses: docker/login-action@v3 + with: + registry: ${{ env.HARBOR_REGISTRY }} + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_PASSWORD }} + + - name: Extract metadata + id: meta + run: | + SHA_SHORT=$(echo $GITHUB_SHA | cut -c1-7) + echo "tag=main-${SHA_SHORT}" >> $GITHUB_OUTPUT + echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + + - name: Build and push API image + uses: docker/build-push-action@v5 + with: + context: . + file: ./packages/api/Dockerfile + push: true + tags: | + ${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-api:latest + ${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-api:${{ steps.meta.outputs.tag }} + labels: | + org.opencontainers.image.created=${{ steps.meta.outputs.date }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push Web image + uses: docker/build-push-action@v5 + with: + context: . + file: ./packages/web/Dockerfile + push: true + tags: | + ${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-web:latest + ${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-web:${{ steps.meta.outputs.tag }} + labels: | + org.opencontainers.image.created=${{ steps.meta.outputs.date }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan API image for vulnerabilities + run: | + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + aquasec/trivy:latest image \ + --exit-code 0 \ + --severity HIGH,CRITICAL \ + --no-progress \ + ${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-api:latest || true + + - name: Scan Web image for vulnerabilities + run: | + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + aquasec/trivy:latest image \ + --exit-code 0 \ + --severity HIGH,CRITICAL \ + --no-progress \ + ${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-web:latest || true + + - name: Image build summary + run: | + echo "### Docker Images Built & Pushed 🐳" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**API Image:**" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-api:latest\`" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-api:${{ steps.meta.outputs.tag }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Web Image:**" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-web:latest\`" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/basil-web:${{ steps.meta.outputs.tag }}\`" >> $GITHUB_STEP_SUMMARY + + trigger-deployment: + name: Trigger Deployment + runs-on: ubuntu-latest + needs: docker-build-and-push + if: success() + steps: + - name: Trigger webhook + run: | + curl -X POST ${{ secrets.WEBHOOK_URL }} \ + -H "Content-Type: application/json" \ + -H "X-Webhook-Secret: ${{ secrets.WEBHOOK_SECRET }}" \ + -d '{ + "branch": "main", + "commit": "${{ github.sha }}", + "message": "${{ github.event.head_commit.message }}", + "tag": "${{ needs.docker-build-and-push.outputs.image_tag }}" + }' || echo "Webhook call failed, but continuing..." + + - name: Deployment triggered + run: | + echo "### Deployment Triggered 🚀" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The deployment webhook has been called." >> $GITHUB_STEP_SUMMARY + echo "Check the server logs to verify deployment status:" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "tail -f /srv/docker-compose/basil/deploy.log" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Application URL:** https://basil.pkartchner.com" >> $GITHUB_STEP_SUMMARY diff --git a/docker-compose.yml b/docker-compose.yml index d0c13d2..2fdb6b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: retries: 5 api: + image: ${DOCKER_REGISTRY:-harbor.pkartchner.com}/${DOCKER_USERNAME:-basil}/basil-api:${IMAGE_TAG:-latest} build: context: . dockerfile: packages/api/Dockerfile @@ -57,6 +58,7 @@ services: - traefik web: + image: ${DOCKER_REGISTRY:-harbor.pkartchner.com}/${DOCKER_USERNAME:-basil}/basil-web:${IMAGE_TAG:-latest} build: context: . dockerfile: packages/web/Dockerfile @@ -69,10 +71,18 @@ services: - internal labels: - "traefik.enable=true" + # HTTP router (will redirect to HTTPS) + - "traefik.http.routers.basil-http.rule=Host(`basil.pkartchner.com`)" + - "traefik.http.routers.basil-http.entrypoints=http" + - "traefik.http.routers.basil-http.middlewares=redirect-to-https" + # HTTPS router - "traefik.http.routers.basil.rule=Host(`basil.pkartchner.com`)" - "traefik.http.routers.basil.entrypoints=https" - "traefik.http.routers.basil.tls.certresolver=letsencrypt" + - "traefik.http.routers.basil.middlewares=geoblock@file,secure-headers@file,crowdsec-bouncer@file" + # Service - "traefik.http.services.basil.loadbalancer.server.port=80" + - "traefik.docker.network=traefik" volumes: postgres_data: diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 0af0797..3dface1 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -10,7 +10,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_DIR="$(dirname "$SCRIPT_DIR")" LOG_FILE="$PROJECT_DIR/deploy.log" BACKUP_DIR="$PROJECT_DIR/backups" -DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" +DOCKER_REGISTRY="${DOCKER_REGISTRY:-harbor.pkartchner.com}" DOCKER_USERNAME="${DOCKER_USERNAME}" IMAGE_TAG="${IMAGE_TAG:-latest}" @@ -42,6 +42,25 @@ check_docker() { log "Docker is running" } +# Login to Harbor registry +login_to_harbor() { + log "Logging in to Harbor registry..." + + if [ -z "$HARBOR_PASSWORD" ]; then + error "HARBOR_PASSWORD environment variable not set" + exit 1 + fi + + echo "$HARBOR_PASSWORD" | docker login "$DOCKER_REGISTRY" \ + --username "robot\$${DOCKER_USERNAME}+basil-cicd" \ + --password-stdin || { + error "Failed to login to Harbor" + exit 1 + } + + log "Successfully logged in to Harbor" +} + # Create backup before deployment create_backup() { log "Creating pre-deployment backup..." @@ -182,6 +201,7 @@ main() { log "=========================================" check_docker + login_to_harbor create_backup pull_images update_docker_compose diff --git a/scripts/webhook-receiver.sh b/scripts/webhook-receiver.sh index 06a7aca..8199962 100755 --- a/scripts/webhook-receiver.sh +++ b/scripts/webhook-receiver.sh @@ -74,9 +74,14 @@ create_webhook_config() { "name": "DOCKER_REGISTRY" }, { - "envname": "IMAGE_TAG", + "envname": "HARBOR_PASSWORD", "source": "string", - "name": "IMAGE_TAG" + "name": "HARBOR_PASSWORD" + }, + { + "envname": "IMAGE_TAG", + "source": "payload", + "name": "tag" } ], "trigger-rule-mismatch-http-response-code": 403