feat: upgrade recipe scraper to Python recipe-scrapers library (v2025.10.1)

## Changes

### Recipe Scraper Enhancement
- Replaced custom Cheerio-based scraper with Python recipe-scrapers library
- Now supports 541+ recipe websites (same as Mealie)
- Added Python 3 and recipe-scrapers to Docker container
- Created Python wrapper script (packages/api/scripts/scrape_recipe.py)
- Updated scraper service to call Python script via subprocess

### Bug Fixes
- Fixed servings field parsing (string to integer conversion)
- Added safe extraction with graceful error handling
- Removed obsolete test file that was breaking builds
- Fixed Prisma binary targets for Alpine Linux

### Infrastructure
- Added Traefik configuration for HTTPS with Let's Encrypt
- Updated CORS settings for production domain
- Configured for basil.pkartchner.com

### Version Management
- Implemented CalVer versioning (Year.Month.Increment)
- Added VERSION file (2025.10.1)
- Created version.sh script for managing releases
- Tagged and pushed Docker images to Harbor registry

### Database
- Updated Prisma schema with correct binary targets
- Applied initial migration for all tables

### Build Improvements
- Excluded test files from TypeScript compilation
- Removed non-existent dependencies
- Optimized Docker build process

## Testing
- Successfully tested with Food Network, Bon Appetit, Food.com
- Verified full import and save workflow
- Confirmed ingredients and instructions display correctly

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

VERSION

@@ -0,0 +1 @@
+2025.10.1

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   postgres:
     image: postgres:16-alpine
@@ -11,8 +9,8 @@ services:
       POSTGRES_DB: basil
     volumes:
       - postgres_data:/var/lib/postgresql/data
-    ports:
-      - "5432:5432"
+    networks:
+      - internal
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U basil"]
       interval: 10s
@@ -34,11 +32,12 @@ services:
       DATABASE_URL: postgresql://basil:basil@postgres:5432/basil?schema=public
       STORAGE_TYPE: local
       LOCAL_STORAGE_PATH: /app/uploads
-      CORS_ORIGIN: http://localhost:5173
+      CORS_ORIGIN: https://basil.pkartchner.com
     volumes:
       - uploads_data:/app/uploads
-    ports:
-      - "3001:3001"
+    networks:
+      - internal
+      - traefik
 
   web:
     build:
@@ -48,9 +47,22 @@ services:
     restart: unless-stopped
     depends_on:
       - api
-    ports:
-      - "5173:80"
+    networks:
+      - traefik
+      - internal
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.basil.rule=Host(`basil.pkartchner.com`)"
+      - "traefik.http.routers.basil.entrypoints=https"
+      - "traefik.http.routers.basil.tls.certresolver=letsencrypt"
+      - "traefik.http.services.basil.loadbalancer.server.port=80"
 
 volumes:
   postgres_data:
   uploads_data:
+
+networks:
+  traefik:
+    external: true
+  internal:
+    driver: bridge

packages/api/Dockerfile

@@ -23,6 +23,12 @@ RUN npm run build
 # Production stage
 FROM node:20-alpine
 
+# Install OpenSSL for Prisma and Python for recipe-scrapers
+RUN apk add --no-cache openssl python3 py3-pip
+
+# Install recipe-scrapers Python package
+RUN pip3 install --break-system-packages recipe-scrapers
+
 WORKDIR /app
 
 # Copy built files and dependencies
@@ -32,13 +38,13 @@ COPY --from=builder /app/packages/shared/dist ./packages/shared/dist
 COPY --from=builder /app/packages/api/package.json ./packages/api/
 COPY --from=builder /app/packages/api/dist ./packages/api/dist
 COPY --from=builder /app/packages/api/prisma ./packages/api/prisma
-COPY --from=builder /app/packages/api/node_modules/.prisma ./packages/api/node_modules/.prisma
+COPY --from=builder /app/packages/api/scripts ./packages/api/scripts
 COPY --from=builder /app/node_modules ./node_modules
 
 WORKDIR /app/packages/api
 
-# Create uploads directory
-RUN mkdir -p /app/uploads
+# Create uploads directory and make Python script executable
+RUN mkdir -p /app/uploads && chmod +x scripts/scrape_recipe.py
 
 EXPOSE 3001
 

packages/api/package.json

@@ -26,8 +26,7 @@
     "dotenv": "^16.3.1",
     "multer": "^1.4.5-lts.1",
     "axios": "^1.6.5",
-    "cheerio": "^1.0.0-rc.12",
-    "recipe-scraper": "^3.0.0"
+    "cheerio": "^1.0.0-rc.12"
   },
   "devDependencies": {
     "@types/express": "^4.17.21",

packages/api/prisma/schema.prisma

@@ -1,5 +1,6 @@
 generator client {
-  provider = "prisma-client-js"
+  provider      = "prisma-client-js"
+  binaryTargets = ["native", "linux-musl-openssl-3.0.x"]
 }
 
 datasource db {

packages/api/scripts/scrape_recipe.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+Recipe scraper script using the recipe-scrapers library.
+This script is called by the Node.js API to scrape recipes from URLs.
+"""
+
+import sys
+import json
+from recipe_scrapers import scrape_me
+
+def safe_extract(scraper, method_name, default=None):
+    """Safely extract data from scraper, returning default if method fails."""
+    try:
+        if hasattr(scraper, method_name):
+            result = getattr(scraper, method_name)()
+            return result if result else default
+        return default
+    except Exception:
+        return default
+
+def parse_servings(servings_str):
+    """Parse servings string into an integer. Returns None if can't parse."""
+    if not servings_str:
+        return None
+    try:
+        # Extract first number from string like "8 servings" or "Serves 8"
+        import re
+        match = re.search(r'\d+', str(servings_str))
+        if match:
+            return int(match.group())
+        return None
+    except Exception:
+        return None
+
+def scrape_recipe(url):
+    """Scrape a recipe from the given URL and return JSON data."""
+    try:
+        scraper = scrape_me(url)
+
+        # Extract recipe data with safe extraction
+        recipe_data = {
+            "success": True,
+            "recipe": {
+                "title": scraper.title(),
+                "description": safe_extract(scraper, 'description'),
+                "totalTime": safe_extract(scraper, 'total_time'),
+                "prepTime": None,  # recipe-scrapers doesn't separate prep time
+                "cookTime": None,  # recipe-scrapers doesn't separate cook time
+                "servings": parse_servings(safe_extract(scraper, 'yields')),
+                "imageUrl": safe_extract(scraper, 'image'),
+                "author": safe_extract(scraper, 'author'),
+                "cuisine": safe_extract(scraper, 'cuisine'),
+                "category": safe_extract(scraper, 'category'),
+                "rating": None,  # Not commonly available
+                "ingredients": [
+                    {
+                        "name": ingredient,
+                        "order": i
+                    }
+                    for i, ingredient in enumerate(scraper.ingredients())
+                ],
+                "instructions": [
+                    {
+                        "step": i + 1,
+                        "text": instruction
+                    }
+                    for i, instruction in enumerate(scraper.instructions_list())
+                ]
+            }
+        }
+
+        return recipe_data
+
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+            "recipe": {}
+        }
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(json.dumps({
+            "success": False,
+            "error": "No URL provided",
+            "recipe": {}
+        }))
+        sys.exit(1)
+
+    url = sys.argv[1]
+    result = scrape_recipe(url)
+    print(json.dumps(result))

packages/api/src/services/scraper.service.test.ts

@@ -1,138 +0,0 @@
-import { describe, it, expect, beforeEach, vi } from 'vitest';
-import axios from 'axios';
-import { ScraperService } from './scraper.service';
-
-vi.mock('axios');
-
-describe('ScraperService', () => {
-  let scraperService: ScraperService;
-
-  beforeEach(() => {
-    vi.clearAllMocks();
-    scraperService = new ScraperService();
-  });
-
-  describe('scrapeRecipe', () => {
-    it('should extract recipe from schema.org JSON-LD', async () => {
-      const mockHtml = `
-        <!DOCTYPE html>
-        <html>
-        <head>
-          <script type="application/ld+json">
-          {
-            "@type": "Recipe",
-            "name": "Test Recipe",
-            "description": "A delicious test recipe",
-            "prepTime": "PT15M",
-            "cookTime": "PT30M",
-            "totalTime": "PT45M",
-            "recipeYield": "4",
-            "recipeIngredient": ["2 cups flour", "1 cup sugar"],
-            "recipeInstructions": [
-              {"text": "Mix ingredients"},
-              {"text": "Bake for 30 minutes"}
-            ],
-            "author": {"name": "Chef Test"},
-            "recipeCuisine": "Italian",
-            "recipeCategory": "Dessert"
-          }
-          </script>
-        </head>
-        <body></body>
-        </html>
-      `;
-
-      vi.mocked(axios.get).mockResolvedValue({ data: mockHtml });
-
-      const result = await scraperService.scrapeRecipe('https://example.com/recipe');
-
-      expect(result.success).toBe(true);
-      expect(result.recipe?.title).toBe('Test Recipe');
-      expect(result.recipe?.description).toBe('A delicious test recipe');
-      expect(result.recipe?.prepTime).toBe(15);
-      expect(result.recipe?.cookTime).toBe(30);
-      expect(result.recipe?.totalTime).toBe(45);
-      expect(result.recipe?.servings).toBe(4);
-      expect(result.recipe?.ingredients).toHaveLength(2);
-      expect(result.recipe?.instructions).toHaveLength(2);
-      expect(result.recipe?.sourceUrl).toBe('https://example.com/recipe');
-    });
-
-    it('should fallback to manual parsing when no schema.org found', async () => {
-      const mockHtml = `
-        <!DOCTYPE html>
-        <html>
-        <head>
-          <title>Test Recipe Page</title>
-          <meta name="description" content="Test description">
-          <meta property="og:image" content="https://example.com/image.jpg">
-        </head>
-        <body>
-          <h1>Fallback Recipe</h1>
-        </body>
-        </html>
-      `;
-
-      vi.mocked(axios.get).mockResolvedValue({ data: mockHtml });
-
-      const result = await scraperService.scrapeRecipe('https://example.com/recipe');
-
-      expect(result.success).toBe(true);
-      expect(result.recipe?.title).toBe('Fallback Recipe');
-      expect(result.recipe?.description).toBe('Test description');
-      expect(result.recipe?.imageUrl).toBe('https://example.com/image.jpg');
-    });
-
-    it('should handle errors gracefully', async () => {
-      vi.mocked(axios.get).mockRejectedValue(new Error('Network error'));
-
-      const result = await scraperService.scrapeRecipe('https://example.com/recipe');
-
-      expect(result.success).toBe(false);
-      expect(result.error).toContain('Network error');
-    });
-
-    it('should parse ISO 8601 duration correctly', async () => {
-      const mockHtml = `
-        <!DOCTYPE html>
-        <html>
-        <head>
-          <script type="application/ld+json">
-          {
-            "@type": "Recipe",
-            "name": "Duration Test",
-            "prepTime": "PT1H30M",
-            "cookTime": "PT45M"
-          }
-          </script>
-        </head>
-        </html>
-      `;
-
-      vi.mocked(axios.get).mockResolvedValue({ data: mockHtml });
-
-      const result = await scraperService.scrapeRecipe('https://example.com/recipe');
-
-      expect(result.recipe?.prepTime).toBe(90); // 1 hour 30 minutes
-      expect(result.recipe?.cookTime).toBe(45); // 45 minutes
-    });
-  });
-
-  describe('downloadImage', () => {
-    it('should download image and return buffer', async () => {
-      const mockImageData = Buffer.from('fake-image-data');
-      vi.mocked(axios.get).mockResolvedValue({ data: mockImageData });
-
-      const result = await scraperService.downloadImage('https://example.com/image.jpg');
-
-      expect(axios.get).toHaveBeenCalledWith(
-        'https://example.com/image.jpg',
-        expect.objectContaining({
-          responseType: 'arraybuffer',
-          timeout: 10000,
-        })
-      );
-      expect(result).toBeInstanceOf(Buffer);
-    });
-  });
-});

packages/api/src/services/scraper.service.ts

@@ -1,43 +1,32 @@
-import axios from 'axios';
-import * as cheerio from 'cheerio';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import path from 'path';
 import { Recipe, RecipeImportResponse } from '@basil/shared';
 
+const execAsync = promisify(exec);
+
 export class ScraperService {
   async scrapeRecipe(url: string): Promise<RecipeImportResponse> {
     try {
-      const response = await axios.get(url, {
-        headers: {
-          'User-Agent': 'Mozilla/5.0 (compatible; BasilBot/1.0)',
-        },
-        timeout: 10000,
+      // Call Python recipe-scrapers script (path relative to working directory /app/packages/api)
+      const scriptPath = 'scripts/scrape_recipe.py';
+      const { stdout, stderr } = await execAsync(`python3 ${scriptPath} "${url}"`, {
+        timeout: 30000, // 30 second timeout
       });
 
-      const html = response.data;
-      const $ = cheerio.load(html);
-
-      // Try to find JSON-LD schema.org Recipe markup
-      const recipeData = this.extractSchemaOrgRecipe($);
-
-      if (recipeData) {
-        return {
-          success: true,
-          recipe: {
-            ...recipeData,
-            sourceUrl: url,
-          },
-        };
+      if (stderr && !stdout) {
+        throw new Error(`Python script error: ${stderr}`);
       }
 
-      // Fallback to manual parsing if no schema found
-      const fallbackData = this.extractRecipeFallback($);
+      // Parse the JSON output from the Python script
+      const result: RecipeImportResponse = JSON.parse(stdout);
 
-      return {
-        success: true,
-        recipe: {
-          ...fallbackData,
-          sourceUrl: url,
-        },
-      };
+      // Add source URL if not present
+      if (result.recipe) {
+        result.recipe.sourceUrl = url;
+      }
+
+      return result;
     } catch (error) {
       console.error('Error scraping recipe:', error);
       return {
@@ -47,120 +36,4 @@ export class ScraperService {
       };
     }
   }
-
-  private extractSchemaOrgRecipe($: cheerio.CheerioAPI): Partial<Recipe> | null {
-    const scripts = $('script[type="application/ld+json"]');
-
-    for (let i = 0; i < scripts.length; i++) {
-      try {
-        const content = $(scripts[i]).html();
-        if (!content) continue;
-
-        const json = JSON.parse(content);
-        const recipeData = Array.isArray(json)
-          ? json.find((item) => item['@type'] === 'Recipe')
-          : json['@type'] === 'Recipe'
-          ? json
-          : null;
-
-        if (recipeData) {
-          return {
-            title: recipeData.name,
-            description: recipeData.description,
-            prepTime: this.parseDuration(recipeData.prepTime),
-            cookTime: this.parseDuration(recipeData.cookTime),
-            totalTime: this.parseDuration(recipeData.totalTime),
-            servings: parseInt(recipeData.recipeYield) || undefined,
-            imageUrl: this.extractImageUrl(recipeData.image),
-            author: recipeData.author?.name || recipeData.author,
-            cuisine: recipeData.recipeCuisine,
-            category: recipeData.recipeCategory,
-            rating: recipeData.aggregateRating?.ratingValue,
-            ingredients: this.parseIngredients(recipeData.recipeIngredient),
-            instructions: this.parseInstructions(recipeData.recipeInstructions),
-          };
-        }
-      } catch (error) {
-        continue;
-      }
-    }
-
-    return null;
-  }
-
-  private extractRecipeFallback($: cheerio.CheerioAPI): Partial<Recipe> {
-    // Basic fallback extraction
-    const title = $('h1').first().text().trim() || $('title').text().trim();
-    const description = $('meta[name="description"]').attr('content');
-    const imageUrl = $('meta[property="og:image"]').attr('content');
-
-    return {
-      title,
-      description,
-      imageUrl,
-      ingredients: [],
-      instructions: [],
-    };
-  }
-
-  private parseDuration(duration?: string): number | undefined {
-    if (!duration) return undefined;
-
-    // Parse ISO 8601 duration format (PT30M, PT1H30M, etc.)
-    const matches = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?/);
-    if (matches) {
-      const hours = parseInt(matches[1]) || 0;
-      const minutes = parseInt(matches[2]) || 0;
-      return hours * 60 + minutes;
-    }
-
-    return undefined;
-  }
-
-  private extractImageUrl(image: any): string | undefined {
-    if (!image) return undefined;
-    if (typeof image === 'string') return image;
-    if (Array.isArray(image)) return image[0];
-    if (image.url) return image.url;
-    return undefined;
-  }
-
-  private parseIngredients(ingredients?: string[]): any[] {
-    if (!ingredients || !Array.isArray(ingredients)) return [];
-
-    return ingredients.map((ingredient, index) => ({
-      name: ingredient,
-      order: index,
-    }));
-  }
-
-  private parseInstructions(instructions?: any): any[] {
-    if (!instructions) return [];
-
-    if (typeof instructions === 'string') {
-      return [{ step: 1, text: instructions }];
-    }
-
-    if (Array.isArray(instructions)) {
-      return instructions.map((instruction, index) => {
-        if (typeof instruction === 'string') {
-          return { step: index + 1, text: instruction };
-        }
-        if (instruction.text) {
-          return { step: index + 1, text: instruction.text };
-        }
-        return { step: index + 1, text: JSON.stringify(instruction) };
-      });
-    }
-
-    return [];
-  }
-
-  async downloadImage(imageUrl: string): Promise<Buffer> {
-    const response = await axios.get(imageUrl, {
-      responseType: 'arraybuffer',
-      timeout: 10000,
-    });
-    return Buffer.from(response.data);
-  }
 }

packages/web/tsconfig.json

@@ -21,5 +21,6 @@
     "noFallthroughCasesInSwitch": true
   },
   "include": ["src"],
+  "exclude": ["src/**/*.test.ts", "src/**/*.test.tsx", "**/*.test.ts", "**/*.test.tsx"],
   "references": [{ "path": "./tsconfig.node.json" }]
 }

scripts/version.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+# Version management script for Basil
+
+set -e
+
+VERSION_FILE="VERSION"
+
+# Get current version
+get_version() {
+    if [ -f "$VERSION_FILE" ]; then
+        cat "$VERSION_FILE" | tr -d '\n'
+    else
+        echo "2025.10.1"
+    fi
+}
+
+# Increment version (bump the increment part)
+bump_version() {
+    current=$(get_version)
+    year=$(echo "$current" | cut -d. -f1)
+    month=$(echo "$current" | cut -d. -f2)
+    increment=$(echo "$current" | cut -d. -f3)
+
+    current_year=$(date +%Y)
+    current_month=$(date +%-m)
+
+    # If year or month changed, reset increment to 1
+    if [ "$year" != "$current_year" ] || [ "$month" != "$current_month" ]; then
+        new_version="${current_year}.${current_month}.1"
+    else
+        # Otherwise increment
+        increment=$((increment + 1))
+        new_version="${year}.${month}.${increment}"
+    fi
+
+    echo "$new_version" > "$VERSION_FILE"
+    echo "$new_version"
+}
+
+# Tag and push Docker images
+tag_and_push() {
+    version=$(get_version)
+
+    echo "Tagging and pushing version: $version"
+
+    # Tag API
+    docker tag basil-api "harbor.pkartchner.com/basil/api:${version}"
+    docker tag basil-api "harbor.pkartchner.com/basil/api:latest"
+
+    # Tag Web
+    docker tag basil-web "harbor.pkartchner.com/basil/web:${version}"
+    docker tag basil-web "harbor.pkartchner.com/basil/web:latest"
+
+    # Push all tags
+    echo "Pushing harbor.pkartchner.com/basil/api:${version}"
+    docker push "harbor.pkartchner.com/basil/api:${version}"
+    echo "Pushing harbor.pkartchner.com/basil/api:latest"
+    docker push "harbor.pkartchner.com/basil/api:latest"
+
+    echo "Pushing harbor.pkartchner.com/basil/web:${version}"
+    docker push "harbor.pkartchner.com/basil/web:${version}"
+    echo "Pushing harbor.pkartchner.com/basil/web:latest"
+    docker push "harbor.pkartchner.com/basil/web:latest"
+
+    echo "Successfully pushed version $version to Harbor"
+}
+
+# Main command handler
+case "${1:-}" in
+    get)
+        get_version
+        ;;
+    bump)
+        bump_version
+        ;;
+    push)
+        tag_and_push
+        ;;
+    *)
+        echo "Usage: $0 {get|bump|push}"
+        echo "  get  - Show current version"
+        echo "  bump - Increment version number"
+        echo "  push - Tag and push current version to Harbor"
+        exit 1
+        ;;
+esac