Merge pull request #2 from crackerky/feature/ai-ocr-vision

Feature/ai ocr vision

Author: crackerky

receipt-scanner-app/receipt-scanner-backend/AI_OCR_README.md

@@ -0,0 +1,82 @@
+# AI-OCR Vision API Feature
+
+This branch adds support for OpenAI's Vision API (GPT-4o) for advanced OCR capabilities.
+
+## Features
+
+- **AI-OCR with GPT-4o Vision**: Direct image analysis without traditional OCR
+- **Automatic fallback**: If Vision API fails, falls back to Tesseract OCR
+- **Better accuracy**: Handles handwritten text, poor quality images, and complex layouts
+- **Multi-language support**: Works with Japanese and English receipts
+
+## Configuration
+
+Add these environment variables to enable Vision API:
+
+```bash
+# Enable/disable Vision API (default: true)
+USE_VISION_API=true
+
+# Vision API model (options: gpt-4o, gpt-4o-mini)
+VISION_API_MODEL=gpt-4o
+
+# Your existing OpenAI API key
+OPENAI_API_KEY=sk-your-api-key
+```
+
+## How it works
+
+1. **Image Upload**: User uploads a receipt image
+2. **Vision API Processing**: 
+   - Image is encoded to base64
+   - Sent to GPT-4o Vision API with structured prompt
+   - AI directly extracts receipt information
+3. **Fallback**: If Vision API fails, uses traditional OCR pipeline:
+   - Tesseract OCR → Text extraction → GPT-3.5 analysis → Regex patterns
+
+## Cost Considerations
+
+- **GPT-4o**: $5.00 per 1M input tokens, $20.00 per 1M output tokens
+- **GPT-4o-mini**: $0.15 per 1M input tokens, $0.60 per 1M output tokens
+- Average receipt image uses 500-2000 tokens (high detail)
+
+## Testing
+
+1. Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+2. Set environment variables:
+```bash
+export USE_VISION_API=true
+export VISION_API_MODEL=gpt-4o
+export OPENAI_API_KEY=your-key
+```
+
+3. Run the application:
+```bash
+uvicorn app.main:app --reload
+```
+
+4. Test with various receipt images:
+   - Clear printed receipts
+   - Handwritten receipts
+   - Blurry or low-quality images
+   - Complex layouts with tables
+
+## Performance
+
+Vision API provides:
+- Better accuracy for difficult images
+- Direct extraction without OCR errors
+- Structured data output
+- Support for various image formats
+
+## Deployment
+
+For Railway deployment, add the environment variables in the Railway dashboard:
+- `USE_VISION_API=true`
+- `VISION_API_MODEL=gpt-4o` (or `gpt-4o-mini` for cost savings)
+
+The feature is backward compatible - if Vision API is disabled or fails, it automatically falls back to the traditional OCR pipeline.

receipt-scanner-app/receipt-scanner-backend/app/config.py

@@ -17,6 +17,10 @@ def __init__(self):
         # Required environment variables
         self.openai_api_key = self._get_required_env("OPENAI_API_KEY")
 
+        # Vision API settings
+        self.use_vision_api = os.getenv("USE_VISION_API", "true").lower() == "true"
+        self.vision_api_model = os.getenv("VISION_API_MODEL", "gpt-4o")  # or "gpt-4o-mini"
+        
         # Optional environment variables with defaults
         self.database_url = os.getenv("DATABASE_URL", "postgresql://localhost:5432/receipt_scanner")
         self.secret_key = os.getenv("SECRET_KEY", "dev-secret-key-change-in-production")
@@ -111,6 +115,11 @@ def is_development(self) -> bool:
     def openai_available(self) -> bool:
         """Check if OpenAI API is available."""
         return bool(self.openai_api_key)
+    
+    @property
+    def vision_api_available(self) -> bool:
+        """Check if Vision API is available and enabled."""
+        return bool(self.openai_api_key) and self.use_vision_api
 
 # Global settings instance
 settings = Settings()

receipt-scanner-app/receipt-scanner-backend/app/receipt_processor.py

@@ -3,9 +3,11 @@
 import logging
 import platform
 import subprocess
+import base64
 from datetime import datetime
 from PIL import Image, ImageFilter, ImageEnhance
 from typing import Dict, Any, Optional, Tuple
+from openai import OpenAI
 
 # HEIFのインポートを条件付きに
 try:
@@ -86,22 +88,37 @@ def setup_tesseract():
 
 
 class ReceiptProcessor:
-    """統合されたレシート処理クラス - AI-OCRの協調処理"""
+"""Secure receipt processing with AI-OCR Vision and fallback OCR functionality."""
+
 
     def __init__(self):
         """Initialize the receipt processor with secure configuration."""
         self.openai_available = settings.openai_available
+        self.vision_api_available = settings.vision_api_available
         self.tesseract_available = tesseract_available
         self.cv2_available = CV2_AVAILABLE
         self.heif_available = HEIF_AVAILABLE
 
-        # 処理モード設定
-        self.processing_mode = self._determine_processing_mode()
+
 
         # OCRプロセッサーの初期化
         self.ocr_processor = OCRProcessor(cv2_available=self.cv2_available)
 
-        # AIプロセッサーの初期化
+        # AIプロセッサーの初期化        # 処理モード設定
+        self.processing_mode = self._determine_processing_mode()
+
+        # Initialize OpenAI client for Vision API
+        if self.vision_api_available:
+            try:
+                self.openai_client = OpenAI(api_key=settings.openai_api_key)
+                logger.info("OpenAI Vision API initialized successfully")
+            except Exception as e:
+                logger.error(f"Failed to initialize OpenAI Vision API: {e}")
+                self.vision_api_available = False
+        
+        if not self.tesseract_available:
+            logger.error("Tesseract OCR is not available. Please install Tesseract OCR.")
+
         self.ai_processor = None
         if self.openai_available:
             try:
@@ -157,6 +174,139 @@ def _check_tesseract_languages(self):
         except Exception as e:
             logger.error(f"Failed to get Tesseract languages: {e}")
 
+    def _create_prompt_template(self) -> ChatPromptTemplate:
+        """Create a secure prompt template for OpenAI."""
+        return ChatPromptTemplate.from_template(
+            """
+            以下は日本のレシートのテキストです。このテキストから以下の情報を抽出してください：
+            1. 日付 (YYYY-MM-DD形式、見つからない場合はnull)
+            2. 店名または会社名
+            3. 合計金額 (数値のみ、見つからない場合はnull)
+            4. 税抜き価格 (あれば、数値のみ)
+            5. 税込み価格 (あれば、数値のみ)
+            
+            JSONフォーマットで回答してください：
+            {{
+                "date": "YYYY-MM-DD" or null,
+                "store_name": "店名",
+                "total_amount": 数値 or null,
+                "tax_excluded_amount": 数値 or null,
+                "tax_included_amount": 数値 or null
+            }}
+            
+            レシートテキスト:
+            {text}
+            """
+        )
+    
+    def _create_vision_prompt(self) -> str:
+        """Create a prompt for Vision API OCR."""
+        return """
+        この画像は日本のレシートです。以下の情報を正確に抽出してください：
+        
+        1. 日付 (YYYY-MM-DD形式、見つからない場合はnull)
+        2. 店名または会社名
+        3. 合計金額 (数値のみ、見つからない場合はnull)
+        4. 税抜き価格 (あれば、数値のみ)
+        5. 税込み価格 (あれば、数値のみ)
+        
+        以下のJSONフォーマットで回答してください：
+        {
+            "date": "YYYY-MM-DD" or null,
+            "store_name": "店名",
+            "total_amount": 数値 or null,
+            "tax_excluded_amount": 数値 or null,
+            "tax_included_amount": 数値 or null
+        }
+        
+        注意事項：
+        - 日付は必ずYYYY-MM-DD形式に変換してください
+        - 金額は数値のみ（カンマや円記号は除く）
+        - 税抜き/税込み価格が明記されていない場合はnull
+        - 不明な情報はnullとしてください
+        """
+    
+    def _extract_with_vision_api(self, image_bytes: bytes) -> Dict[str, Any]:
+        """Extract receipt information using GPT-4o Vision API."""
+        try:
+            # Convert image to base64
+            base64_image = base64.b64encode(image_bytes).decode('utf-8')
+            
+            logger.info("Sending image to Vision API for OCR...")
+            
+            # Call Vision API
+            response = self.openai_client.chat.completions.create(
+                model="gpt-4o",  # or "gpt-4o-mini" for cost savings
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": self._create_vision_prompt()
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{base64_image}",
+                                    "detail": "high"  # Use "high" for better OCR accuracy
+                                }
+                            }
+                        ]
+                    }
+                ],
+                max_tokens=1000,
+                response_format={"type": "json_object"}  # Ensure JSON response
+            )
+            
+            # Parse the response
+            result_text = response.choices[0].message.content
+            logger.info(f"Vision API response: {result_text}")
+            
+            # Parse JSON response
+            data = json.loads(result_text)
+            
+            # Validate and process the data
+            processed_data = {
+                "date": data.get("date"),
+                "store_name": data.get("store_name"),
+                "total_amount": float(data.get("total_amount")) if data.get("total_amount") else None,
+                "tax_excluded_amount": float(data.get("tax_excluded_amount")) if data.get("tax_excluded_amount") else None,
+                "tax_included_amount": float(data.get("tax_included_amount")) if data.get("tax_included_amount") else None,
+                "expense_category": None
+            }
+            
+            # Validate required fields
+            if not processed_data.get("store_name"):
+                return {
+                    "success": False,
+                    "message": "Vision APIで店名を抽出できませんでした。",
+                    "data": None
+                }
+            
+            logger.info("Successfully extracted receipt data with Vision API")
+            
+            return {
+                "success": True,
+                "message": "AI-OCR (Vision API)でレシート情報を抽出しました。",
+                "data": processed_data
+            }
+            
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to parse JSON from Vision API response: {e}")
+            return {
+                "success": False,
+                "message": "Vision APIのレスポンスが無効でした。",
+                "data": None
+            }
+        except Exception as e:
+            logger.error(f"Vision API extraction error: {e}")
+            return {
+                "success": False,
+                "message": f"Vision API処理中にエラーが発生しました: {str(e)}",
+                "data": None
+            }
+
     def _convert_heic_to_jpeg(self, image_bytes: bytes) -> bytes:
         """Convert HEIC/HEIF image to JPEG format."""
         if not self.heif_available:
@@ -321,6 +471,29 @@ def process_image(self, image_bytes: bytes, processing_mode: Optional[str] = Non
             # 処理モードの決定
             if not processing_mode:
                 processing_mode = "auto"
+
+            # Try Vision API first if available
+            if self.vision_api_available:
+                logger.info("Attempting AI-OCR with Vision API...")
+                result = self._extract_with_vision_api(image_bytes)
+                if result["success"]:
+                    # 日付が抽出できなかった場合、現在の日時を使用
+                    if result["data"] and not result["data"].get("date"):
+                        result["data"]["date"] = datetime.now().strftime("%Y-%m-%d")
+                        result["message"] += " 日付は現在の日付で補完しました。"
+                    return result
+                else:
+                    logger.warning("Vision API failed, falling back to traditional OCR")
+
+            # Fall back to traditional OCR processing
+            # Tesseractが利用できない場合のエラー
+            if not self.tesseract_available:
+                return {
+                    "success": False,
+                    "message": "OCRエンジンが利用できません。Tesseract OCRをインストールしてください。",
+                    "data": None
+                }
+
 
             # Vision APIモードの場合
             if processing_mode == "vision" and self.openai_client:

receipt-scanner-app/receipt-scanner-backend/requirements.txt

@@ -19,3 +19,4 @@ alembic>=1.13.0
 python-jose[cryptography]>=3.3.0
 passlib[bcrypt]>=1.7.4
 email-validator>=2.0.0
+