Multi-Modal AI Systems

Production Implementation

from openai import OpenAI
import base64
from io import BytesIO
from PIL import Image

class VisionLLM:
    def __init__(self, api_key):
        self.client = OpenAI(api_key=api_key)
    
    def encode_image(self, image_path):
        """Encode image for API submission"""
        with open(image_path, "rb") as image_file:
            return base64.b64encode(image_file.read()).decode('utf-8')
    
    def analyze_image(self, image_path, prompt="Describe this image"):
        """Analyze image with custom prompt"""
        base64_image = self.encode_image(image_path)
        
        response = self.client.chat.completions.create(
            model="gpt-4-vision-preview",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": prompt},
                        {
                            "type": "image_url",
                            "image_url": {
                                "url": f"data:image/jpeg;base64,{base64_image}"
                            }
                        }
                    ]
                }
            ],
            max_tokens=1000
        )
        
        return response.choices[0].message.content
    
    def extract_text_from_image(self, image_path):
        """OCR functionality"""
        prompt = """
        Extract all text from this image. 
        Maintain the original formatting and structure.
        If there are tables, preserve the table structure.
        """
        return self.analyze_image(image_path, prompt)
    
    def analyze_chart(self, image_path):
        """Specialized chart analysis"""
        prompt = """
        Analyze this chart or graph:
        1. What type of visualization is this?
        2. What are the key insights or trends?
        3. What are the axes and units?
        4. Summarize the main message.
        """
        return self.analyze_image(image_path, prompt)

# Usage example
vision_llm = VisionLLM(api_key="your-api-key")

# Document analysis
text_content = vision_llm.extract_text_from_image("document.jpg")

# Chart interpretation
chart_analysis = vision_llm.analyze_chart("sales_chart.png")

# General image understanding
description = vision_llm.analyze_image(
    "product_image.jpg", 
    "Describe this product and suggest marketing copy"
)