Role of Machine Learning and Computation in
MEMENTOby Vaibhav Jain

"""
Step 2: Model Architecture Definition

This module defines the ML model architectures used for classifying mementos.
These models will be trained in Step 3 using user mementos from Firebase.
"""

import os
import json
import pickle
import numpy as np
import pandas as pd
import logging
from typing import Dict, List, Tuple, Optional
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import classification_report, accuracy_score, f1_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MultiLabelBinarizer

class MementoModelTrainer:
    def __init__(self, 
                categories_path: str, 
                tags_path: str,
                durations_path: str,
                output_dir: str = "."):
        """Initialize the model trainer"""
        self.categories_path = categories_path
        self.tags_path = tags_path
        self.durations_path = durations_path
        self.output_dir = output_dir
        
        # Create output directory
        os.makedirs(output_dir, exist_ok=True)
        
        # Load metadata
        self.categories, self.category_ids = self._load_categories()
        self.tags, self.tag_ids = self._load_tags()
        self.durations, self.duration_ids = self._load_durations()
        
        # Initialize models
        self.vectorizer = None
        self.category_model = None
        self.tags_model = None
        self.duration_model = None

    def train_models(self, 
                     data_path: str, 
                     test_size: float = 0.2, 
                     random_state: int = 42,
                     use_grid_search: bool = False) -> Dict:
        """Train category, tag, and duration models"""
        # Load and prepare data
        X, y_category, y_tags, y_duration = self.load_data(data_path)
        
        # Create text vectorizer
        self.vectorizer = TfidfVectorizer(
            max_features=5000,
            min_df=2,
            max_df=0.8,
            ngram_range=(1, 2),
            stop_words='english'
        )
        
        # Train models
        self._train_category_model(X, y_category)
        self._train_tags_model(X, y_tags)
        self._train_duration_model(X, y_duration)
        
        # Evaluate and save models
        metrics = self._evaluate_models(X, y_category, y_tags, y_duration)
        self._save_models()
        self.save_model_info(metrics)
        
        return metrics

"""
Step 3: Model Training

This module trains the ML models defined in Step 2 using user mementos from Step 1.
The trained models will be used to classify scraped data in Step 5.
"""

import os
import json
import logging
import pandas as pd
import numpy as np
from typing import Dict, List, Tuple, Optional
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, f1_score
from sklearn.preprocessing import MultiLabelBinarizer
import pickle
from datetime import datetime

class MementoModelTrainer:
    def __init__(self, 
                 input_dir: str = "ml_pipeline/output/step1_data_processing/processed_data",
                 output_dir: str = "ml_pipeline/output/step3_model_training"):
        """Initialize the trainer"""
        self.input_dir = input_dir
        self.output_dir = output_dir
        self.models_dir = os.path.join(output_dir, "models")
        self.metrics_dir = os.path.join(output_dir, "metrics")
        self.reports_dir = os.path.join(output_dir, "reports")
        
        # Create output directories
        os.makedirs(self.models_dir, exist_ok=True)
        os.makedirs(self.metrics_dir, exist_ok=True)
        os.makedirs(self.reports_dir, exist_ok=True)
        
        # Define hyperparameter grids
        self.param_grid = {
            'C': [0.1, 1, 10],
            'max_iter': [1000],
            'class_weight': ['balanced'],
            'solver': ['liblinear', 'saga']
        }

    def train_models(self):
        """Train all models"""
        # Load and prepare data
        df = self.load_training_data()
        X, y_dict = self.prepare_training_data(df)
        
        # Train models
        category_metrics = self.train_category_model(X, y_dict['category'])
        tags_metrics = self.train_tags_model(X, y_dict['tags'])
        duration_metrics = self.train_duration_model(X, y_dict['duration'])
        
        # Save vectorizer
        self.save_vectorizer()
        
        # Generate and save reports
        metrics = {
            'category': category_metrics,
            'tags': tags_metrics,
            'duration': duration_metrics
        }
        self.generate_model_report(metrics)

"""
Step 4: Scrape Data

This module scrapes data from Secret NYC to create a testing dataset.
"""

import requests
from bs4 import BeautifulSoup
import json
import os
import re
import time
from datetime import datetime
from langdetect import detect
from typing import Dict, List, Optional
import logging
from requests.adapters import HTTPAdapter
from urllib3.util import Retry

class SecretNYCScraper:
    def __init__(self, 
                 base_url: str = "https://secretnyc.co/things-to-do/",
                 output_dir: str = "ml_pipeline/output/step4_scraped_data"):
        """Initialize scraper"""
        self.base_url = base_url
        self.output_dir = output_dir
        self.raw_data_dir = os.path.join(output_dir, "raw_data")
        os.makedirs(self.raw_data_dir, exist_ok=True)
        self.default_user = "Secret NYC"

    def create_session(self) -> requests.Session:
        """Create session with retry strategy"""
        session = requests.Session()
        retry_strategy = Retry(
            total=3,
            backoff_factor=1,
            status_forcelist=[429, 500, 502, 503, 504],
        )
        adapter = HTTPAdapter(max_retries=retry_strategy)
        session.mount("http://", adapter)
        session.mount("https://", adapter)
        session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
        return session

    def scrape_article(self, article_url: str) -> Optional[Dict]:
        """Scrape a single article"""
        session = self.create_session()
        try:
            response = session.get(article_url, timeout=10)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, "html.parser")
            
            # Extract article data
            title = soup.find("h1").get_text(strip=True) if soup.find("h1") else "Untitled"
            paragraphs = soup.select("section.article__body p")
            desc = "\n".join(p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True))
            
            # Process and clean data
            cleaned = self.clean_description(desc)
            location = self.extract_fallback_location(desc, title)
            duration = self.extract_duration(desc)
            
            # Create memento object
            memento = {
                "userId": self.default_user,
                "location": self.geocode_location(location) if location else {},
                "media": self.extract_media(soup),
                "name": title,
                "description": cleaned,
                "category": "Other",
                "timestamp": self.parse_date(self.extract_date(soup)),
                "tags": ["Other"],
                "link": article_url,
                "mementoType": "public",
                "duration": duration
            }
            
            return memento
            
        except Exception as e:
            logging.error(f"Error scraping {article_url}: {e}")
            return None

"""
Step 5: Process Scraped Data

This module processes scraped data from Step 4 through trained models from Step 3.
It classifies each scraped memento into categories, tags, and durations.
"""

import json
import os
import pandas as pd
import numpy as np
from typing import List, Dict, Optional, Tuple
import logging
import pickle
from sklearn.preprocessing import MultiLabelBinarizer
from datetime import datetime

class ScrapedDataProcessor:
    def __init__(self, 
                 model_dir: str = "ml_pipeline/output/step3_model_training/models",
                 output_dir: str = "ml_pipeline/output/step5_processed_data"):
        """Initialize the processor"""
        self.model_dir = model_dir
        self.output_dir = output_dir
        self.processed_data_dir = os.path.join(output_dir, "processed_data")
        self.reports_dir = os.path.join(output_dir, "reports")
        self.validation_dir = os.path.join(output_dir, "validation")
        
        # Confidence thresholds
        self.confidence_thresholds = {
            "category": 0.4,
            "tags": 0.3,
            "duration": 0.4
        }
        
        # Create output directories
        os.makedirs(self.processed_data_dir, exist_ok=True)
        os.makedirs(self.reports_dir, exist_ok=True)
        os.makedirs(self.validation_dir, exist_ok=True)
        
        # Load models
        self._load_models()

    def process_memento(self, memento: Dict) -> Dict:
        """Process a single memento"""
        # Extract text for ML
        text = self._extract_text_for_ml(memento)
        
        # Vectorize text
        text_vec = self.vectorizer.transform([text])
        
        # Get predictions
        category_pred, category_conf = self._predict_category(text_vec)
        tags_pred, tags_conf = self._predict_tags(text_vec)
        duration_pred, duration_conf = self._predict_duration(text_vec)
        
        # Validate predictions
        validation = self._validate_predictions(
            category_pred, category_conf,
            tags_pred, tags_conf,
            duration_pred, duration_conf
        )
        
        # Update memento with predictions
        memento.update({
            "category": category_pred,
            "category_confidence": category_conf,
            "tags": tags_pred,
            "tags_confidence": tags_conf,
            "duration": duration_pred,
            "duration_confidence": duration_conf,
            "validation": validation
        })
        
        return memento

"""
Step 6: ML Model Predictor

This module provides a production-ready predictor class that uses trained ML models
to classify new mementos with categories, tags, and durations.
"""

import os
import json
import pickle
import logging
import numpy as np
from typing import List, Dict, Optional, Union

class MementoPredictor:
    def __init__(self, 
                 models_dir: Optional[str] = None,
                 categories_path: Optional[str] = None,
                 tags_path: Optional[str] = None,
                 durations_path: Optional[str] = None,
                 threshold: float = 0.2):
        """Initialize the predictor with trained models"""
        # Set up paths
        self.script_dir = os.path.dirname(os.path.abspath(__file__))
        self.root_dir = os.path.dirname(self.script_dir)
        
        # Set models_dir if not provided
        if models_dir is None:
            models_dir = os.path.join(self.script_dir, "models")
        self.models_dir = models_dir
        
        # Set paths for category, tag, and duration definitions
        self.categories_path = categories_path or os.path.join(self.root_dir, "memento_categories_combined.json")
        self.tags_path = tags_path or os.path.join(self.root_dir, "memento_tags_combined.json")
        self.durations_path = durations_path or os.path.join(self.root_dir, "memento_durations.json")
        
        # Set threshold
        self.threshold = threshold
        
        # Initialize models and data
        self.vectorizer = None
        self.category_model = None
        self.tags_model = None
        self.duration_model = None
        self.categories = None
        self.tags = None
        self.durations = None
        
        # Load models and data
        self._load_categories_and_tags()
        self._load_models()

    def classify_memento(self, 
                        description: str, 
                        context: Optional[Dict] = None) -> Dict[str, Union[str, List[str]]]:
        """Classify a memento with categories, tags, and duration"""
        # Get predictions
        category = self.predict_category(description)
        tags = self.predict_tags(description)
        duration = self.predict_duration(description)
        
        # Return complete classification
        return {
            "category": category,
            "tags": tags,
            "duration": duration
        }

"""
Step 7: ML Pipeline Integration

This module provides tools to integrate the trained ML models with production systems,
particularly focusing on updating existing scrapers to use ML-based classification.
"""

import os
import sys
import argparse
import logging
import shutil
from datetime import datetime
from typing import Tuple

# Import the predictor
from step6_predictor import MementoPredictor

def find_scrapers(search_dir: str = None) -> list:
    """Find all potential scraper files in the given directory"""
    if search_dir is None:
        search_dir = os.path.dirname(script_dir)
    
    potential_scrapers = []
    
    # Search patterns that indicate a scraper file
    patterns = [
        "scrape_all_pages",
        "scrape_",
        "_scraper",
        "crawler",
    ]
    
    # Walk through the directory
    for root, _, files in os.walk(search_dir):
        for file in files:
            if file.endswith(".py"):
                file_path = os.path.join(root, file)
                
                # Check if filename matches any pattern
                if any(pattern in file.lower() for pattern in patterns):
                    potential_scrapers.append(file_path)
                else:
                    # Check file contents for key functions
                    try:
                        with open(file_path, "r", encoding="utf-8") as f:
                            content = f.read()
                            if "def assign_category" in content and "def assign_tags" in content:
                                potential_scrapers.append(file_path)
                    except Exception:
                        pass
    
    return potential_scrapers

def update_scraper(scraper_path: str, dryrun: bool = False) -> bool:
    """Update a scraper file to use ML-based classification"""
    # First analyze the scraper
    can_update, info = analyze_scraper(scraper_path)
    
    if not can_update:
        logging.error(f"Cannot update {scraper_path}: {info.get('error', 'Missing required functions')}")
        return False
    
    if info.get("already_using_ml", False):
        logging.warning(f"Scraper {scraper_path} is already using ML predictor")
        return False
    
    if dryrun:
        logging.info("DRY RUN: Would update scraper with ML predictor")
        return True
    
    # Create a backup
    backup_path = f"{scraper_path}.bak.{datetime.now().strftime('%Y%m%d_%H%M%S')}"
    try:
        shutil.copy2(scraper_path, backup_path)
        logging.info(f"Created backup at {backup_path}")
    except Exception as e:
        logging.error(f"Failed to create backup: {e}")
        return False
    
    # Update the scraper
    try:
        predictor = MementoPredictor()
        success = predictor.update_scraper(scraper_path)
        
        if success:
            logging.info(f"Successfully updated {scraper_path}")
            return True
        else:
            logging.error(f"Failed to update {scraper_path}")
            return False
    except Exception as e:
        logging.error(f"Error updating scraper: {e}")
        
        # Try to restore from backup
        try:
            shutil.copy2(backup_path, scraper_path)
            logging.info(f"Restored from backup {backup_path}")
        except Exception as restore_error:
            logging.error(f"Failed to restore from backup: {restore_error}")
        
        return False