test_sources.py

from data_sources import HistoricalDataCollector
from datetime import datetime, timedelta
import json
import time
import os

def pretty_print_event(event):
    """Format and print an event nicely"""
    print("\n" + "="*50)
    print(f"Source: {event['source']}")
    print(f"Date: {event['month']}/{event['day']}/{event['year']}")
    print(f"Description: {event['description']}")
    print("="*50)

def save_events_to_file(events, date, filename):
    """Save events to a text file in a readable format"""
    with open(filename, 'w', encoding='utf-8') as f:
        f.write(f"Historical Events for {date.strftime('%B %d')}\n")
        f.write("="*50 + "\n\n")
        
        # Sort events by year
        sorted_events = sorted(events, key=lambda x: str(x['year']))
        
        for event in sorted_events:
            f.write(f"Year: {event['year']}\n")
            f.write(f"Description: {event['description']}\n")
            f.write(f"Source: {event['source']}\n")
            f.write("-"*50 + "\n\n")
        
        f.write(f"\nTotal Events Found: {len(events)}")

def fetch_events_for_date(collector, date):
    """Fetch events for a specific date"""
    try:
        print(f"\nFetching events for {date.strftime('%B %d')}...")
        events = collector.get_wikipedia_events(date.month, date.day)
        print(f"Found {len(events)} events")
        return events
    except Exception as e:
        print(f"Error fetching events: {str(e)}")
        return []

def batch_collect_events(days=3):
    """Collect events for the specified number of days"""
    # Create output directory if it doesn't exist
    output_dir = "historical_events"
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    # Initialize collector
    collector = HistoricalDataCollector()
    
    # Start from today
    current_date = datetime.now()
    
    print(f"Starting batch collection for {days} days...")
    
    for i in range(days):
        # Calculate the date
        target_date = current_date + timedelta(days=i)
        
        # Create filename for this date
        filename = os.path.join(output_dir, f"events_{target_date.strftime('%Y%m%d')}.txt")
        
        # Fetch and save events
        events = fetch_events_for_date(collector, target_date)
        if events:
            save_events_to_file(events, target_date, filename)
            print(f"Saved {len(events)} events to {filename}")
        
        # Add a small delay to avoid overwhelming the API
        time.sleep(1)
        
        # Progress update
        print(f"Progress: {i+1}/{days} days processed ({((i+1)/days*100):.1f}%)")

if __name__ == "__main__":
    batch_collect_events(100)