Inhalt von Webseiten auslesen

import asyncio
import aiohttp
import random
import time
import re
import json
import logging
from datetime import datetime
from typing import List, Dict, Optional, Tuple
from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from fake_useragent import UserAgent
import pandas as pd
from PIL import Image
import pytesseract
from sklearn.feature_extraction.text import TfidfVectorizer
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

# Logging-Konfiguration
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler("scraper.log"),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

# Globale Einstellungen
USER_AGENT = UserAgent()
PROXIES = [
    "http://138.197.209.105:3128",
    "http://167.71.230.124:8080",
    "http://142.93.198.186:3128",
    # Fügen Sie hier weitere Proxies hinzu
]
RATE_LIMIT_DELAY = random.uniform(1.5, 3.5)
MAX_RETRIES = 3
VALID_DOMAINS = ["example.com", "newsite.org", "infoportal.net"]
OUTPUT_FILE = f"scraped_data_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"

# NLTK-Vorbereitung
try:
    nltk.data.find('tokenizers/punkt')
    nltk.data.find('corpora/stopwords')
except LookupError:
    nltk.download('punkt')
    nltk.download('stopwords')

STOPWORDS = set(stopwords.words('german') + stopwords.words('english'))

class AdvancedScraper:
    def __init__(self):
        self.visited_urls = set()
        self.session = None
        self.selenium_driver = self._init_selenium_driver()
        self.tfidf_vectorizer = TfidfVectorizer(max_features=1000)

    def _init_selenium_driver(self) -> webdriver.Chrome:
        """Initialisiert den Selenium-Webdriver mit Headless-Modus"""
        chrome_options = Options()
        chrome_options.add_argument("--headless")
        chrome_options.add_argument("--disable-gpu")
        chrome_options.add_argument("--no-sandbox")
        chrome_options.add_argument("--disable-dev-shm-usage")
        chrome_options.add_argument(f"user-agent={USER_AGENT.random}")
        
        # Proxy für Selenium einrichten
        proxy = random.choice(PROXIES)
        chrome_options.add_argument(f'--proxy-server={proxy}')
        
        driver = webdriver.Chrome(
            service=Service(ChromeDriverManager().install()),
            options=chrome_options
        )
        driver.set_page_load_timeout(30)
        return driver

    async def _get_random_headers(self) -> Dict[str, str]:
        """Generiert zufällige HTTP-Header"""
        return {
            "User-Agent": USER_AGENT.random,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": random.choice(["de-DE,de;q=0.9", "en-US,en;q=0.9", "fr-FR,fr;q=0.9"]),
            "Referer": f"https://{random.choice(VALID_DOMAINS)}/",
            "Connection": "keep-alive",
        }

    async def _get_random_proxy(self) -> Optional[str]:
        """Gibt einen zufälligen Proxy zurück"""
        return random.choice(PROXIES) if PROXIES else None

    async def _solve_captcha(self, image_url: str) -> str:
        """Löst einfache CAPTCHAs mit OCR"""
        try:
            async with self.session.get(image_url) as response:
                if response.status == 200:
                    image_data = await response.read()
                    with open("temp_captcha.png", "wb") as f:
                        f.write(image_data)
                    
                    # OCR-Verarbeitung
                    img = Image.open("temp_captcha.png")
                    text = pytesseract.image_to_string(img)
                    # Bereinigung des Ergebnisses
                    return re.sub(r'[^A-Za-z0-9]', '', text).strip()
        except Exception as e:
            logger.error(f"Fehler bei der CAPTCHA-Verarbeitung: {str(e)}")
        return ""

    async def _fetch_with_aiohttp(self, url: str) -> Tuple[Optional[str], int]:
        """Ruft eine URL asynchron mit aiohttp ab"""
        for attempt in range(MAX_RETRIES):
            try:
                headers = await self._get_random_headers()
                proxy = await self._get_random_proxy()
                
                async with self.session.get(
                    url,
                    headers=headers,
                    proxy=proxy,
                    timeout=aiohttp.ClientTimeout(total=15)
                ) as response:
                    content = await response.text()
                    
                    # CAPTCHA-Erkennung
                    if "captcha" in content.lower() or response.status == 403:
                        logger.warning(f"CAPTCHA erkannt auf {url}")
                        # Suche nach CAPTCHA-Bild
                        soup = BeautifulSoup(content, 'html.parser')
                        captcha_img = soup.find('img', src=re.compile(r'captcha'))
                        if captcha_img:
                            captcha_url = urljoin(url, captcha_img['src'])
                            captcha_text = await self._solve_captcha(captcha_url)
                            if captcha_text:
                                # Versuche, CAPTCHA zu senden (seitenabhängig)
                                logger.info(f"CAPTCHA-Lösung: {captcha_text}")
                                # Hier würde die Logik zum Senden der CAPTCHA-Lösung hin
                    
                    return content, response.status
                    
            except Exception as e:
                logger.warning(f"Fehler beim Abrufen von {url} (Versuch {attempt+1}): {str(e)}")
                if attempt < MAX_RETRIES - 1:
                    await asyncio.sleep(RATE_LIMIT_DELAY * (attempt + 1))
        
        return None, 0

    def _fetch_with_selenium(self, url: str) -> Optional[str]:
        """Ruft eine URL mit Selenium ab (für dynamischen Inhalt)"""
        try:
            self.selenium_driver.get(url)
            
            # Warte auf dynamischen Inhalt
            WebDriverWait(self.selenium_driver, 10).until(
                EC.presence_of_element_located((By.TAG_NAME, "body"))
            )
            
            # Scrollen, um weitere Inhalte zu laden
            last_height = self.selenium_driver.execute_script("return document.body.scrollHeight")
            while True:
                self.selenium_driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                time.sleep(1)
                new_height = self.selenium_driver.execute_script("return document.body.scrollHeight")
                if new_height == last_height:
                    break
                last_height = new_height
            
            return self.selenium_driver.page_source
        except Exception as e:
            logger.error(f"Selenium-Fehler bei {url}: {str(e)}")
            return None

    def _extract_information(self, html: str, url: str) -> Dict[str, any]:
        """Extrahiert strukturierte Informationen aus HTML"""
        soup = BeautifulSoup(html, 'html.parser')
        
        # Basis-Informationen
        title = soup.title.string.strip() if soup.title else ""
        meta_description = ""
        meta_tags = soup.find_all('meta')
        for tag in meta_tags:
            if tag.get('name') == 'description':
                meta_description = tag.get('content', '').strip()
                break
        
        # Text-Extraktion
        paragraphs = [p.get_text(strip=True) for p in soup.find_all('p') if p.get_text(strip=True)]
        full_text = ' '.join(paragraphs)
        
        # Links extrahieren
        links = []
        for a_tag in soup.find_all('a', href=True):
            href = a_tag['href']
            full_url = urljoin(url, href)
            if any(domain in full_url for domain in VALID_DOMAINS) and full_url not in self.visited_urls:
                links.append(full_url)
        
        # Bilder extrahieren
        images = []
        for img_tag in soup.find_all('img', src=True):
            img_src = img_tag['src']
            full_img_url = urljoin(url, img_src)
            images.append({
                'url': full_img_url,
                'alt': img_tag.get('alt', '').strip()
            })
        
        # Datum erkennen
        date_patterns = [
            r'\b\d{2}\.\d{2}\.\d{4}\b',  # TT.MM.JJJJ
            r'\b\d{4}-\d{2}-\d{2}\b',    # JJJJ-TT-MM
            r'\b(Januar|Februar|März|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember) \d{4}\b'
        ]
        dates = []
        for pattern in date_patterns:
            dates.extend(re.findall(pattern, full_text))
        
        # Schlüsselwörter extrahieren
        keywords = self._extract_keywords(full_text)
        
        return {
            'url': url,
            'title': title,
            'meta_description': meta_description,
            'text_length': len(full_text),
            'paragraph_count': len(paragraphs),
            'extracted_dates': list(set(dates)),
            'keywords': keywords,
            'links': links,
            'images': images,
            'scraped_at': datetime.now().isoformat()
        }

    def _extract_keywords(self, text: str) -> List[str]:
        """Extrahiert Schlüsselwörter aus Text mit TF-IDF"""
        if not text:
            return []
            
        # Textbereinigung
        text = re.sub(r'[^\w\s]', '', text.lower())
        tokens = word_tokenize(text)
        tokens = [t for t in tokens if t not in STOPWORDS and len(t) > 2]
        cleaned_text = ' '.join(tokens)
        
        # TF-IDF-Berechnung
        try:
            tfidf_matrix = self.tfidf_vectorizer.fit_transform([cleaned_text])
            feature_names = self.tfidf_vectorizer.get_feature_names_out()
            tfidf_scores = tfidf_matrix.toarray()[0]
            
            # Top-10-Schlüsselwörter
            keyword_indices = tfidf_scores.argsort()[-10:][::-1]
            return [feature_names[i] for i in keyword_indices if tfidf_scores[i] > 0]
        except Exception as e:
            logger.error(f"Fehler bei der Schlüsselwortextraktion: {str(e)}")
            return []

    def _validate_data(self, data: Dict[str, any]) -> bool:
        """Validiert extrahierte Daten"""
        if not data.get('url'):
            return False
        if len(data.get('title', '')) < 5:
            logger.warning(f"Ungültiger Titel für {data['url']}")
            return False
        if data.get('text_length', 0) < 100:
            logger.warning(f"Zu wenig Textinhalt für {data['url']}")
            return False
        return True

    async def _process_url(self, url: str, use_selenium: bool = False) -> Optional[Dict[str, any]]:
        """Verarbeitet eine einzelne URL"""
        if url in self.visited_urls:
            return None
            
        self.visited_urls.add(url)
        logger.info(f"Verarbeite URL: {url}")
        
        # Rate-Limiting
        await asyncio.sleep(RATE_LIMIT_DELAY)
        
        # Inhalt abrufen
        if use_selenium:
            html = self._fetch_with_selenium(url)
            status_code = 200 if html else 0
        else:
            html, status_code = await self._fetch_with_aiohttp(url)
        
        if not html or status_code not in [200, 403]:  # 403 wird weiterverarbeitet wegen CAPTCHA
            logger.error(f"Konnte {url} nicht abrufen (Status: {status_code})")
            return None
        
        # Informationen extrahieren
        extracted_data = self._extract_information(html, url)
        
        # Daten validieren
        if not self._validate_data(extracted_data):
            logger.warning(f"Ungültige Daten für {url}")
            return None
            
        return extracted_data

    async def crawl(self, start_urls: List[str], max_depth: int = 2) -> List[Dict[str, any]]:
        """Führt einen rekursiven Crawl durch"""
        all_data = []
        queue = [(url, 0) for url in start_urls]
        
        async with aiohttp.ClientSession() as session:
            self.session = session
            
            while queue:
                url, depth = queue.pop(0)
                
                if depth > max_depth:
                    continue
                    
                # Versuche zuerst mit aiohttp, falls fehlgeschlagen mit Selenium
                data = await self._process_url(url)
                if not data:
                    logger.info(f"Versuche mit Selenium: {url}")
                    data = await self._process_url(url, use_selenium=True)
                
                if data:
                    all_data.append(data)
                    logger.info(f"Erfolgreich verarbeitet: {url} (Tiefe: {depth})")
                    
                    # Füge neue URLs zur Warteschlange hinzu
                    for link in data['links']:
                        if link not in self.visited_urls:
                            queue.append((link, depth + 1))
        
        return all_data

    def save_results(self, data: List[Dict[str, any]]) -> None:
        """Speichert die Ergebnisse in verschiedenen Formaten"""
        if not data:
            logger.warning("Keine Daten zum Speichern")
            return
            
        # DataFrame für CSV-Export vorbereiten
        simplified_data = []
        for item in data:
            simplified = {
                'url': item['url'],
                'title': item['title'],
                'scraped_at': item['scraped_at'],
                'text_length': item['text_length'],
                'paragraph_count': item['paragraph_count'],
                'keywords': ', '.join(item['keywords']),
                'dates': ', '.join(item['extracted_dates'])
            }
            simplified_data.append(simplified)
        
        df = pd.DataFrame(simplified_data)
        df.to_csv(OUTPUT_FILE, index=False, encoding='utf-8-sig')
        logger.info(f"Daten gespeichert in {OUTPUT_FILE}")
        
        # Vollständige Daten in JSON speichern
        json_file = OUTPUT_FILE.replace('.csv', '.json')
        with open(json_file, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)

    def close(self):
        """Räumt Ressourcen auf"""
        if self.selenium_driver:
            self.selenium_driver.quit()
        logger.info("Scraper beendet und Ressourcen aufgeräumt")

async def main():
    # Initialisierung
    scraper = AdvancedScraper()
    
    try:
        # Start-URLs
        start_urls = [
            f"https://{domain}/" for domain in VALID_DOMAINS
        ]
        
        # Crawl starten
        logger.info("Starte Web-Scraping...")
        results = await scraper.crawl(start_urls, max_depth=2)
        
        # Ergebnisse speichern
        scraper.save_results(results)
        
        logger.info(f"Erfolgreich {len(results)} Seiten verarbeitet")
        
    except Exception as e:
        logger.error(f"Hauptfehler: {str(e)}", exc_info=True)
    finally:
        # Aufräumen
        scraper.close()

if __name__ == "__main__":
    # Asynchronen Event-Loop starten
    asyncio.run(main())