#!/usr/bin/env python3
"""
Scraper: Encuentra24.com - Bienes Raices Centroamerica (5 paises)
Paises: Costa Rica, Guatemala, Honduras, Nicaragua, El Salvador
DB: encuentra24_ca
Paginacion: pagina 1 = URL base, pagina N>=2 = URL + '.{N}'
"""

import os
import re
import time
import logging
from datetime import datetime

import requests
from bs4 import BeautifulSoup
import psycopg2
from psycopg2.extras import execute_values

PROXY_URL = 'socks5h://127.0.0.1:1090'
PROXIES   = {'http': PROXY_URL, 'https': PROXY_URL}

DB_HOST = '100.75.240.87'
DB_PORT = 5432
DB_NAME = 'encuentra24_ca'
DB_USER = 'pgadmin'
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'

UA = (
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
    'AppleWebKit/537.36 (KHTML, like Gecko) '
    'Chrome/124.0.0.0 Safari/537.36'
)

DELAY      = 1.2
MAX_PAGES  = int(os.environ.get('MAX_PAGES', '15'))
NEW_STREAK = 3

BASE_URL = 'https://www.encuentra24.com'

COUNTRIES = [
    ('cr', 'costa-rica'),
    ('gt', 'guatemala'),
    ('hn', 'honduras'),
    ('ni', 'nicaragua'),
    ('sv', 'el-salvador'),
]

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
)
log = logging.getLogger(__name__)


def get_session():
    sess = requests.Session()
    sess.proxies.update(PROXIES)
    sess.headers.update({
        'User-Agent':               UA,
        'Accept':                   'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language':          'es-CR,es;q=0.9,en;q=0.8',
        'Connection':               'keep-alive',
        'Cache-Control':            'no-cache',
        'Upgrade-Insecure-Requests':'1',
    })
    return sess


def create_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS listings (
            listing_id    TEXT PRIMARY KEY,
            titulo        TEXT,
            precio        BIGINT,
            moneda        TEXT,
            tipo          TEXT,
            ubicacion     TEXT,
            url           TEXT UNIQUE,
            pais          TEXT,
            superficie_m2 DECIMAL,
            scraped_at    TIMESTAMP DEFAULT NOW()
        )
    """)
    conn.commit()
    cur.close()
    log.info("Tabla listings verificada/creada OK")


def upsert(conn, rows):
    if not rows:
        return 0, 0
    deduped = list({r['listing_id']: r for r in rows}.values())
    cur = conn.cursor()
    ids = [r['listing_id'] for r in deduped]
    cur.execute('SELECT listing_id FROM listings WHERE listing_id = ANY(%s)', (ids,))
    existing = {rec[0] for rec in cur.fetchall()}
    nuevos = sum(1 for r in deduped if r['listing_id'] not in existing)
    actualizados = len(deduped) - nuevos
    now = datetime.utcnow()
    execute_values(cur, """
        INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, pais, superficie_m2, scraped_at)
        VALUES %s
        ON CONFLICT (listing_id) DO UPDATE SET
            titulo        = EXCLUDED.titulo,
            precio        = EXCLUDED.precio,
            moneda        = EXCLUDED.moneda,
            tipo          = EXCLUDED.tipo,
            ubicacion     = EXCLUDED.ubicacion,
            url           = EXCLUDED.url,
            pais          = EXCLUDED.pais,
            superficie_m2 = EXCLUDED.superficie_m2,
            scraped_at    = EXCLUDED.scraped_at
    """, [
        (r['listing_id'], r.get('titulo'), r.get('precio'), r.get('moneda'),
         r.get('tipo'), r.get('ubicacion'), r.get('url'), r.get('pais'),
         r.get('superficie_m2'), now)
        for r in deduped
    ])
    conn.commit()
    cur.close()
    return nuevos, actualizados


def page_url(cc, country_slug, page):
    base = f'{BASE_URL}/{country_slug}-es/bienes-raices-venta-de-propiedades'
    if page == 1:
        return base
    return f'{base}.{page}'


def _detect_tipo(text):
    tl = text.lower()
    if any(k in tl for k in ('alquiler', 'arriendo', 'arrendar', 'renta', 'rento')):
        return 'arriendo'
    if any(k in tl for k in ('venta', 'vender', 'en venta', 'se vende')):
        return 'venta'
    return 'venta'


def _clean_precio(text):
    m = re.search(r'([\d\.,]+)', text.replace('\xa0', '').replace(' ', ''))
    if not m:
        return None
    try:
        return int(m.group(1).replace(',', '').replace('.', '').rstrip('0') or '0')
    except ValueError:
        return None


def _extract_listing_id(href):
    path = href.split('?')[0].split('#')[0].rstrip('/')
    segments = [s for s in path.split('/') if s]
    if not segments:
        return None
    last = segments[-1]
    m = re.search(r'(\d{5,})', last)
    if m:
        return m.group(1)
    if len(last) > 4:
        return last
    return None


def scrape_page(sess, url, cc):
    backoffs = [3, 8, 20]
    resp = None
    for attempt, wait in enumerate(backoffs, start=1):
        try:
            resp = sess.get(url, timeout=35)
        except requests.RequestException as exc:
            log.warning("Red intento %d/3 [%s]: %s", attempt, url, exc)
            if attempt < 3:
                time.sleep(wait)
                continue
            return []

        if resp.status_code in (403, 429):
            log.warning("HTTP %d en %s — espera 45s", resp.status_code, url)
            time.sleep(45)
            if attempt >= 3:
                return []
            continue

        if resp.status_code == 404:
            log.info("404 — fin de paginacion en %s", url)
            return None

        if resp.status_code != 200:
            log.warning("HTTP %d inesperado en %s", resp.status_code, url)
            return []

        break

    if resp is None or not resp.text.strip():
        return []

    soup = BeautifulSoup(resp.text, 'lxml')

    card_selectors = [
        'div.listing-card',
        'article.listing',
        'div[class*="listing-card"]',
        'div[class*="result-item"]',
        'li[class*="listing-item"]',
        'div[class*="ad-card"]',
        'article[class*="property"]',
        'div[class*="property-card"]',
    ]
    cards = []
    for sel in card_selectors:
        found = soup.select(sel)
        if found:
            log.debug("Cards con '%s': %d", sel, len(found))
            cards = found
            break

    if not cards:
        links = soup.select('a[href*="/bienes-raices"]')
        if links:
            cards = links

    if not cards:
        log.info("Sin cards en %s", url)
        return []

    results = []
    seen = set()

    for card in cards:
        try:
            if card.name == 'a':
                link_tag = card
            else:
                link_tag = (
                    card.select_one('a[href*="/bienes-raices"]')
                    or card.select_one('a[href*="/inmuebles"]')
                    or card.select_one('a')
                )

            href = (link_tag.get('href', '') if link_tag else '').strip()
            if not href:
                continue

            full_url = href if href.startswith('http') else BASE_URL + href

            listing_id = _extract_listing_id(href)
            if not listing_id or listing_id in seen:
                continue
            seen.add(listing_id)

            card_text = card.get_text(separator=' ', strip=True)

            titulo = None
            for sel in ('h2', 'h3', 'h1', '[class*="title"]', '[class*="titulo"]', '[class*="heading"]'):
                el = card.select_one(sel)
                if el:
                    t = el.get_text(strip=True)
                    if t:
                        titulo = t[:255]
                        break
            if not titulo and link_tag:
                titulo = link_tag.get_text(strip=True)[:255]
            titulo = titulo or 'Sin titulo'

            precio = None
            moneda = 'USD'
            for sel in ('[class*="price"]', '[class*="precio"]', '[class*="valor"]', '.amount', '.value'):
                el = card.select_one(sel)
                if el:
                    t = el.get_text(strip=True)
                    if 'USD' in t or '$' in t:
                        moneda = 'USD'
                    precio = _clean_precio(t)
                    if precio and precio > 100:
                        break

            tipo = _detect_tipo(href + ' ' + card_text)

            ubicacion = None
            for sel in ('[class*="location"]', '[class*="ubicacion"]', '[class*="city"]',
                        '[class*="ciudad"]', '[class*="address"]', '[class*="region"]'):
                el = card.select_one(sel)
                if el:
                    t = el.get_text(strip=True)
                    if t:
                        ubicacion = t[:200]
                        break

            superficie_m2 = None
            m2 = re.search(r'(\d+(?:[,\.]\d+)?)\s*m[²2]', card_text, re.IGNORECASE)
            if m2:
                try:
                    superficie_m2 = float(m2.group(1).replace(',', '.'))
                except ValueError:
                    pass

            results.append({
                'listing_id':   listing_id,
                'titulo':       titulo,
                'precio':       precio,
                'moneda':       moneda,
                'tipo':         tipo,
                'ubicacion':    ubicacion,
                'url':          full_url,
                'pais':         cc,
                'superficie_m2': superficie_m2,
            })
        except Exception as exc:
            log.debug("Error parseando card: %s", exc)

    log.info("  -> %d listings en %s", len(results), url)
    return results


def main():
    total_nuevos = 0
    total_actualizados = 0

    log.info("=== Scraper Encuentra24 CA Bienes Raices iniciado ===")

    try:
        conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME,
                                user=DB_USER, password=DB_PASS, connect_timeout=15)
        log.info("Conectado a PostgreSQL: %s@%s/%s", DB_USER, DB_HOST, DB_NAME)
    except Exception as exc:
        raise RuntimeError(f"No se pudo conectar a PostgreSQL: {exc}") from exc

    try:
        create_table(conn)
        sess = get_session()

        for cc, country_slug in COUNTRIES:
            log.info("== Pais: %s (%s) ==", country_slug.upper(), cc)
            streak_sin_nuevos = 0

            for page in range(1, MAX_PAGES + 1):
                url = page_url(cc, country_slug, page)
                log.info("[%s] Pagina %d/%d -> %s", cc, page, MAX_PAGES, url)

                rows = scrape_page(sess, url, cc)

                if rows is None:
                    log.info("[%s] Pagina %d devolvio 404 — fin para este pais", cc, page)
                    break

                if not rows:
                    streak_sin_nuevos += 1
                    if streak_sin_nuevos >= NEW_STREAK:
                        log.info("[%s] %d paginas sin resultados — siguiente pais", cc, NEW_STREAK)
                        break
                    time.sleep(DELAY)
                    continue

                nuevos, actualizados = upsert(conn, rows)
                total_nuevos += nuevos
                total_actualizados += actualizados

                log.info("[%s] Pag %d: %d nuevos, %d actualizados (acum %d/%d)",
                         cc, page, nuevos, actualizados, total_nuevos, total_actualizados)

                if nuevos == 0:
                    streak_sin_nuevos += 1
                    if streak_sin_nuevos >= NEW_STREAK:
                        log.info("[%s] %d paginas sin nuevos — siguiente pais", cc, NEW_STREAK)
                        break
                else:
                    streak_sin_nuevos = 0

                time.sleep(DELAY)

            time.sleep(DELAY * 2)

    finally:
        try:
            conn.close()
            log.info("Conexion PostgreSQL cerrada")
        except Exception:
            pass

    print(f"Finalizado: {total_nuevos} nuevos, {total_actualizados} actualizados")
    log.info("=== Scraper finalizado. Nuevos: %d | Actualizados: %d ===",
             total_nuevos, total_actualizados)


if __name__ == '__main__':
    main()