#!/usr/bin/env python3
"""
Scraper Yapo.cl – Bienes Raices Chile (Schibsted)
Iteracion por subcategorias: ~30 tiles SSR por subcategoria (~150 unicos/dia)
DB: yapo_cl
"""

import re
import time
import logging
from datetime import datetime

import requests
from bs4 import BeautifulSoup
import psycopg2
from psycopg2.extras import execute_values

PROXY   = 'socks5h://127.0.0.1:1090'
PROXIES = {'http': PROXY, 'https': PROXY}

DB_HOST = '100.75.240.87'
DB_PORT = 5432
DB_NAME = 'yapo_cl'
DB_USER = 'pgadmin'
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'

UA = (
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
    'AppleWebKit/537.36 (KHTML, like Gecko) '
    'Chrome/124.0.0.0 Safari/537.36'
)

DELAY    = 1.5
BASE_URL = 'https://www.yapo.cl'

SUBCATEGORIES = [
    '/bienes-raices-venta-de-propiedades',
    '/bienes-raices-venta-de-propiedades-casas',
    '/bienes-raices-venta-de-propiedades-apartamentos',
    '/bienes-raices-alquiler',
    '/bienes-raices-alquiler-casas',
    '/bienes-raices-alquiler-apartamentos',
    '/bienes-raices-proyectos-nuevos',
]

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
)
log = logging.getLogger(__name__)


def get_session():
    sess = requests.Session()
    sess.proxies.update(PROXIES)
    sess.headers.update({
        'User-Agent':      UA,
        'Accept':          'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'es-CL,es;q=0.9,en;q=0.8',
        'Connection':      'keep-alive',
        'Cache-Control':   'no-cache',
    })
    return sess


def create_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS listings (
            listing_id    TEXT PRIMARY KEY,
            titulo        TEXT,
            precio        DECIMAL,
            moneda        TEXT DEFAULT 'UF',
            tipo          TEXT,
            ubicacion     TEXT,
            url           TEXT UNIQUE,
            superficie_m2 DECIMAL,
            ambientes     INT,
            scraped_at    TIMESTAMP DEFAULT NOW()
        )
    """)
    conn.commit()
    cur.close()
    log.info("Tabla listings verificada/creada OK")


def upsert(conn, rows):
    if not rows:
        return 0, 0
    deduped = list({r['listing_id']: r for r in rows}.values())
    cur = conn.cursor()
    ids = [r['listing_id'] for r in deduped]
    cur.execute('SELECT listing_id FROM listings WHERE listing_id = ANY(%s)', (ids,))
    existing = {rec[0] for rec in cur.fetchall()}
    nuevos = sum(1 for r in deduped if r['listing_id'] not in existing)
    actualizados = len(deduped) - nuevos
    now = datetime.utcnow()
    execute_values(cur, """
        INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, superficie_m2, ambientes, scraped_at)
        VALUES %s
        ON CONFLICT (listing_id) DO UPDATE SET
            titulo        = EXCLUDED.titulo,
            precio        = EXCLUDED.precio,
            moneda        = EXCLUDED.moneda,
            tipo          = EXCLUDED.tipo,
            ubicacion     = EXCLUDED.ubicacion,
            url           = EXCLUDED.url,
            superficie_m2 = EXCLUDED.superficie_m2,
            ambientes     = EXCLUDED.ambientes,
            scraped_at    = EXCLUDED.scraped_at
    """, [
        (r['listing_id'], r.get('titulo'), r.get('precio'), r.get('moneda', 'UF'),
         r.get('tipo'), r.get('ubicacion'), r.get('url'),
         r.get('superficie_m2'), r.get('ambientes'), now)
        for r in deduped
    ])
    conn.commit()
    cur.close()
    return nuevos, actualizados


def _detect_tipo(href):
    if 'alquiler' in href or 'arriendo' in href or 'renta' in href:
        return 'arriendo'
    if 'venta' in href or 'vender' in href:
        return 'venta'
    return None


def scrape_subcategory(sess, path):
    url = BASE_URL + path
    for attempt in range(3):
        try:
            r = sess.get(url, timeout=30)
            if r.status_code in (403, 429):
                log.warning("HTTP %d en %s — esperando 30s", r.status_code, url)
                time.sleep(30)
                continue
            if r.status_code == 404:
                log.info("404 en %s — ignorando", url)
                return []
            r.raise_for_status()
            break
        except requests.RequestException as exc:
            log.warning("Intento %d/3 fallido: %s", attempt + 1, exc)
            if attempt < 2:
                time.sleep([3, 8][attempt])
            else:
                return []

    soup = BeautifulSoup(r.text, 'lxml')
    tiles = soup.select('.d3-ad-tile')
    if not tiles:
        log.info("Sin tiles en %s", path)
        return []

    results = []
    seen = set()

    for tile in tiles:
        try:
            fav = tile.select_one('[data-adid]')
            if not fav:
                continue
            listing_id = fav.get('data-adid', '').strip()
            if not listing_id or listing_id in seen:
                continue
            seen.add(listing_id)

            precio_uf = None
            raw_price = fav.get('data-price', '')
            if raw_price:
                try:
                    precio_uf = float(raw_price)
                except ValueError:
                    pass

            links = tile.select('a[href]')
            href = ''
            for lnk in links:
                h = lnk.get('href', '')
                if listing_id in h and '/bienes-raices' in h:
                    href = h
                    break
            if not href:
                for lnk in links:
                    h = lnk.get('href', '')
                    if h and h != '#' and '/bienes-raices' in h:
                        href = h
                        break

            full_url = BASE_URL + href if href.startswith('/') else href

            tipo = _detect_tipo(href)

            title_el = tile.select_one('h2, h3, [class*=title]')
            titulo = title_el.get_text(strip=True)[:255] if title_el else None

            desc_el = tile.select_one('.d3-ad-tile__description, [class*=description]')
            if not titulo and desc_el:
                titulo = desc_el.get_text(strip=True)[:255]

            card_text = tile.get_text(separator=' ', strip=True)

            ubicacion = None
            loc_el = tile.select_one('[class*=location], [class*=ubicacion], [class*=address]')
            if loc_el:
                ubicacion = loc_el.get_text(strip=True)[:200]
            else:
                m_loc = re.search(r'(Santiago|Providencia|Las Condes|Ñuñoa|Vitacura|Maipú|La Florida|Viña del Mar|Valparaíso|Concepción|Antofagasta|La Serena|Rancagua|Temuco|Puerto Montt|Valdivia|Calama|Iquique|Arica)', card_text, re.IGNORECASE)
                if m_loc:
                    ubicacion = m_loc.group(0)

            superficie_m2 = None
            m2 = re.search(r'(\d+(?:[,.]\d+)?)\s*m[²2]', card_text, re.IGNORECASE)
            if m2:
                try:
                    superficie_m2 = float(m2.group(1).replace(',', '.'))
                except ValueError:
                    pass

            ambientes = None
            m_amb = re.search(r'(\d+)\s*(?:dorm|hab|amb)', card_text, re.IGNORECASE)
            if m_amb:
                try:
                    ambientes = int(m_amb.group(1))
                except ValueError:
                    pass

            results.append({
                'listing_id':   listing_id,
                'titulo':       titulo or 'Sin titulo',
                'precio':       precio_uf,
                'moneda':       'UF',
                'tipo':         tipo,
                'ubicacion':    ubicacion,
                'url':          full_url if full_url != BASE_URL else None,
                'superficie_m2': superficie_m2,
                'ambientes':    ambientes,
            })
        except Exception as exc:
            log.debug("Error parseando tile: %s", exc)

    log.info("%s: %d listings", path, len(results))
    return results


def main():
    log.info("=== Scraper Yapo.cl iniciado ===")

    try:
        conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME,
                                user=DB_USER, password=DB_PASS, connect_timeout=15)
        log.info("Conexion DB OK (%s@%s/%s)", DB_USER, DB_HOST, DB_NAME)
    except Exception as exc:
        raise RuntimeError(f"Error conectando a PostgreSQL: {exc}") from exc

    try:
        create_table(conn)
        sess = get_session()
        total_nuevos = 0
        total_actualizados = 0

        for path in SUBCATEGORIES:
            rows = scrape_subcategory(sess, path)
            if rows:
                nuevos, actualizados = upsert(conn, rows)
                total_nuevos += nuevos
                total_actualizados += actualizados
                log.info("[%s] %d nuevos, %d actualizados", path, nuevos, actualizados)
            time.sleep(DELAY)

    finally:
        conn.close()
        log.info("Conexion DB cerrada")

    print(f"Finalizado: {total_nuevos} nuevos, {total_actualizados} actualizados")
    log.info("=== Scraper Yapo.cl finalizado ===")


if __name__ == '__main__':
    main()