#!/usr/bin/env python3
"""
Scraper ArgenProp.com.ar – Bienes Raices Argentina
Estrategia: iterar URLs del sitemap de listing (por barrio/tipo) y extraer
listing cards con datos SSR completos en atributos del <a class="card">.
DB: argenprop_ar
Checkpoint: /opt/scrapers/argenprop_ar/state.json
  {"sitemap_idx": N, "url_idx": M, "total_new": X}
"""
import os
import re
import time
import json
import gzip
import logging
from datetime import datetime

import requests
from bs4 import BeautifulSoup
import psycopg2
from psycopg2.extras import execute_values

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s %(levelname)s %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
)
log = logging.getLogger('argenprop')

PROXY   = 'socks5h://127.0.0.1:1090'
PROXIES = {'http': PROXY, 'https': PROXY}
UA      = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
BASE    = 'https://www.argenprop.com'
DELAY   = 1.5

DB_HOST = '100.75.240.87'
DB_PORT = 5432
DB_NAME = 'argenprop_ar'
DB_USER = 'pgadmin'
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'

DAILY   = bool(os.environ.get('ARGENPROP_DAILY'))  # True en modo daily, False en backfill
MAX_SITEMAPS = int(os.environ.get('MAX_SITEMAPS', '99'))  # cuantos sub-sitemaps procesar

STATE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'state.json')

# Sub-sitemaps de listing (se descubren dinamicamente desde el índice)
SITEMAP_INDEX = 'https://www.argenprop.com/sitemap.xml'


def load_state():
    if os.path.exists(STATE_FILE):
        with open(STATE_FILE) as f:
            st = json.load(f)
        log.info('Checkpoint: sitemap_idx=%d url_idx=%d total_new=%d',
                 st['sitemap_idx'], st['url_idx'], st.get('total_new', 0))
        return st
    return {'sitemap_idx': 0, 'url_idx': 0, 'total_new': 0}


def save_state(si, ui, total_new):
    with open(STATE_FILE, 'w') as f:
        json.dump({'sitemap_idx': si, 'url_idx': ui, 'total_new': total_new,
                   'ts': datetime.utcnow().isoformat()}, f)


def get_session():
    sess = requests.Session()
    sess.proxies.update(PROXIES)
    sess.headers.update({
        'User-Agent':      UA,
        'Accept':          'text/html,application/xhtml+xml,*/*;q=0.8',
        'Accept-Language': 'es-AR,es;q=0.9',
        'Accept-Encoding': 'gzip, deflate',
        'Connection':      'keep-alive',
    })
    return sess


def get_conn():
    return psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME,
                            user=DB_USER, password=DB_PASS, connect_timeout=15)


def create_db(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS listings (
            listing_id    TEXT PRIMARY KEY,
            titulo        TEXT,
            precio        BIGINT,
            moneda        TEXT DEFAULT 'USD',
            tipo          TEXT,
            ubicacion     TEXT,
            url           TEXT UNIQUE,
            superficie_m2 DECIMAL,
            ambientes     INT,
            scraped_at    TIMESTAMP DEFAULT NOW()
        )
    """)
    conn.commit()
    cur.close()


def upsert(conn, rows):
    if not rows:
        return 0
    deduped = list({r['listing_id']: r for r in rows}.values())
    cur = conn.cursor()
    ids = [r['listing_id'] for r in deduped]
    cur.execute('SELECT listing_id FROM listings WHERE listing_id = ANY(%s)', (ids,))
    existing = {rec[0] for rec in cur.fetchall()}
    nuevos = sum(1 for r in deduped if r['listing_id'] not in existing)
    now = datetime.utcnow()
    execute_values(cur, """
        INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion,
                              url, superficie_m2, ambientes, scraped_at)
        VALUES %s
        ON CONFLICT (listing_id) DO UPDATE SET
            titulo        = EXCLUDED.titulo,
            precio        = EXCLUDED.precio,
            moneda        = EXCLUDED.moneda,
            tipo          = EXCLUDED.tipo,
            ubicacion     = EXCLUDED.ubicacion,
            url           = EXCLUDED.url,
            superficie_m2 = EXCLUDED.superficie_m2,
            ambientes     = EXCLUDED.ambientes,
            scraped_at    = EXCLUDED.scraped_at
    """, [(r['listing_id'], r.get('titulo'), r.get('precio'), r.get('moneda', 'USD'),
           r.get('tipo'), r.get('ubicacion'), r.get('url'),
           r.get('superficie_m2'), r.get('ambientes'), now) for r in deduped])
    conn.commit()
    cur.close()
    return nuevos


def fetch_gz(sess, url, retries=3):
    """Descarga un .xml.gz y retorna lista de <loc> URLs."""
    for attempt in range(retries):
        try:
            r = sess.get(url, timeout=30)
            if r.status_code in (404, 410):
                return []
            r.raise_for_status()
            try:
                content = gzip.decompress(r.content).decode('utf-8', errors='replace')
            except Exception:
                content = r.text
            return re.findall(r'<loc>(.*?)</loc>', content)
        except Exception as e:
            log.warning('sitemap %s intento %d: %s', url, attempt+1, e)
            if attempt < retries - 1:
                time.sleep([5, 15][attempt % 2])
    return []


def fetch_page(sess, url, retries=4):
    """Descarga una página de listado y extrae listing items."""
    backoffs = [2, 5, 15, 30]
    for attempt in range(retries):
        try:
            r = sess.get(url, timeout=35)
            # HTTP 202 = AWS WAF challenge, reintentar
            if r.status_code == 202:
                log.debug('HTTP 202 en %s (intento %d)', url, attempt+1)
                if attempt < retries - 1:
                    time.sleep(backoffs[attempt])
                    continue
                return []
            if r.status_code in (403, 429):
                log.warning('HTTP %d en %s', r.status_code, url)
                time.sleep(45)
                continue
            if r.status_code in (404, 410):
                return []
            if r.status_code != 200:
                log.warning('HTTP %d en %s', r.status_code, url)
                return []
            break
        except requests.RequestException as e:
            log.warning('Red %s intento %d: %s', url, attempt+1, e)
            if attempt < retries - 1:
                time.sleep(backoffs[attempt])
            else:
                return []

    soup = BeautifulSoup(r.text, 'lxml')
    items = soup.find_all('div', class_='listing__item')
    if not items:
        return []

    results = []
    seen = set()

    for item in items:
        try:
            card = item.find('a', class_='card')
            if not card:
                continue

            listing_id = card.get('idaviso', '').strip()
            if not listing_id or listing_id in seen:
                continue
            seen.add(listing_id)

            href = card.get('href', '')
            full_url = BASE + href if href.startswith('/') else href

            # Precio en USD desde atributo montonormalizado
            try:
                precio = int(card.get('montonormalizado', 0) or 0)
            except (ValueError, TypeError):
                precio = None
            moneda = 'USD'  # argenprop lista en USD

            # Dormitorios
            try:
                ambientes = int(card.get('dormitorios', 0) or 0) or None
            except (ValueError, TypeError):
                ambientes = None

            # Tipo de inmueble desde la URL
            tipo = 'venta'
            if 'arriendo' in href.lower() or 'alquiler' in href.lower():
                tipo = 'arriendo'

            # Tipo de propiedad
            tipo_prop = None
            for t in ('casa', 'departamento', 'terreno', 'campo', 'cochera', 'oficina', 'local', 'ph'):
                if t in href.lower():
                    tipo_prop = t
                    break

            # Ubicación desde URL: /casa-en-venta-en-palermo-... → palermo
            loc_m = re.search(r'-en-(?:venta|alquiler|arriendo)-en-([a-z\-]+?)(?:-\d|-\-|$)', href)
            ubicacion = loc_m.group(1).replace('-', ' ').title() if loc_m else None

            # Título y m2 del texto del card
            card_text = card.get_text(separator=' ', strip=True)
            titulo_m = re.search(r'((?:Casa|Departamento|Terreno|Campo|Oficina|Local|PH|Cochera)[^|]{5,80})', card_text, re.I)
            titulo = titulo_m.group(1).strip()[:255] if titulo_m else card_text[:100]

            m2_m = re.search(r'(\d+(?:[,\.]\d+)?)\s*m[2²]', card_text, re.I)
            superficie_m2 = None
            if m2_m:
                try:
                    superficie_m2 = float(m2_m.group(1).replace(',', '.'))
                except ValueError:
                    pass

            results.append({
                'listing_id':    listing_id,
                'titulo':        titulo or 'Sin titulo',
                'precio':        precio or None,
                'moneda':        moneda,
                'tipo':          tipo_prop or tipo,
                'ubicacion':     ubicacion,
                'url':           full_url,
                'superficie_m2': superficie_m2,
                'ambientes':     ambientes,
            })
        except Exception as e:
            log.debug('Error card: %s', e)

    return results


def get_listing_sitemaps(sess):
    """Retorna lista de sub-sitemaps de tipo listing."""
    r = sess.get(SITEMAP_INDEX, timeout=20)
    all_sms = re.findall(r'<loc>(.*?)</loc>', r.text)
    # Solo los sitemaps de listing (no ficha)
    listing_sms = [u for u in all_sms if 'sitemap-listing' in u]
    log.info('Sitemaps de listing: %d', len(listing_sms))
    return listing_sms


def main():
    state     = load_state()
    si_start  = state['sitemap_idx']
    ui_start  = state['url_idx']
    total_new = state.get('total_new', 0)

    log.info('=== Scraper ArgenProp AR — modo %s ===',
             'DAILY' if DAILY else 'BACKFILL')
    log.info('Desde sitemap %d url %d', si_start, ui_start)

    sess = get_session()
    conn = get_conn()
    create_db(conn)

    # Descubrir sitemaps de listing
    listing_sms = get_listing_sitemaps(sess)
    listing_sms = listing_sms[:MAX_SITEMAPS]

    for si in range(si_start, len(listing_sms)):
        sm_url = listing_sms[si]
        log.info('== Sitemap %d/%d: %s ==', si+1, len(listing_sms), sm_url)

        # Descargar el índice del sub-sitemap (puede tener sub-sub-sitemaps)
        sm_urls = fetch_gz(sess, sm_url)
        if not sm_urls:
            # Intentar sin gz
            sm_urls = [sm_url.replace('.gz', '')]
        log.info('  sub-sitemaps/urls: %d', len(sm_urls))

        # Si son sub-sitemaps (terminan en .xml.gz), expandir
        final_urls = []
        for u in sm_urls:
            if u.endswith('.xml.gz') or u.endswith('.xml'):
                inner = fetch_gz(sess, u)
                final_urls.extend(inner)
                time.sleep(0.5)
            else:
                final_urls.append(u)

        log.info('  URLs totales a procesar: %d', len(final_urls))

        ui_ini = ui_start if si == si_start else 0

        for ui in range(ui_ini, len(final_urls)):
            page_url = final_urls[ui]
            rows = fetch_page(sess, page_url)

            if rows:
                n = upsert(conn, rows)
                total_new += n
                if n > 0:
                    log.info('[%d/%d] %s: %d nuevos (acum %d)',
                             ui+1, len(final_urls), page_url[-60:], n, total_new)
            else:
                log.debug('[%d/%d] %s: sin resultados', ui+1, len(final_urls), page_url[-60:])

            save_state(si, ui + 1, total_new)
            time.sleep(DELAY)

            # En modo daily parar temprano si no hay nuevos (ya cubierto)
            if DAILY and ui > 200 and total_new == 0:
                log.info('Daily: 200 URLs sin nuevos — deteniendo')
                break

        # Avanzar al siguiente sitemap
        save_state(si + 1, 0, total_new)
        time.sleep(DELAY * 2)

    conn.close()
    log.info('=== Finalizado: %d nuevos ===', total_new)
    print(f'Finalizado: {total_new} nuevos')


if __name__ == '__main__':
    main()