#!/usr/bin/env python3
"""
Scraper Portal Inmobiliario Chile (MercadoLibre) -> postgres17-central / portal_inmobiliario_cl
URL: /venta/casas_Desde_{N}_NoIndex_True  (step=48 por pagina)
ID: extraido del JSON embebido -> "id":"MLC{numero}"
"""
import os, re, time, logging
import requests
import psycopg2
from psycopg2.extras import execute_values
from datetime import datetime

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
log = logging.getLogger('portal_inmobiliario')

PROXY    = 'socks5h://127.0.0.1:1090'
PROXIES  = {'http': PROXY, 'https': PROXY}
DB_HOST  = '100.75.240.87'
DB_NAME  = 'portal_inmobiliario_cl'
DB_USER  = 'pgadmin'
DB_PASS  = 'J5BVlq65JvedWxZVrcY96OQX'
UA       = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
BASE     = 'https://www.portalinmobiliario.com'
PAGE1_URL = BASE + '/venta/casas'
PAGE_URL  = BASE + '/venta/casas_Desde_{}_NoIndex_True'
STEP      = 48
DELAY     = 1.2
MAX_PAGES = int(os.environ.get('PI_MAX_PAGES', '40'))
NEW_STREAK = 2


def get_session():
    s = requests.Session()
    s.proxies.update(PROXIES)
    s.headers.update({
        'User-Agent': UA,
        'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8',
        'Accept-Language': 'es-CL,es;q=0.9',
    })
    return s


def get_conn():
    return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME,
                            user=DB_USER, password=DB_PASS, connect_timeout=10)


def create_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS listings (
            listing_id  TEXT PRIMARY KEY,
            titulo      TEXT,
            precio      BIGINT,
            moneda      TEXT DEFAULT 'CLP',
            tipo        TEXT DEFAULT 'venta',
            ubicacion   TEXT,
            url         TEXT,
            ambientes   INT,
            superficie_m2 DECIMAL,
            scraped_at  TIMESTAMP DEFAULT NOW()
        )
    """)
    conn.commit()
    cur.close()


def upsert(conn, rows):
    if not rows:
        return 0
    cur = conn.cursor()
    execute_values(cur,
        """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at)
           VALUES %s
           ON CONFLICT (listing_id) DO UPDATE SET
             titulo=EXCLUDED.titulo, precio=EXCLUDED.precio, scraped_at=EXCLUDED.scraped_at""",
        [(r['listing_id'], r['titulo'], r['precio'], r.get('moneda', 'CLP'),
          'venta', r.get('ubicacion', 'Chile'), r['url'], datetime.utcnow()) for r in rows]
    )
    n = cur.rowcount
    conn.commit()
    cur.close()
    return n


def extract_listings(html):
    results = []
    mlc_ids = list(dict.fromkeys(re.findall(r'"id"\s*:\s*"(MLC\d+)"', html)))

    for mlc_id in mlc_ids:
        pos = html.find(f'"id":"{mlc_id}"')
        if pos == -1:
            pos = html.find(f'"id": "{mlc_id}"')
        if pos == -1:
            continue

        start = max(0, pos - 500)
        end   = min(len(html), pos + 1000)
        ctx   = html[start:end]

        titulo = ''
        precio = None
        moneda = 'CLP'
        ubicacion = 'Chile'
        url = BASE + '/' + mlc_id

        m = re.search(r'"title"\s*:\s*"([^"]{5,200})"', ctx)
        if not m:
            m = re.search(r'"name"\s*:\s*"([^"]{5,200})"', ctx)
        if m:
            titulo = m.group(1)

        m = re.search(r'"amount"\s*:\s*(\d+)', ctx)
        if not m:
            m = re.search(r'"price"\s*:\s*(\d+)', ctx)
        if m:
            precio = int(m.group(1))

        m = re.search(r'"currency_id"\s*:\s*"([A-Z]{3})"', ctx)
        if m:
            moneda = m.group(1)

        m = re.search(r'"permalink"\s*:\s*"(https://www\.portalinmobiliario\.com/[^"]+)"', ctx)
        if m:
            url = m.group(1)

        m = re.search(r'"city_name"\s*:\s*"([^"]+)"', ctx)
        if m:
            ubicacion = m.group(1)

        results.append({
            'listing_id': mlc_id,
            'titulo':     titulo[:300] or 'Sin titulo',
            'precio':     precio,
            'moneda':     moneda,
            'ubicacion':  ubicacion[:200],
            'url':        url,
        })

    return results


def fetch_page(sess, page):
    if page == 1:
        url = PAGE1_URL
    else:
        url = PAGE_URL.format((page - 1) * STEP)

    for attempt in range(3):
        try:
            resp = sess.get(url, timeout=30)
            if resp.status_code == 404:
                return None
            if resp.status_code in (403, 429):
                log.warning('HTTP %d en %s', resp.status_code, url)
                time.sleep(30)
                continue
            resp.raise_for_status()
            break
        except Exception as e:
            if attempt == 2:
                log.error('Error %s: %s', url, e)
                return []
            time.sleep([2, 5, 15][attempt])

    listings = extract_listings(resp.text)
    log.info('Pagina %d (offset=%d): %d listings de %s', page, (page-1)*STEP, len(listings), url)
    return listings


def main():
    log.info('=== Iniciando scraper Portal Inmobiliario Chile ===')
    sess = get_session()
    conn = get_conn()
    create_table(conn)

    total_new = 0
    streak = 0

    for page in range(1, MAX_PAGES + 1):
        rows = fetch_page(sess, page)
        if rows is None:
            log.info('Pagina %d: fin de paginacion', page)
            break
        if not rows:
            streak += 1
            log.info('Pagina %d: sin listings (racha %d/%d)', page, streak, NEW_STREAK)
            if streak >= NEW_STREAK:
                break
            time.sleep(DELAY)
            continue

        n = upsert(conn, rows)
        streak = 0 if n > 0 else streak + 1
        total_new += n
        log.info('Pagina %d: %d upserted', page, n)

        if streak >= NEW_STREAK:
            log.info('Deteniendo: %d paginas sin nuevos', streak)
            break
        time.sleep(DELAY)

    conn.close()
    print(f'Finalizado: {total_new} nuevos, 0 actualizados')
    log.info('Finalizado: %d nuevos, 0 actualizados', total_new)


if __name__ == '__main__':
    main()