#!/usr/bin/env python3
"""
Backfill scraper FincaRaiz.com.co Colombia
Checkpoint: /opt/scrapers/fincaraiz_co/backfill_state.json
  {"pass_idx": 0|1, "page": N, "total_new": M}
  pass_idx 0=venta, 1=arriendo
Para reiniciar desde cero: borrar el archivo de checkpoint.
"""
import os, re, time, logging, json
import requests
from bs4 import BeautifulSoup
import psycopg2
from psycopg2.extras import execute_values
from datetime import datetime

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s %(levelname)s %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
)
log = logging.getLogger('fincaraiz_backfill')

PROXY      = 'socks5h://127.0.0.1:1090'
PROXIES    = {'http': PROXY, 'https': PROXY}
DB_HOST    = '100.75.240.87'
DB_NAME    = 'fincaraiz_co'
DB_USER    = 'pgadmin'
DB_PASS    = 'J5BVlq65JvedWxZVrcY96OQX'
UA         = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
BASE       = 'https://www.fincaraiz.com.co'
DELAY      = 1.0
MAX_PAGES  = int(os.environ.get('BF_MAX_PAGES', '300'))
MAX_STREAK = int(os.environ.get('BF_STREAK', '8'))

PASADAS = [
    {'tipo': 'venta',    'base': '/venta/apartamentos'},
    {'tipo': 'arriendo', 'base': '/arriendo/apartamentos'},
]

STATE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'backfill_state.json')


def load_state():
    if os.path.exists(STATE_FILE):
        with open(STATE_FILE) as f:
            st = json.load(f)
        log.info('Checkpoint: pass_idx=%d pagina=%d total_new=%d',
                 st['pass_idx'], st['page'], st.get('total_new', 0))
        return st
    return {'pass_idx': 0, 'page': 1, 'total_new': 0}


def save_state(pass_idx, page, total_new):
    with open(STATE_FILE, 'w') as f:
        json.dump({'pass_idx': pass_idx, 'page': page, 'total_new': total_new,
                   'ts': datetime.utcnow().isoformat()}, f)


def get_session():
    s = requests.Session()
    s.proxies.update(PROXIES)
    s.headers.update({'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8',
                      'Accept-Language': 'es-CO,es;q=0.9', 'Referer': BASE + '/'})
    return s


def get_conn():
    return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME,
                            user=DB_USER, password=DB_PASS, connect_timeout=10)


def create_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS listings (
            listing_id   TEXT PRIMARY KEY,
            titulo       TEXT,
            precio       BIGINT,
            moneda       TEXT DEFAULT 'COP',
            tipo         TEXT,
            ubicacion    TEXT,
            url          TEXT UNIQUE,
            superficie_m2 DECIMAL,
            scraped_at   TIMESTAMP DEFAULT NOW()
        )
    """)
    conn.commit()
    cur.close()


def upsert(conn, rows):
    if not rows:
        return 0
    deduped = list({r['listing_id']: r for r in rows}.values())
    cur = conn.cursor()
    execute_values(cur,
        """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at)
           VALUES %s
           ON CONFLICT (listing_id) DO UPDATE SET
             titulo=EXCLUDED.titulo, precio=EXCLUDED.precio,
             tipo=EXCLUDED.tipo, scraped_at=EXCLUDED.scraped_at""",
        [(r['listing_id'], r['titulo'], r['precio'], 'COP',
          r['tipo'], r['ubicacion'], r['url'], datetime.utcnow()) for r in deduped]
    )
    n = cur.rowcount
    conn.commit()
    cur.close()
    return n


def fetch_page(sess, url, tipo):
    for attempt in range(3):
        try:
            resp = sess.get(url, timeout=30)
            if resp.status_code == 404:
                return None
            if resp.status_code in (403, 429):
                log.warning('HTTP %d en %s', resp.status_code, url)
                time.sleep(60)
                continue
            resp.raise_for_status()
            break
        except Exception as e:
            if attempt == 2:
                log.error('Error %s: %s', url, e)
                return []
            time.sleep([2, 5, 15][attempt])

    soup = BeautifulSoup(resp.text, 'lxml')
    cards = soup.find_all(class_=re.compile(r'listingCard'))
    results = []

    for card in cards:
        try:
            link_tag = card.find('a', href=True)
            if not link_tag:
                continue
            href = link_tag['href']
            m = re.search(r'/(\d{6,12})(?:[/?#]|$)', href)
            if not m:
                continue
            listing_id = m.group(1)
            url_full = BASE + href if href.startswith('/') else href
            titulo_tag = card.find(['h2', 'h3', 'p'])
            titulo = titulo_tag.get_text(strip=True)[:300] if titulo_tag else 'Sin titulo'
            precio = None
            precio_tag = card.find(string=re.compile(r'[\$\d][.\d]{3,}'))
            if precio_tag:
                nums = re.findall(r'[\d.]+', str(precio_tag))
                if nums:
                    try:
                        precio = int(nums[0].replace('.', ''))
                    except ValueError:
                        pass
            loc_tag = card.find(string=re.compile(r'Colombia|Bogot|Medell|Cali|Barranq', re.I))
            ubicacion = str(loc_tag).strip()[:200] if loc_tag else 'Colombia'
            results.append({'listing_id': listing_id, 'titulo': titulo,
                            'precio': precio, 'tipo': tipo, 'ubicacion': ubicacion, 'url': url_full})
        except Exception as e:
            log.debug('Error card: %s', e)

    log.info('[%s] %s: %d cards', tipo, url, len(results))
    return results


def scrape_pass(sess, conn, pass_idx, start_page, total_new_in):
    pasada = PASADAS[pass_idx]
    tipo = pasada['tipo']
    base_path = pasada['base']
    log.info('--- Pasada %d: %s (desde pagina %d) ---', pass_idx, tipo, start_page)
    streak = 0
    total = total_new_in

    for page in range(start_page, MAX_PAGES + 1):
        url = BASE + base_path if page == 1 else BASE + base_path + f'/pagina{page}'
        rows = fetch_page(sess, url, tipo)

        if rows is None:
            log.info('[%s] Pagina %d: 404, fin', tipo, page)
            save_state(pass_idx + 1, 1, total)
            return total, True  # avanzar a la siguiente pasada

        if not rows:
            streak += 1
            save_state(pass_idx, page + 1, total)
            if streak >= MAX_STREAK:
                log.info('[%s] Deteniendo: %d paginas sin resultados', tipo, streak)
                save_state(pass_idx + 1, 1, total)
                return total, True
            time.sleep(DELAY)
            continue

        n = upsert(conn, rows)
        streak = 0 if n > 0 else streak + 1
        total += n
        save_state(pass_idx, page + 1, total)
        log.info('[%s] Pagina %d: %d nuevos (acumulado %d)', tipo, page, n, total)

        if streak >= MAX_STREAK:
            log.info('[%s] Deteniendo: %d paginas sin nuevos', tipo, streak)
            save_state(pass_idx + 1, 1, total)
            return total, True

        time.sleep(DELAY)

    save_state(pass_idx + 1, 1, total)
    return total, True


def main():
    state     = load_state()
    pass_idx  = state['pass_idx']
    start_page = state['page']
    total_new  = state.get('total_new', 0)
    log.info('=== Backfill FincaRaiz CO — pass=%d pagina=%d ===', pass_idx, start_page)

    if pass_idx >= len(PASADAS):
        log.info('Todas las pasadas completadas segun checkpoint. Para reiniciar borra backfill_state.json')
        print(f'Finalizado: {total_new} nuevos')
        return

    sess = get_session()
    conn = get_conn()
    create_table(conn)

    # Pasadas pendientes desde el checkpoint
    for idx in range(pass_idx, len(PASADAS)):
        sp = start_page if idx == pass_idx else 1
        total_new, _ = scrape_pass(sess, conn, idx, sp, total_new)

    conn.close()
    log.info('=== Backfill finalizado: %d nuevos ===', total_new)
    print(f'Finalizado: {total_new} nuevos')


if __name__ == '__main__':
    main()