portales-hispanohablantes/inmuebles/fincaraiz_co/scraper.py

#!/usr/bin/env python3
"""
Scraper FincaRaiz.com.co Colombia -> postgres17-central / fincaraiz_co
URL venta: /venta/apartamentos, /venta/apartamentos/pagina{N}
URL arriendo: /arriendo/apartamentos, /arriendo/apartamentos/pagina{N}
ID: numero al final de la URL de detalle  /{slug}/{numeric_id}
Selector: class='listingCard'
"""
import os, re, time, logging
import requests
from bs4 import BeautifulSoup
import psycopg2
from psycopg2.extras import execute_values
from datetime import datetime

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
log = logging.getLogger('fincaraiz')

PROXY    = 'socks5h://127.0.0.1:1090'
PROXIES  = {'http': PROXY, 'https': PROXY}
DB_HOST  = '100.75.240.87'
DB_NAME  = 'fincaraiz_co'
DB_USER  = 'pgadmin'
DB_PASS  = 'J5BVlq65JvedWxZVrcY96OQX'
UA       = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
BASE     = 'https://www.fincaraiz.com.co'
DELAY    = 1.0
MAX_PAGES   = int(os.environ.get('FR_MAX_PAGES', '30'))
NEW_STREAK  = 2

PASADAS = [
    {'tipo': 'venta',    'base': '/venta/apartamentos'},
    {'tipo': 'arriendo', 'base': '/arriendo/apartamentos'},
]


def get_session():
    s = requests.Session()
    s.proxies.update(PROXIES)
    s.headers.update({
        'User-Agent': UA,
        'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8',
        'Accept-Language': 'es-CO,es;q=0.9',
        'Referer': BASE + '/',
    })
    return s


def get_conn():
    return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME,
                            user=DB_USER, password=DB_PASS, connect_timeout=10)


def create_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS listings (
            listing_id   TEXT PRIMARY KEY,
            titulo       TEXT,
            precio       BIGINT,
            moneda       TEXT DEFAULT 'COP',
            tipo         TEXT,
            ubicacion    TEXT,
            url          TEXT UNIQUE,
            superficie_m2 DECIMAL,
            scraped_at   TIMESTAMP DEFAULT NOW()
        )
    """)
    conn.commit()
    cur.close()


def upsert(conn, rows):
    if not rows:
        return 0
    cur = conn.cursor()
    execute_values(cur,
        """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at)
           VALUES %s
           ON CONFLICT (listing_id) DO UPDATE SET
             titulo=EXCLUDED.titulo, precio=EXCLUDED.precio,
             tipo=EXCLUDED.tipo, scraped_at=EXCLUDED.scraped_at""",
        [(r['listing_id'], r['titulo'], r['precio'], 'COP',
          r['tipo'], r['ubicacion'], r['url'], datetime.utcnow()) for r in rows]
    )
    n = cur.rowcount
    conn.commit()
    cur.close()
    return n


def fetch_page(sess, url, tipo):
    for attempt in range(3):
        try:
            resp = sess.get(url, timeout=30)
            if resp.status_code == 404:
                return None
            if resp.status_code in (403, 429):
                log.warning('HTTP %d en %s', resp.status_code, url)
                time.sleep(30)
                continue
            resp.raise_for_status()
            break
        except Exception as e:
            if attempt == 2:
                log.error('Error %s: %s', url, e)
                return []
            time.sleep([2, 5, 15][attempt])

    soup = BeautifulSoup(resp.text, 'lxml')
    cards = soup.find_all(class_=re.compile(r'listingCard'))
    results = []

    for card in cards:
        try:
            link_tag = card.find('a', href=True)
            if not link_tag:
                continue
            href = link_tag['href']
            # ID: numero al final de la URL
            m = re.search(r'/(\d{6,12})(?:[/?#]|$)', href)
            if not m:
                continue
            listing_id = m.group(1)
            url_full = BASE + href if href.startswith('/') else href

            titulo_tag = card.find(['h2', 'h3', 'p'])
            titulo = titulo_tag.get_text(strip=True)[:300] if titulo_tag else 'Sin titulo'

            precio = None
            precio_tag = card.find(string=re.compile(r'[\$\d][.\d]{3,}'))
            if precio_tag:
                nums = re.findall(r'[\d.]+', str(precio_tag))
                if nums:
                    try:
                        precio = int(nums[0].replace('.', ''))
                    except ValueError:
                        pass

            loc_tag = card.find(string=re.compile(r'Colombia|Bogot|Medell|Cali|Barranq', re.I))
            ubicacion = str(loc_tag).strip()[:200] if loc_tag else 'Colombia'

            results.append({
                'listing_id': listing_id,
                'titulo':     titulo,
                'precio':     precio,
                'tipo':       tipo,
                'ubicacion':  ubicacion,
                'url':        url_full,
            })
        except Exception as e:
            log.debug('Error parseando card: %s', e)

    log.info('[%s] %s: %d cards', tipo, url, len(results))
    return results


def scrape_pasada(sess, conn, pasada):
    tipo = pasada['tipo']
    base_path = pasada['base']
    log.info('--- Pasada: %s ---', tipo)
    streak = 0
    total = 0

    for page in range(1, MAX_PAGES + 1):
        if page == 1:
            url = BASE + base_path
        else:
            url = BASE + base_path + f'/pagina{page}'

        rows = fetch_page(sess, url, tipo)
        if rows is None:
            log.info('Pagina %d: 404, fin', page)
            break
        if not rows:
            streak += 1
            if streak >= NEW_STREAK:
                log.info('Deteniendo pasada %s: %d paginas sin resultados', tipo, streak)
                break
            time.sleep(DELAY)
            continue

        n = upsert(conn, rows)
        streak = 0 if n > 0 else streak + 1
        total += n
        if streak >= NEW_STREAK:
            log.info('Deteniendo pasada %s: %d paginas sin nuevos', tipo, streak)
            break
        time.sleep(DELAY)

    return total


def main():
    log.info('=== Iniciando scraper FincaRaiz Colombia ===')
    sess = get_session()
    conn = get_conn()
    create_table(conn)

    total = 0
    for pasada in PASADAS:
        total += scrape_pasada(sess, conn, pasada)

    conn.close()
    print(f'Finalizado: {total} nuevos, 0 actualizados')
    log.info('Finalizado: %d nuevos, 0 actualizados', total)


if __name__ == '__main__':
    main()