feat: FincaRaiz Colombia scraper (390+ listings, /paginaN)

inmuebles/fincaraiz_co/scraper.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""
+Scraper FincaRaiz.com.co Colombia -> postgres17-central / fincaraiz_co
+URL venta: /venta/apartamentos, /venta/apartamentos/pagina{N}
+URL arriendo: /arriendo/apartamentos, /arriendo/apartamentos/pagina{N}
+ID: numero al final de la URL de detalle  /{slug}/{numeric_id}
+Selector: class='listingCard'
+"""
+import os, re, time, logging
+import requests
+from bs4 import BeautifulSoup
+import psycopg2
+from psycopg2.extras import execute_values
+from datetime import datetime
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+log = logging.getLogger('fincaraiz')
+
+PROXY    = 'socks5h://127.0.0.1:1090'
+PROXIES  = {'http': PROXY, 'https': PROXY}
+DB_HOST  = '100.75.240.87'
+DB_NAME  = 'fincaraiz_co'
+DB_USER  = 'pgadmin'
+DB_PASS  = 'J5BVlq65JvedWxZVrcY96OQX'
+UA       = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
+BASE     = 'https://www.fincaraiz.com.co'
+DELAY    = 1.0
+MAX_PAGES   = int(os.environ.get('FR_MAX_PAGES', '30'))
+NEW_STREAK  = 2
+
+PASADAS = [
+    {'tipo': 'venta',    'base': '/venta/apartamentos'},
+    {'tipo': 'arriendo', 'base': '/arriendo/apartamentos'},
+]
+
+
+def get_session():
+    s = requests.Session()
+    s.proxies.update(PROXIES)
+    s.headers.update({
+        'User-Agent': UA,
+        'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8',
+        'Accept-Language': 'es-CO,es;q=0.9',
+        'Referer': BASE + '/',
+    })
+    return s
+
+
+def get_conn():
+    return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME,
+                            user=DB_USER, password=DB_PASS, connect_timeout=10)
+
+
+def create_table(conn):
+    cur = conn.cursor()
+    cur.execute("""
+        CREATE TABLE IF NOT EXISTS listings (
+            listing_id   TEXT PRIMARY KEY,
+            titulo       TEXT,
+            precio       BIGINT,
+            moneda       TEXT DEFAULT 'COP',
+            tipo         TEXT,
+            ubicacion    TEXT,
+            url          TEXT UNIQUE,
+            superficie_m2 DECIMAL,
+            scraped_at   TIMESTAMP DEFAULT NOW()
+        )
+    """)
+    conn.commit()
+    cur.close()
+
+
+def upsert(conn, rows):
+    if not rows:
+        return 0
+    cur = conn.cursor()
+    execute_values(cur,
+        """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at)
+           VALUES %s
+           ON CONFLICT (listing_id) DO UPDATE SET
+             titulo=EXCLUDED.titulo, precio=EXCLUDED.precio,
+             tipo=EXCLUDED.tipo, scraped_at=EXCLUDED.scraped_at""",
+        [(r['listing_id'], r['titulo'], r['precio'], 'COP',
+          r['tipo'], r['ubicacion'], r['url'], datetime.utcnow()) for r in rows]
+    )
+    n = cur.rowcount
+    conn.commit()
+    cur.close()
+    return n
+
+
+def fetch_page(sess, url, tipo):
+    for attempt in range(3):
+        try:
+            resp = sess.get(url, timeout=30)
+            if resp.status_code == 404:
+                return None
+            if resp.status_code in (403, 429):
+                log.warning('HTTP %d en %s', resp.status_code, url)
+                time.sleep(30)
+                continue
+            resp.raise_for_status()
+            break
+        except Exception as e:
+            if attempt == 2:
+                log.error('Error %s: %s', url, e)
+                return []
+            time.sleep([2, 5, 15][attempt])
+
+    soup = BeautifulSoup(resp.text, 'lxml')
+    cards = soup.find_all(class_=re.compile(r'listingCard'))
+    results = []
+
+    for card in cards:
+        try:
+            link_tag = card.find('a', href=True)
+            if not link_tag:
+                continue
+            href = link_tag['href']
+            # ID: numero al final de la URL
+            m = re.search(r'/(\d{6,12})(?:[/?#]|$)', href)
+            if not m:
+                continue
+            listing_id = m.group(1)
+            url_full = BASE + href if href.startswith('/') else href
+
+            titulo_tag = card.find(['h2', 'h3', 'p'])
+            titulo = titulo_tag.get_text(strip=True)[:300] if titulo_tag else 'Sin titulo'
+
+            precio = None
+            precio_tag = card.find(string=re.compile(r'[\$\d][.\d]{3,}'))
+            if precio_tag:
+                nums = re.findall(r'[\d.]+', str(precio_tag))
+                if nums:
+                    try:
+                        precio = int(nums[0].replace('.', ''))
+                    except ValueError:
+                        pass
+
+            loc_tag = card.find(string=re.compile(r'Colombia|Bogot|Medell|Cali|Barranq', re.I))
+            ubicacion = str(loc_tag).strip()[:200] if loc_tag else 'Colombia'
+
+            results.append({
+                'listing_id': listing_id,
+                'titulo':     titulo,
+                'precio':     precio,
+                'tipo':       tipo,
+                'ubicacion':  ubicacion,
+                'url':        url_full,
+            })
+        except Exception as e:
+            log.debug('Error parseando card: %s', e)
+
+    log.info('[%s] %s: %d cards', tipo, url, len(results))
+    return results
+
+
+def scrape_pasada(sess, conn, pasada):
+    tipo = pasada['tipo']
+    base_path = pasada['base']
+    log.info('--- Pasada: %s ---', tipo)
+    streak = 0
+    total = 0
+
+    for page in range(1, MAX_PAGES + 1):
+        if page == 1:
+            url = BASE + base_path
+        else:
+            url = BASE + base_path + f'/pagina{page}'
+
+        rows = fetch_page(sess, url, tipo)
+        if rows is None:
+            log.info('Pagina %d: 404, fin', page)
+            break
+        if not rows:
+            streak += 1
+            if streak >= NEW_STREAK:
+                log.info('Deteniendo pasada %s: %d paginas sin resultados', tipo, streak)
+                break
+            time.sleep(DELAY)
+            continue
+
+        n = upsert(conn, rows)
+        streak = 0 if n > 0 else streak + 1
+        total += n
+        if streak >= NEW_STREAK:
+            log.info('Deteniendo pasada %s: %d paginas sin nuevos', tipo, streak)
+            break
+        time.sleep(DELAY)
+
+    return total
+
+
+def main():
+    log.info('=== Iniciando scraper FincaRaiz Colombia ===')
+    sess = get_session()
+    conn = get_conn()
+    create_table(conn)
+
+    total = 0
+    for pasada in PASADAS:
+        total += scrape_pasada(sess, conn, pasada)
+
+    conn.close()
+    print(f'Finalizado: {total} nuevos, 0 actualizados')
+    log.info('Finalizado: %d nuevos, 0 actualizados', total)
+
+
+if __name__ == '__main__':
+    main()