feat: add backfill fincaraiz_co (2 pasadas venta+arriendo)

inmuebles/fincaraiz_co/backfill.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""
+Backfill scraper FincaRaiz.com.co Colombia
+Checkpoint: /opt/scrapers/fincaraiz_co/backfill_state.json
+  {"pass_idx": 0|1, "page": N, "total_new": M}
+  pass_idx 0=venta, 1=arriendo
+Para reiniciar desde cero: borrar el archivo de checkpoint.
+"""
+import os, re, time, logging, json
+import requests
+from bs4 import BeautifulSoup
+import psycopg2
+from psycopg2.extras import execute_values
+from datetime import datetime
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s %(levelname)s %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+)
+log = logging.getLogger('fincaraiz_backfill')
+
+PROXY      = 'socks5h://127.0.0.1:1090'
+PROXIES    = {'http': PROXY, 'https': PROXY}
+DB_HOST    = '100.75.240.87'
+DB_NAME    = 'fincaraiz_co'
+DB_USER    = 'pgadmin'
+DB_PASS    = 'J5BVlq65JvedWxZVrcY96OQX'
+UA         = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
+BASE       = 'https://www.fincaraiz.com.co'
+DELAY      = 1.0
+MAX_PAGES  = int(os.environ.get('BF_MAX_PAGES', '300'))
+MAX_STREAK = int(os.environ.get('BF_STREAK', '8'))
+
+PASADAS = [
+    {'tipo': 'venta',    'base': '/venta/apartamentos'},
+    {'tipo': 'arriendo', 'base': '/arriendo/apartamentos'},
+]
+
+STATE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'backfill_state.json')
+
+
+def load_state():
+    if os.path.exists(STATE_FILE):
+        with open(STATE_FILE) as f:
+            st = json.load(f)
+        log.info('Checkpoint: pass_idx=%d pagina=%d total_new=%d',
+                 st['pass_idx'], st['page'], st.get('total_new', 0))
+        return st
+    return {'pass_idx': 0, 'page': 1, 'total_new': 0}
+
+
+def save_state(pass_idx, page, total_new):
+    with open(STATE_FILE, 'w') as f:
+        json.dump({'pass_idx': pass_idx, 'page': page, 'total_new': total_new,
+                   'ts': datetime.utcnow().isoformat()}, f)
+
+
+def get_session():
+    s = requests.Session()
+    s.proxies.update(PROXIES)
+    s.headers.update({'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8',
+                      'Accept-Language': 'es-CO,es;q=0.9', 'Referer': BASE + '/'})
+    return s
+
+
+def get_conn():
+    return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME,
+                            user=DB_USER, password=DB_PASS, connect_timeout=10)
+
+
+def create_table(conn):
+    cur = conn.cursor()
+    cur.execute("""
+        CREATE TABLE IF NOT EXISTS listings (
+            listing_id   TEXT PRIMARY KEY,
+            titulo       TEXT,
+            precio       BIGINT,
+            moneda       TEXT DEFAULT 'COP',
+            tipo         TEXT,
+            ubicacion    TEXT,
+            url          TEXT UNIQUE,
+            superficie_m2 DECIMAL,
+            scraped_at   TIMESTAMP DEFAULT NOW()
+        )
+    """)
+    conn.commit()
+    cur.close()
+
+
+def upsert(conn, rows):
+    if not rows:
+        return 0
+    deduped = list({r['listing_id']: r for r in rows}.values())
+    cur = conn.cursor()
+    execute_values(cur,
+        """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at)
+           VALUES %s
+           ON CONFLICT (listing_id) DO UPDATE SET
+             titulo=EXCLUDED.titulo, precio=EXCLUDED.precio,
+             tipo=EXCLUDED.tipo, scraped_at=EXCLUDED.scraped_at""",
+        [(r['listing_id'], r['titulo'], r['precio'], 'COP',
+          r['tipo'], r['ubicacion'], r['url'], datetime.utcnow()) for r in deduped]
+    )
+    n = cur.rowcount
+    conn.commit()
+    cur.close()
+    return n
+
+
+def fetch_page(sess, url, tipo):
+    for attempt in range(3):
+        try:
+            resp = sess.get(url, timeout=30)
+            if resp.status_code == 404:
+                return None
+            if resp.status_code in (403, 429):
+                log.warning('HTTP %d en %s', resp.status_code, url)
+                time.sleep(60)
+                continue
+            resp.raise_for_status()
+            break
+        except Exception as e:
+            if attempt == 2:
+                log.error('Error %s: %s', url, e)
+                return []
+            time.sleep([2, 5, 15][attempt])
+
+    soup = BeautifulSoup(resp.text, 'lxml')
+    cards = soup.find_all(class_=re.compile(r'listingCard'))
+    results = []
+
+    for card in cards:
+        try:
+            link_tag = card.find('a', href=True)
+            if not link_tag:
+                continue
+            href = link_tag['href']
+            m = re.search(r'/(\d{6,12})(?:[/?#]|$)', href)
+            if not m:
+                continue
+            listing_id = m.group(1)
+            url_full = BASE + href if href.startswith('/') else href
+            titulo_tag = card.find(['h2', 'h3', 'p'])
+            titulo = titulo_tag.get_text(strip=True)[:300] if titulo_tag else 'Sin titulo'
+            precio = None
+            precio_tag = card.find(string=re.compile(r'[\$\d][.\d]{3,}'))
+            if precio_tag:
+                nums = re.findall(r'[\d.]+', str(precio_tag))
+                if nums:
+                    try:
+                        precio = int(nums[0].replace('.', ''))
+                    except ValueError:
+                        pass
+            loc_tag = card.find(string=re.compile(r'Colombia|Bogot|Medell|Cali|Barranq', re.I))
+            ubicacion = str(loc_tag).strip()[:200] if loc_tag else 'Colombia'
+            results.append({'listing_id': listing_id, 'titulo': titulo,
+                            'precio': precio, 'tipo': tipo, 'ubicacion': ubicacion, 'url': url_full})
+        except Exception as e:
+            log.debug('Error card: %s', e)
+
+    log.info('[%s] %s: %d cards', tipo, url, len(results))
+    return results
+
+
+def scrape_pass(sess, conn, pass_idx, start_page, total_new_in):
+    pasada = PASADAS[pass_idx]
+    tipo = pasada['tipo']
+    base_path = pasada['base']
+    log.info('--- Pasada %d: %s (desde pagina %d) ---', pass_idx, tipo, start_page)
+    streak = 0
+    total = total_new_in
+
+    for page in range(start_page, MAX_PAGES + 1):
+        url = BASE + base_path if page == 1 else BASE + base_path + f'/pagina{page}'
+        rows = fetch_page(sess, url, tipo)
+
+        if rows is None:
+            log.info('[%s] Pagina %d: 404, fin', tipo, page)
+            save_state(pass_idx + 1, 1, total)
+            return total, True  # avanzar a la siguiente pasada
+
+        if not rows:
+            streak += 1
+            save_state(pass_idx, page + 1, total)
+            if streak >= MAX_STREAK:
+                log.info('[%s] Deteniendo: %d paginas sin resultados', tipo, streak)
+                save_state(pass_idx + 1, 1, total)
+                return total, True
+            time.sleep(DELAY)
+            continue
+
+        n = upsert(conn, rows)
+        streak = 0 if n > 0 else streak + 1
+        total += n
+        save_state(pass_idx, page + 1, total)
+        log.info('[%s] Pagina %d: %d nuevos (acumulado %d)', tipo, page, n, total)
+
+        if streak >= MAX_STREAK:
+            log.info('[%s] Deteniendo: %d paginas sin nuevos', tipo, streak)
+            save_state(pass_idx + 1, 1, total)
+            return total, True
+
+        time.sleep(DELAY)
+
+    save_state(pass_idx + 1, 1, total)
+    return total, True
+
+
+def main():
+    state     = load_state()
+    pass_idx  = state['pass_idx']
+    start_page = state['page']
+    total_new  = state.get('total_new', 0)
+    log.info('=== Backfill FincaRaiz CO — pass=%d pagina=%d ===', pass_idx, start_page)
+
+    if pass_idx >= len(PASADAS):
+        log.info('Todas las pasadas completadas segun checkpoint. Para reiniciar borra backfill_state.json')
+        print(f'Finalizado: {total_new} nuevos')
+        return
+
+    sess = get_session()
+    conn = get_conn()
+    create_table(conn)
+
+    # Pasadas pendientes desde el checkpoint
+    for idx in range(pass_idx, len(PASADAS)):
+        sp = start_page if idx == pass_idx else 1
+        total_new, _ = scrape_pass(sess, conn, idx, sp, total_new)
+
+    conn.close()
+    log.info('=== Backfill finalizado: %d nuevos ===', total_new)
+    print(f'Finalizado: {total_new} nuevos')
+
+
+if __name__ == '__main__':
+    main()