diff --git a/inmuebles/fincaraiz_co/scraper.py b/inmuebles/fincaraiz_co/scraper.py new file mode 100644 index 0000000..67c9fe2 --- /dev/null +++ b/inmuebles/fincaraiz_co/scraper.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +""" +Scraper FincaRaiz.com.co Colombia -> postgres17-central / fincaraiz_co +URL venta: /venta/apartamentos, /venta/apartamentos/pagina{N} +URL arriendo: /arriendo/apartamentos, /arriendo/apartamentos/pagina{N} +ID: numero al final de la URL de detalle /{slug}/{numeric_id} +Selector: class='listingCard' +""" +import os, re, time, logging +import requests +from bs4 import BeautifulSoup +import psycopg2 +from psycopg2.extras import execute_values +from datetime import datetime + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +log = logging.getLogger('fincaraiz') + +PROXY = 'socks5h://127.0.0.1:1090' +PROXIES = {'http': PROXY, 'https': PROXY} +DB_HOST = '100.75.240.87' +DB_NAME = 'fincaraiz_co' +DB_USER = 'pgadmin' +DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' +UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36' +BASE = 'https://www.fincaraiz.com.co' +DELAY = 1.0 +MAX_PAGES = int(os.environ.get('FR_MAX_PAGES', '30')) +NEW_STREAK = 2 + +PASADAS = [ + {'tipo': 'venta', 'base': '/venta/apartamentos'}, + {'tipo': 'arriendo', 'base': '/arriendo/apartamentos'}, +] + + +def get_session(): + s = requests.Session() + s.proxies.update(PROXIES) + s.headers.update({ + 'User-Agent': UA, + 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8', + 'Accept-Language': 'es-CO,es;q=0.9', + 'Referer': BASE + '/', + }) + return s + + +def get_conn(): + return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME, + user=DB_USER, password=DB_PASS, connect_timeout=10) + + +def create_table(conn): + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS listings ( + listing_id TEXT PRIMARY KEY, + titulo TEXT, + precio BIGINT, + moneda TEXT DEFAULT 'COP', + tipo TEXT, + ubicacion TEXT, + url TEXT UNIQUE, + superficie_m2 DECIMAL, + scraped_at TIMESTAMP DEFAULT NOW() + ) + """) + conn.commit() + cur.close() + + +def upsert(conn, rows): + if not rows: + return 0 + cur = conn.cursor() + execute_values(cur, + """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at) + VALUES %s + ON CONFLICT (listing_id) DO UPDATE SET + titulo=EXCLUDED.titulo, precio=EXCLUDED.precio, + tipo=EXCLUDED.tipo, scraped_at=EXCLUDED.scraped_at""", + [(r['listing_id'], r['titulo'], r['precio'], 'COP', + r['tipo'], r['ubicacion'], r['url'], datetime.utcnow()) for r in rows] + ) + n = cur.rowcount + conn.commit() + cur.close() + return n + + +def fetch_page(sess, url, tipo): + for attempt in range(3): + try: + resp = sess.get(url, timeout=30) + if resp.status_code == 404: + return None + if resp.status_code in (403, 429): + log.warning('HTTP %d en %s', resp.status_code, url) + time.sleep(30) + continue + resp.raise_for_status() + break + except Exception as e: + if attempt == 2: + log.error('Error %s: %s', url, e) + return [] + time.sleep([2, 5, 15][attempt]) + + soup = BeautifulSoup(resp.text, 'lxml') + cards = soup.find_all(class_=re.compile(r'listingCard')) + results = [] + + for card in cards: + try: + link_tag = card.find('a', href=True) + if not link_tag: + continue + href = link_tag['href'] + # ID: numero al final de la URL + m = re.search(r'/(\d{6,12})(?:[/?#]|$)', href) + if not m: + continue + listing_id = m.group(1) + url_full = BASE + href if href.startswith('/') else href + + titulo_tag = card.find(['h2', 'h3', 'p']) + titulo = titulo_tag.get_text(strip=True)[:300] if titulo_tag else 'Sin titulo' + + precio = None + precio_tag = card.find(string=re.compile(r'[\$\d][.\d]{3,}')) + if precio_tag: + nums = re.findall(r'[\d.]+', str(precio_tag)) + if nums: + try: + precio = int(nums[0].replace('.', '')) + except ValueError: + pass + + loc_tag = card.find(string=re.compile(r'Colombia|Bogot|Medell|Cali|Barranq', re.I)) + ubicacion = str(loc_tag).strip()[:200] if loc_tag else 'Colombia' + + results.append({ + 'listing_id': listing_id, + 'titulo': titulo, + 'precio': precio, + 'tipo': tipo, + 'ubicacion': ubicacion, + 'url': url_full, + }) + except Exception as e: + log.debug('Error parseando card: %s', e) + + log.info('[%s] %s: %d cards', tipo, url, len(results)) + return results + + +def scrape_pasada(sess, conn, pasada): + tipo = pasada['tipo'] + base_path = pasada['base'] + log.info('--- Pasada: %s ---', tipo) + streak = 0 + total = 0 + + for page in range(1, MAX_PAGES + 1): + if page == 1: + url = BASE + base_path + else: + url = BASE + base_path + f'/pagina{page}' + + rows = fetch_page(sess, url, tipo) + if rows is None: + log.info('Pagina %d: 404, fin', page) + break + if not rows: + streak += 1 + if streak >= NEW_STREAK: + log.info('Deteniendo pasada %s: %d paginas sin resultados', tipo, streak) + break + time.sleep(DELAY) + continue + + n = upsert(conn, rows) + streak = 0 if n > 0 else streak + 1 + total += n + if streak >= NEW_STREAK: + log.info('Deteniendo pasada %s: %d paginas sin nuevos', tipo, streak) + break + time.sleep(DELAY) + + return total + + +def main(): + log.info('=== Iniciando scraper FincaRaiz Colombia ===') + sess = get_session() + conn = get_conn() + create_table(conn) + + total = 0 + for pasada in PASADAS: + total += scrape_pasada(sess, conn, pasada) + + conn.close() + print(f'Finalizado: {total} nuevos, 0 actualizados') + log.info('Finalizado: %d nuevos, 0 actualizados', total) + + +if __name__ == '__main__': + main()