diff --git a/inmuebles/encuentra24_ca/scraper.py b/inmuebles/encuentra24_ca/scraper.py new file mode 100644 index 0000000..d6e310d --- /dev/null +++ b/inmuebles/encuentra24_ca/scraper.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +""" +Scraper: Encuentra24.com - Bienes Raices Centroamerica (5 paises) +Paises: Costa Rica, Guatemala, Honduras, Nicaragua, El Salvador +DB: encuentra24_ca +Paginacion: pagina 1 = URL base, pagina N>=2 = URL + '.{N}' +""" + +import os +import re +import time +import logging +from datetime import datetime + +import requests +from bs4 import BeautifulSoup +import psycopg2 +from psycopg2.extras import execute_values + +PROXY_URL = 'socks5h://127.0.0.1:1090' +PROXIES = {'http': PROXY_URL, 'https': PROXY_URL} + +DB_HOST = '100.75.240.87' +DB_PORT = 5432 +DB_NAME = 'encuentra24_ca' +DB_USER = 'pgadmin' +DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' + +UA = ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36' +) + +DELAY = 1.2 +MAX_PAGES = int(os.environ.get('MAX_PAGES', '15')) +NEW_STREAK = 3 + +BASE_URL = 'https://www.encuentra24.com' + +COUNTRIES = [ + ('cr', 'costa-rica'), + ('gt', 'guatemala'), + ('hn', 'honduras'), + ('ni', 'nicaragua'), + ('sv', 'el-salvador'), +] + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', +) +log = logging.getLogger(__name__) + + +def get_session(): + sess = requests.Session() + sess.proxies.update(PROXIES) + sess.headers.update({ + 'User-Agent': UA, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'es-CR,es;q=0.9,en;q=0.8', + 'Connection': 'keep-alive', + 'Cache-Control': 'no-cache', + 'Upgrade-Insecure-Requests':'1', + }) + return sess + + +def create_table(conn): + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS listings ( + listing_id TEXT PRIMARY KEY, + titulo TEXT, + precio BIGINT, + moneda TEXT, + tipo TEXT, + ubicacion TEXT, + url TEXT UNIQUE, + pais TEXT, + superficie_m2 DECIMAL, + scraped_at TIMESTAMP DEFAULT NOW() + ) + """) + conn.commit() + cur.close() + log.info("Tabla listings verificada/creada OK") + + +def upsert(conn, rows): + if not rows: + return 0, 0 + deduped = list({r['listing_id']: r for r in rows}.values()) + cur = conn.cursor() + ids = [r['listing_id'] for r in deduped] + cur.execute('SELECT listing_id FROM listings WHERE listing_id = ANY(%s)', (ids,)) + existing = {rec[0] for rec in cur.fetchall()} + nuevos = sum(1 for r in deduped if r['listing_id'] not in existing) + actualizados = len(deduped) - nuevos + now = datetime.utcnow() + execute_values(cur, """ + INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, pais, superficie_m2, scraped_at) + VALUES %s + ON CONFLICT (listing_id) DO UPDATE SET + titulo = EXCLUDED.titulo, + precio = EXCLUDED.precio, + moneda = EXCLUDED.moneda, + tipo = EXCLUDED.tipo, + ubicacion = EXCLUDED.ubicacion, + url = EXCLUDED.url, + pais = EXCLUDED.pais, + superficie_m2 = EXCLUDED.superficie_m2, + scraped_at = EXCLUDED.scraped_at + """, [ + (r['listing_id'], r.get('titulo'), r.get('precio'), r.get('moneda'), + r.get('tipo'), r.get('ubicacion'), r.get('url'), r.get('pais'), + r.get('superficie_m2'), now) + for r in deduped + ]) + conn.commit() + cur.close() + return nuevos, actualizados + + +def page_url(cc, country_slug, page): + base = f'{BASE_URL}/{country_slug}-es/bienes-raices-venta-de-propiedades' + if page == 1: + return base + return f'{base}.{page}' + + +def _detect_tipo(text): + tl = text.lower() + if any(k in tl for k in ('alquiler', 'arriendo', 'arrendar', 'renta', 'rento')): + return 'arriendo' + if any(k in tl for k in ('venta', 'vender', 'en venta', 'se vende')): + return 'venta' + return 'venta' + + +def _clean_precio(text): + m = re.search(r'([\d\.,]+)', text.replace('\xa0', '').replace(' ', '')) + if not m: + return None + try: + return int(m.group(1).replace(',', '').replace('.', '').rstrip('0') or '0') + except ValueError: + return None + + +def _extract_listing_id(href): + path = href.split('?')[0].split('#')[0].rstrip('/') + segments = [s for s in path.split('/') if s] + if not segments: + return None + last = segments[-1] + m = re.search(r'(\d{5,})', last) + if m: + return m.group(1) + if len(last) > 4: + return last + return None + + +def scrape_page(sess, url, cc): + backoffs = [3, 8, 20] + resp = None + for attempt, wait in enumerate(backoffs, start=1): + try: + resp = sess.get(url, timeout=35) + except requests.RequestException as exc: + log.warning("Red intento %d/3 [%s]: %s", attempt, url, exc) + if attempt < 3: + time.sleep(wait) + continue + return [] + + if resp.status_code in (403, 429): + log.warning("HTTP %d en %s — espera 45s", resp.status_code, url) + time.sleep(45) + if attempt >= 3: + return [] + continue + + if resp.status_code == 404: + log.info("404 — fin de paginacion en %s", url) + return None + + if resp.status_code != 200: + log.warning("HTTP %d inesperado en %s", resp.status_code, url) + return [] + + break + + if resp is None or not resp.text.strip(): + return [] + + soup = BeautifulSoup(resp.text, 'lxml') + + card_selectors = [ + 'div.listing-card', + 'article.listing', + 'div[class*="listing-card"]', + 'div[class*="result-item"]', + 'li[class*="listing-item"]', + 'div[class*="ad-card"]', + 'article[class*="property"]', + 'div[class*="property-card"]', + ] + cards = [] + for sel in card_selectors: + found = soup.select(sel) + if found: + log.debug("Cards con '%s': %d", sel, len(found)) + cards = found + break + + if not cards: + links = soup.select('a[href*="/bienes-raices"]') + if links: + cards = links + + if not cards: + log.info("Sin cards en %s", url) + return [] + + results = [] + seen = set() + + for card in cards: + try: + if card.name == 'a': + link_tag = card + else: + link_tag = ( + card.select_one('a[href*="/bienes-raices"]') + or card.select_one('a[href*="/inmuebles"]') + or card.select_one('a') + ) + + href = (link_tag.get('href', '') if link_tag else '').strip() + if not href: + continue + + full_url = href if href.startswith('http') else BASE_URL + href + + listing_id = _extract_listing_id(href) + if not listing_id or listing_id in seen: + continue + seen.add(listing_id) + + card_text = card.get_text(separator=' ', strip=True) + + titulo = None + for sel in ('h2', 'h3', 'h1', '[class*="title"]', '[class*="titulo"]', '[class*="heading"]'): + el = card.select_one(sel) + if el: + t = el.get_text(strip=True) + if t: + titulo = t[:255] + break + if not titulo and link_tag: + titulo = link_tag.get_text(strip=True)[:255] + titulo = titulo or 'Sin titulo' + + precio = None + moneda = 'USD' + for sel in ('[class*="price"]', '[class*="precio"]', '[class*="valor"]', '.amount', '.value'): + el = card.select_one(sel) + if el: + t = el.get_text(strip=True) + if 'USD' in t or '$' in t: + moneda = 'USD' + precio = _clean_precio(t) + if precio and precio > 100: + break + + tipo = _detect_tipo(href + ' ' + card_text) + + ubicacion = None + for sel in ('[class*="location"]', '[class*="ubicacion"]', '[class*="city"]', + '[class*="ciudad"]', '[class*="address"]', '[class*="region"]'): + el = card.select_one(sel) + if el: + t = el.get_text(strip=True) + if t: + ubicacion = t[:200] + break + + superficie_m2 = None + m2 = re.search(r'(\d+(?:[,\.]\d+)?)\s*m[²2]', card_text, re.IGNORECASE) + if m2: + try: + superficie_m2 = float(m2.group(1).replace(',', '.')) + except ValueError: + pass + + results.append({ + 'listing_id': listing_id, + 'titulo': titulo, + 'precio': precio, + 'moneda': moneda, + 'tipo': tipo, + 'ubicacion': ubicacion, + 'url': full_url, + 'pais': cc, + 'superficie_m2': superficie_m2, + }) + except Exception as exc: + log.debug("Error parseando card: %s", exc) + + log.info(" -> %d listings en %s", len(results), url) + return results + + +def main(): + total_nuevos = 0 + total_actualizados = 0 + + log.info("=== Scraper Encuentra24 CA Bienes Raices iniciado ===") + + try: + conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, + user=DB_USER, password=DB_PASS, connect_timeout=15) + log.info("Conectado a PostgreSQL: %s@%s/%s", DB_USER, DB_HOST, DB_NAME) + except Exception as exc: + raise RuntimeError(f"No se pudo conectar a PostgreSQL: {exc}") from exc + + try: + create_table(conn) + sess = get_session() + + for cc, country_slug in COUNTRIES: + log.info("== Pais: %s (%s) ==", country_slug.upper(), cc) + streak_sin_nuevos = 0 + + for page in range(1, MAX_PAGES + 1): + url = page_url(cc, country_slug, page) + log.info("[%s] Pagina %d/%d -> %s", cc, page, MAX_PAGES, url) + + rows = scrape_page(sess, url, cc) + + if rows is None: + log.info("[%s] Pagina %d devolvio 404 — fin para este pais", cc, page) + break + + if not rows: + streak_sin_nuevos += 1 + if streak_sin_nuevos >= NEW_STREAK: + log.info("[%s] %d paginas sin resultados — siguiente pais", cc, NEW_STREAK) + break + time.sleep(DELAY) + continue + + nuevos, actualizados = upsert(conn, rows) + total_nuevos += nuevos + total_actualizados += actualizados + + log.info("[%s] Pag %d: %d nuevos, %d actualizados (acum %d/%d)", + cc, page, nuevos, actualizados, total_nuevos, total_actualizados) + + if nuevos == 0: + streak_sin_nuevos += 1 + if streak_sin_nuevos >= NEW_STREAK: + log.info("[%s] %d paginas sin nuevos — siguiente pais", cc, NEW_STREAK) + break + else: + streak_sin_nuevos = 0 + + time.sleep(DELAY) + + time.sleep(DELAY * 2) + + finally: + try: + conn.close() + log.info("Conexion PostgreSQL cerrada") + except Exception: + pass + + print(f"Finalizado: {total_nuevos} nuevos, {total_actualizados} actualizados") + log.info("=== Scraper finalizado. Nuevos: %d | Actualizados: %d ===", + total_nuevos, total_actualizados) + + +if __name__ == '__main__': + main()