390 lines
12 KiB
Python
390 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Scraper: Encuentra24.com - Bienes Raices Centroamerica (5 paises)
|
|
Paises: Costa Rica, Guatemala, Honduras, Nicaragua, El Salvador
|
|
DB: encuentra24_ca
|
|
Paginacion: pagina 1 = URL base, pagina N>=2 = URL + '.{N}'
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import psycopg2
|
|
from psycopg2.extras import execute_values
|
|
|
|
PROXY_URL = 'socks5h://127.0.0.1:1090'
|
|
PROXIES = {'http': PROXY_URL, 'https': PROXY_URL}
|
|
|
|
DB_HOST = '100.75.240.87'
|
|
DB_PORT = 5432
|
|
DB_NAME = 'encuentra24_ca'
|
|
DB_USER = 'pgadmin'
|
|
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'
|
|
|
|
UA = (
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
|
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
|
'Chrome/124.0.0.0 Safari/537.36'
|
|
)
|
|
|
|
DELAY = 1.2
|
|
MAX_PAGES = int(os.environ.get('MAX_PAGES', '15'))
|
|
NEW_STREAK = 3
|
|
|
|
BASE_URL = 'https://www.encuentra24.com'
|
|
|
|
COUNTRIES = [
|
|
('cr', 'costa-rica'),
|
|
('gt', 'guatemala'),
|
|
('hn', 'honduras'),
|
|
('ni', 'nicaragua'),
|
|
('sv', 'el-salvador'),
|
|
]
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s [%(levelname)s] %(message)s',
|
|
datefmt='%Y-%m-%d %H:%M:%S',
|
|
)
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def get_session():
|
|
sess = requests.Session()
|
|
sess.proxies.update(PROXIES)
|
|
sess.headers.update({
|
|
'User-Agent': UA,
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'es-CR,es;q=0.9,en;q=0.8',
|
|
'Connection': 'keep-alive',
|
|
'Cache-Control': 'no-cache',
|
|
'Upgrade-Insecure-Requests':'1',
|
|
})
|
|
return sess
|
|
|
|
|
|
def create_table(conn):
|
|
cur = conn.cursor()
|
|
cur.execute("""
|
|
CREATE TABLE IF NOT EXISTS listings (
|
|
listing_id TEXT PRIMARY KEY,
|
|
titulo TEXT,
|
|
precio BIGINT,
|
|
moneda TEXT,
|
|
tipo TEXT,
|
|
ubicacion TEXT,
|
|
url TEXT UNIQUE,
|
|
pais TEXT,
|
|
superficie_m2 DECIMAL,
|
|
scraped_at TIMESTAMP DEFAULT NOW()
|
|
)
|
|
""")
|
|
conn.commit()
|
|
cur.close()
|
|
log.info("Tabla listings verificada/creada OK")
|
|
|
|
|
|
def upsert(conn, rows):
|
|
if not rows:
|
|
return 0, 0
|
|
deduped = list({r['listing_id']: r for r in rows}.values())
|
|
cur = conn.cursor()
|
|
ids = [r['listing_id'] for r in deduped]
|
|
cur.execute('SELECT listing_id FROM listings WHERE listing_id = ANY(%s)', (ids,))
|
|
existing = {rec[0] for rec in cur.fetchall()}
|
|
nuevos = sum(1 for r in deduped if r['listing_id'] not in existing)
|
|
actualizados = len(deduped) - nuevos
|
|
now = datetime.utcnow()
|
|
execute_values(cur, """
|
|
INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, pais, superficie_m2, scraped_at)
|
|
VALUES %s
|
|
ON CONFLICT (listing_id) DO UPDATE SET
|
|
titulo = EXCLUDED.titulo,
|
|
precio = EXCLUDED.precio,
|
|
moneda = EXCLUDED.moneda,
|
|
tipo = EXCLUDED.tipo,
|
|
ubicacion = EXCLUDED.ubicacion,
|
|
url = EXCLUDED.url,
|
|
pais = EXCLUDED.pais,
|
|
superficie_m2 = EXCLUDED.superficie_m2,
|
|
scraped_at = EXCLUDED.scraped_at
|
|
""", [
|
|
(r['listing_id'], r.get('titulo'), r.get('precio'), r.get('moneda'),
|
|
r.get('tipo'), r.get('ubicacion'), r.get('url'), r.get('pais'),
|
|
r.get('superficie_m2'), now)
|
|
for r in deduped
|
|
])
|
|
conn.commit()
|
|
cur.close()
|
|
return nuevos, actualizados
|
|
|
|
|
|
def page_url(cc, country_slug, page):
|
|
base = f'{BASE_URL}/{country_slug}-es/bienes-raices-venta-de-propiedades'
|
|
if page == 1:
|
|
return base
|
|
return f'{base}.{page}'
|
|
|
|
|
|
def _detect_tipo(text):
|
|
tl = text.lower()
|
|
if any(k in tl for k in ('alquiler', 'arriendo', 'arrendar', 'renta', 'rento')):
|
|
return 'arriendo'
|
|
if any(k in tl for k in ('venta', 'vender', 'en venta', 'se vende')):
|
|
return 'venta'
|
|
return 'venta'
|
|
|
|
|
|
def _clean_precio(text):
|
|
m = re.search(r'([\d\.,]+)', text.replace('\xa0', '').replace(' ', ''))
|
|
if not m:
|
|
return None
|
|
try:
|
|
return int(m.group(1).replace(',', '').replace('.', '').rstrip('0') or '0')
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _extract_listing_id(href):
|
|
path = href.split('?')[0].split('#')[0].rstrip('/')
|
|
segments = [s for s in path.split('/') if s]
|
|
if not segments:
|
|
return None
|
|
last = segments[-1]
|
|
m = re.search(r'(\d{5,})', last)
|
|
if m:
|
|
return m.group(1)
|
|
if len(last) > 4:
|
|
return last
|
|
return None
|
|
|
|
|
|
def scrape_page(sess, url, cc):
|
|
backoffs = [3, 8, 20]
|
|
resp = None
|
|
for attempt, wait in enumerate(backoffs, start=1):
|
|
try:
|
|
resp = sess.get(url, timeout=35)
|
|
except requests.RequestException as exc:
|
|
log.warning("Red intento %d/3 [%s]: %s", attempt, url, exc)
|
|
if attempt < 3:
|
|
time.sleep(wait)
|
|
continue
|
|
return []
|
|
|
|
if resp.status_code in (403, 429):
|
|
log.warning("HTTP %d en %s — espera 45s", resp.status_code, url)
|
|
time.sleep(45)
|
|
if attempt >= 3:
|
|
return []
|
|
continue
|
|
|
|
if resp.status_code == 404:
|
|
log.info("404 — fin de paginacion en %s", url)
|
|
return None
|
|
|
|
if resp.status_code != 200:
|
|
log.warning("HTTP %d inesperado en %s", resp.status_code, url)
|
|
return []
|
|
|
|
break
|
|
|
|
if resp is None or not resp.text.strip():
|
|
return []
|
|
|
|
soup = BeautifulSoup(resp.text, 'lxml')
|
|
|
|
card_selectors = [
|
|
'div.listing-card',
|
|
'article.listing',
|
|
'div[class*="listing-card"]',
|
|
'div[class*="result-item"]',
|
|
'li[class*="listing-item"]',
|
|
'div[class*="ad-card"]',
|
|
'article[class*="property"]',
|
|
'div[class*="property-card"]',
|
|
]
|
|
cards = []
|
|
for sel in card_selectors:
|
|
found = soup.select(sel)
|
|
if found:
|
|
log.debug("Cards con '%s': %d", sel, len(found))
|
|
cards = found
|
|
break
|
|
|
|
if not cards:
|
|
links = soup.select('a[href*="/bienes-raices"]')
|
|
if links:
|
|
cards = links
|
|
|
|
if not cards:
|
|
log.info("Sin cards en %s", url)
|
|
return []
|
|
|
|
results = []
|
|
seen = set()
|
|
|
|
for card in cards:
|
|
try:
|
|
if card.name == 'a':
|
|
link_tag = card
|
|
else:
|
|
link_tag = (
|
|
card.select_one('a[href*="/bienes-raices"]')
|
|
or card.select_one('a[href*="/inmuebles"]')
|
|
or card.select_one('a')
|
|
)
|
|
|
|
href = (link_tag.get('href', '') if link_tag else '').strip()
|
|
if not href:
|
|
continue
|
|
|
|
full_url = href if href.startswith('http') else BASE_URL + href
|
|
|
|
listing_id = _extract_listing_id(href)
|
|
if not listing_id or listing_id in seen:
|
|
continue
|
|
seen.add(listing_id)
|
|
|
|
card_text = card.get_text(separator=' ', strip=True)
|
|
|
|
titulo = None
|
|
for sel in ('h2', 'h3', 'h1', '[class*="title"]', '[class*="titulo"]', '[class*="heading"]'):
|
|
el = card.select_one(sel)
|
|
if el:
|
|
t = el.get_text(strip=True)
|
|
if t:
|
|
titulo = t[:255]
|
|
break
|
|
if not titulo and link_tag:
|
|
titulo = link_tag.get_text(strip=True)[:255]
|
|
titulo = titulo or 'Sin titulo'
|
|
|
|
precio = None
|
|
moneda = 'USD'
|
|
for sel in ('[class*="price"]', '[class*="precio"]', '[class*="valor"]', '.amount', '.value'):
|
|
el = card.select_one(sel)
|
|
if el:
|
|
t = el.get_text(strip=True)
|
|
if 'USD' in t or '$' in t:
|
|
moneda = 'USD'
|
|
precio = _clean_precio(t)
|
|
if precio and precio > 100:
|
|
break
|
|
|
|
tipo = _detect_tipo(href + ' ' + card_text)
|
|
|
|
ubicacion = None
|
|
for sel in ('[class*="location"]', '[class*="ubicacion"]', '[class*="city"]',
|
|
'[class*="ciudad"]', '[class*="address"]', '[class*="region"]'):
|
|
el = card.select_one(sel)
|
|
if el:
|
|
t = el.get_text(strip=True)
|
|
if t:
|
|
ubicacion = t[:200]
|
|
break
|
|
|
|
superficie_m2 = None
|
|
m2 = re.search(r'(\d+(?:[,\.]\d+)?)\s*m[²2]', card_text, re.IGNORECASE)
|
|
if m2:
|
|
try:
|
|
superficie_m2 = float(m2.group(1).replace(',', '.'))
|
|
except ValueError:
|
|
pass
|
|
|
|
results.append({
|
|
'listing_id': listing_id,
|
|
'titulo': titulo,
|
|
'precio': precio,
|
|
'moneda': moneda,
|
|
'tipo': tipo,
|
|
'ubicacion': ubicacion,
|
|
'url': full_url,
|
|
'pais': cc,
|
|
'superficie_m2': superficie_m2,
|
|
})
|
|
except Exception as exc:
|
|
log.debug("Error parseando card: %s", exc)
|
|
|
|
log.info(" -> %d listings en %s", len(results), url)
|
|
return results
|
|
|
|
|
|
def main():
|
|
total_nuevos = 0
|
|
total_actualizados = 0
|
|
|
|
log.info("=== Scraper Encuentra24 CA Bienes Raices iniciado ===")
|
|
|
|
try:
|
|
conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME,
|
|
user=DB_USER, password=DB_PASS, connect_timeout=15)
|
|
log.info("Conectado a PostgreSQL: %s@%s/%s", DB_USER, DB_HOST, DB_NAME)
|
|
except Exception as exc:
|
|
raise RuntimeError(f"No se pudo conectar a PostgreSQL: {exc}") from exc
|
|
|
|
try:
|
|
create_table(conn)
|
|
sess = get_session()
|
|
|
|
for cc, country_slug in COUNTRIES:
|
|
log.info("== Pais: %s (%s) ==", country_slug.upper(), cc)
|
|
streak_sin_nuevos = 0
|
|
|
|
for page in range(1, MAX_PAGES + 1):
|
|
url = page_url(cc, country_slug, page)
|
|
log.info("[%s] Pagina %d/%d -> %s", cc, page, MAX_PAGES, url)
|
|
|
|
rows = scrape_page(sess, url, cc)
|
|
|
|
if rows is None:
|
|
log.info("[%s] Pagina %d devolvio 404 — fin para este pais", cc, page)
|
|
break
|
|
|
|
if not rows:
|
|
streak_sin_nuevos += 1
|
|
if streak_sin_nuevos >= NEW_STREAK:
|
|
log.info("[%s] %d paginas sin resultados — siguiente pais", cc, NEW_STREAK)
|
|
break
|
|
time.sleep(DELAY)
|
|
continue
|
|
|
|
nuevos, actualizados = upsert(conn, rows)
|
|
total_nuevos += nuevos
|
|
total_actualizados += actualizados
|
|
|
|
log.info("[%s] Pag %d: %d nuevos, %d actualizados (acum %d/%d)",
|
|
cc, page, nuevos, actualizados, total_nuevos, total_actualizados)
|
|
|
|
if nuevos == 0:
|
|
streak_sin_nuevos += 1
|
|
if streak_sin_nuevos >= NEW_STREAK:
|
|
log.info("[%s] %d paginas sin nuevos — siguiente pais", cc, NEW_STREAK)
|
|
break
|
|
else:
|
|
streak_sin_nuevos = 0
|
|
|
|
time.sleep(DELAY)
|
|
|
|
time.sleep(DELAY * 2)
|
|
|
|
finally:
|
|
try:
|
|
conn.close()
|
|
log.info("Conexion PostgreSQL cerrada")
|
|
except Exception:
|
|
pass
|
|
|
|
print(f"Finalizado: {total_nuevos} nuevos, {total_actualizados} actualizados")
|
|
log.info("=== Scraper finalizado. Nuevos: %d | Actualizados: %d ===",
|
|
total_nuevos, total_actualizados)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|