284 lines
9.0 KiB
Python
284 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Scraper Yapo.cl – Bienes Raices Chile (Schibsted)
|
||
Iteracion por subcategorias: ~30 tiles SSR por subcategoria (~150 unicos/dia)
|
||
DB: yapo_cl
|
||
"""
|
||
|
||
import re
|
||
import time
|
||
import logging
|
||
from datetime import datetime
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
import psycopg2
|
||
from psycopg2.extras import execute_values
|
||
|
||
PROXY = 'socks5h://127.0.0.1:1090'
|
||
PROXIES = {'http': PROXY, 'https': PROXY}
|
||
|
||
DB_HOST = '100.75.240.87'
|
||
DB_PORT = 5432
|
||
DB_NAME = 'yapo_cl'
|
||
DB_USER = 'pgadmin'
|
||
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'
|
||
|
||
UA = (
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
||
'Chrome/124.0.0.0 Safari/537.36'
|
||
)
|
||
|
||
DELAY = 1.5
|
||
BASE_URL = 'https://www.yapo.cl'
|
||
|
||
SUBCATEGORIES = [
|
||
'/bienes-raices-venta-de-propiedades',
|
||
'/bienes-raices-venta-de-propiedades-casas',
|
||
'/bienes-raices-venta-de-propiedades-apartamentos',
|
||
'/bienes-raices-alquiler',
|
||
'/bienes-raices-alquiler-casas',
|
||
'/bienes-raices-alquiler-apartamentos',
|
||
'/bienes-raices-proyectos-nuevos',
|
||
]
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s [%(levelname)s] %(message)s',
|
||
datefmt='%Y-%m-%d %H:%M:%S',
|
||
)
|
||
log = logging.getLogger(__name__)
|
||
|
||
|
||
def get_session():
|
||
sess = requests.Session()
|
||
sess.proxies.update(PROXIES)
|
||
sess.headers.update({
|
||
'User-Agent': UA,
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||
'Accept-Language': 'es-CL,es;q=0.9,en;q=0.8',
|
||
'Connection': 'keep-alive',
|
||
'Cache-Control': 'no-cache',
|
||
})
|
||
return sess
|
||
|
||
|
||
def create_table(conn):
|
||
cur = conn.cursor()
|
||
cur.execute("""
|
||
CREATE TABLE IF NOT EXISTS listings (
|
||
listing_id TEXT PRIMARY KEY,
|
||
titulo TEXT,
|
||
precio DECIMAL,
|
||
moneda TEXT DEFAULT 'UF',
|
||
tipo TEXT,
|
||
ubicacion TEXT,
|
||
url TEXT UNIQUE,
|
||
superficie_m2 DECIMAL,
|
||
ambientes INT,
|
||
scraped_at TIMESTAMP DEFAULT NOW()
|
||
)
|
||
""")
|
||
conn.commit()
|
||
cur.close()
|
||
log.info("Tabla listings verificada/creada OK")
|
||
|
||
|
||
def upsert(conn, rows):
|
||
if not rows:
|
||
return 0, 0
|
||
deduped = list({r['listing_id']: r for r in rows}.values())
|
||
cur = conn.cursor()
|
||
ids = [r['listing_id'] for r in deduped]
|
||
cur.execute('SELECT listing_id FROM listings WHERE listing_id = ANY(%s)', (ids,))
|
||
existing = {rec[0] for rec in cur.fetchall()}
|
||
nuevos = sum(1 for r in deduped if r['listing_id'] not in existing)
|
||
actualizados = len(deduped) - nuevos
|
||
now = datetime.utcnow()
|
||
execute_values(cur, """
|
||
INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, superficie_m2, ambientes, scraped_at)
|
||
VALUES %s
|
||
ON CONFLICT (listing_id) DO UPDATE SET
|
||
titulo = EXCLUDED.titulo,
|
||
precio = EXCLUDED.precio,
|
||
moneda = EXCLUDED.moneda,
|
||
tipo = EXCLUDED.tipo,
|
||
ubicacion = EXCLUDED.ubicacion,
|
||
url = EXCLUDED.url,
|
||
superficie_m2 = EXCLUDED.superficie_m2,
|
||
ambientes = EXCLUDED.ambientes,
|
||
scraped_at = EXCLUDED.scraped_at
|
||
""", [
|
||
(r['listing_id'], r.get('titulo'), r.get('precio'), r.get('moneda', 'UF'),
|
||
r.get('tipo'), r.get('ubicacion'), r.get('url'),
|
||
r.get('superficie_m2'), r.get('ambientes'), now)
|
||
for r in deduped
|
||
])
|
||
conn.commit()
|
||
cur.close()
|
||
return nuevos, actualizados
|
||
|
||
|
||
def _detect_tipo(href):
|
||
if 'alquiler' in href or 'arriendo' in href or 'renta' in href:
|
||
return 'arriendo'
|
||
if 'venta' in href or 'vender' in href:
|
||
return 'venta'
|
||
return None
|
||
|
||
|
||
def scrape_subcategory(sess, path):
|
||
url = BASE_URL + path
|
||
for attempt in range(3):
|
||
try:
|
||
r = sess.get(url, timeout=30)
|
||
if r.status_code in (403, 429):
|
||
log.warning("HTTP %d en %s — esperando 30s", r.status_code, url)
|
||
time.sleep(30)
|
||
continue
|
||
if r.status_code == 404:
|
||
log.info("404 en %s — ignorando", url)
|
||
return []
|
||
r.raise_for_status()
|
||
break
|
||
except requests.RequestException as exc:
|
||
log.warning("Intento %d/3 fallido: %s", attempt + 1, exc)
|
||
if attempt < 2:
|
||
time.sleep([3, 8][attempt])
|
||
else:
|
||
return []
|
||
|
||
soup = BeautifulSoup(r.text, 'lxml')
|
||
tiles = soup.select('.d3-ad-tile')
|
||
if not tiles:
|
||
log.info("Sin tiles en %s", path)
|
||
return []
|
||
|
||
results = []
|
||
seen = set()
|
||
|
||
for tile in tiles:
|
||
try:
|
||
fav = tile.select_one('[data-adid]')
|
||
if not fav:
|
||
continue
|
||
listing_id = fav.get('data-adid', '').strip()
|
||
if not listing_id or listing_id in seen:
|
||
continue
|
||
seen.add(listing_id)
|
||
|
||
precio_uf = None
|
||
raw_price = fav.get('data-price', '')
|
||
if raw_price:
|
||
try:
|
||
precio_uf = float(raw_price)
|
||
except ValueError:
|
||
pass
|
||
|
||
links = tile.select('a[href]')
|
||
href = ''
|
||
for lnk in links:
|
||
h = lnk.get('href', '')
|
||
if listing_id in h and '/bienes-raices' in h:
|
||
href = h
|
||
break
|
||
if not href:
|
||
for lnk in links:
|
||
h = lnk.get('href', '')
|
||
if h and h != '#' and '/bienes-raices' in h:
|
||
href = h
|
||
break
|
||
|
||
full_url = BASE_URL + href if href.startswith('/') else href
|
||
|
||
tipo = _detect_tipo(href)
|
||
|
||
title_el = tile.select_one('h2, h3, [class*=title]')
|
||
titulo = title_el.get_text(strip=True)[:255] if title_el else None
|
||
|
||
desc_el = tile.select_one('.d3-ad-tile__description, [class*=description]')
|
||
if not titulo and desc_el:
|
||
titulo = desc_el.get_text(strip=True)[:255]
|
||
|
||
card_text = tile.get_text(separator=' ', strip=True)
|
||
|
||
ubicacion = None
|
||
loc_el = tile.select_one('[class*=location], [class*=ubicacion], [class*=address]')
|
||
if loc_el:
|
||
ubicacion = loc_el.get_text(strip=True)[:200]
|
||
else:
|
||
m_loc = re.search(r'(Santiago|Providencia|Las Condes|Ñuñoa|Vitacura|Maipú|La Florida|Viña del Mar|Valparaíso|Concepción|Antofagasta|La Serena|Rancagua|Temuco|Puerto Montt|Valdivia|Calama|Iquique|Arica)', card_text, re.IGNORECASE)
|
||
if m_loc:
|
||
ubicacion = m_loc.group(0)
|
||
|
||
superficie_m2 = None
|
||
m2 = re.search(r'(\d+(?:[,.]\d+)?)\s*m[²2]', card_text, re.IGNORECASE)
|
||
if m2:
|
||
try:
|
||
superficie_m2 = float(m2.group(1).replace(',', '.'))
|
||
except ValueError:
|
||
pass
|
||
|
||
ambientes = None
|
||
m_amb = re.search(r'(\d+)\s*(?:dorm|hab|amb)', card_text, re.IGNORECASE)
|
||
if m_amb:
|
||
try:
|
||
ambientes = int(m_amb.group(1))
|
||
except ValueError:
|
||
pass
|
||
|
||
results.append({
|
||
'listing_id': listing_id,
|
||
'titulo': titulo or 'Sin titulo',
|
||
'precio': precio_uf,
|
||
'moneda': 'UF',
|
||
'tipo': tipo,
|
||
'ubicacion': ubicacion,
|
||
'url': full_url if full_url != BASE_URL else None,
|
||
'superficie_m2': superficie_m2,
|
||
'ambientes': ambientes,
|
||
})
|
||
except Exception as exc:
|
||
log.debug("Error parseando tile: %s", exc)
|
||
|
||
log.info("%s: %d listings", path, len(results))
|
||
return results
|
||
|
||
|
||
def main():
|
||
log.info("=== Scraper Yapo.cl iniciado ===")
|
||
|
||
try:
|
||
conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME,
|
||
user=DB_USER, password=DB_PASS, connect_timeout=15)
|
||
log.info("Conexion DB OK (%s@%s/%s)", DB_USER, DB_HOST, DB_NAME)
|
||
except Exception as exc:
|
||
raise RuntimeError(f"Error conectando a PostgreSQL: {exc}") from exc
|
||
|
||
try:
|
||
create_table(conn)
|
||
sess = get_session()
|
||
total_nuevos = 0
|
||
total_actualizados = 0
|
||
|
||
for path in SUBCATEGORIES:
|
||
rows = scrape_subcategory(sess, path)
|
||
if rows:
|
||
nuevos, actualizados = upsert(conn, rows)
|
||
total_nuevos += nuevos
|
||
total_actualizados += actualizados
|
||
log.info("[%s] %d nuevos, %d actualizados", path, nuevos, actualizados)
|
||
time.sleep(DELAY)
|
||
|
||
finally:
|
||
conn.close()
|
||
log.info("Conexion DB cerrada")
|
||
|
||
print(f"Finalizado: {total_nuevos} nuevos, {total_actualizados} actualizados")
|
||
log.info("=== Scraper Yapo.cl finalizado ===")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|