From b30f75e8371e7fa1893f702be483f1fcb914ffe2 Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 29 Jul 2026 06:49:31 +0200 Subject: [PATCH] feat: Portal Inmobiliario Chile scraper (630+ listings, MLC IDs JSON) --- inmuebles/portal_inmobiliario_cl/scraper.py | 209 ++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 inmuebles/portal_inmobiliario_cl/scraper.py diff --git a/inmuebles/portal_inmobiliario_cl/scraper.py b/inmuebles/portal_inmobiliario_cl/scraper.py new file mode 100644 index 0000000..227e4f2 --- /dev/null +++ b/inmuebles/portal_inmobiliario_cl/scraper.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +""" +Scraper Portal Inmobiliario Chile (MercadoLibre) -> postgres17-central / portal_inmobiliario_cl +URL: /venta/casas_Desde_{N}_NoIndex_True (step=48 por pagina) +ID: extraido del JSON embebido -> "id":"MLC{numero}" +""" +import os, re, time, logging +import requests +import psycopg2 +from psycopg2.extras import execute_values +from datetime import datetime + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +log = logging.getLogger('portal_inmobiliario') + +PROXY = 'socks5h://127.0.0.1:1090' +PROXIES = {'http': PROXY, 'https': PROXY} +DB_HOST = '100.75.240.87' +DB_NAME = 'portal_inmobiliario_cl' +DB_USER = 'pgadmin' +DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' +UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36' +BASE = 'https://www.portalinmobiliario.com' +PAGE1_URL = BASE + '/venta/casas' +PAGE_URL = BASE + '/venta/casas_Desde_{}_NoIndex_True' +STEP = 48 +DELAY = 1.2 +MAX_PAGES = int(os.environ.get('PI_MAX_PAGES', '40')) +NEW_STREAK = 2 + + +def get_session(): + s = requests.Session() + s.proxies.update(PROXIES) + s.headers.update({ + 'User-Agent': UA, + 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8', + 'Accept-Language': 'es-CL,es;q=0.9', + }) + return s + + +def get_conn(): + return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME, + user=DB_USER, password=DB_PASS, connect_timeout=10) + + +def create_table(conn): + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS listings ( + listing_id TEXT PRIMARY KEY, + titulo TEXT, + precio BIGINT, + moneda TEXT DEFAULT 'CLP', + tipo TEXT DEFAULT 'venta', + ubicacion TEXT, + url TEXT, + ambientes INT, + superficie_m2 DECIMAL, + scraped_at TIMESTAMP DEFAULT NOW() + ) + """) + conn.commit() + cur.close() + + +def upsert(conn, rows): + if not rows: + return 0 + cur = conn.cursor() + execute_values(cur, + """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at) + VALUES %s + ON CONFLICT (listing_id) DO UPDATE SET + titulo=EXCLUDED.titulo, precio=EXCLUDED.precio, scraped_at=EXCLUDED.scraped_at""", + [(r['listing_id'], r['titulo'], r['precio'], r.get('moneda', 'CLP'), + 'venta', r.get('ubicacion', 'Chile'), r['url'], datetime.utcnow()) for r in rows] + ) + n = cur.rowcount + conn.commit() + cur.close() + return n + + +def extract_listings(html): + results = [] + mlc_ids = list(dict.fromkeys(re.findall(r'"id"\s*:\s*"(MLC\d+)"', html))) + + for mlc_id in mlc_ids: + pos = html.find(f'"id":"{mlc_id}"') + if pos == -1: + pos = html.find(f'"id": "{mlc_id}"') + if pos == -1: + continue + + start = max(0, pos - 500) + end = min(len(html), pos + 1000) + ctx = html[start:end] + + titulo = '' + precio = None + moneda = 'CLP' + ubicacion = 'Chile' + url = BASE + '/' + mlc_id + + m = re.search(r'"title"\s*:\s*"([^"]{5,200})"', ctx) + if not m: + m = re.search(r'"name"\s*:\s*"([^"]{5,200})"', ctx) + if m: + titulo = m.group(1) + + m = re.search(r'"amount"\s*:\s*(\d+)', ctx) + if not m: + m = re.search(r'"price"\s*:\s*(\d+)', ctx) + if m: + precio = int(m.group(1)) + + m = re.search(r'"currency_id"\s*:\s*"([A-Z]{3})"', ctx) + if m: + moneda = m.group(1) + + m = re.search(r'"permalink"\s*:\s*"(https://www\.portalinmobiliario\.com/[^"]+)"', ctx) + if m: + url = m.group(1) + + m = re.search(r'"city_name"\s*:\s*"([^"]+)"', ctx) + if m: + ubicacion = m.group(1) + + results.append({ + 'listing_id': mlc_id, + 'titulo': titulo[:300] or 'Sin titulo', + 'precio': precio, + 'moneda': moneda, + 'ubicacion': ubicacion[:200], + 'url': url, + }) + + return results + + +def fetch_page(sess, page): + if page == 1: + url = PAGE1_URL + else: + url = PAGE_URL.format((page - 1) * STEP) + + for attempt in range(3): + try: + resp = sess.get(url, timeout=30) + if resp.status_code == 404: + return None + if resp.status_code in (403, 429): + log.warning('HTTP %d en %s', resp.status_code, url) + time.sleep(30) + continue + resp.raise_for_status() + break + except Exception as e: + if attempt == 2: + log.error('Error %s: %s', url, e) + return [] + time.sleep([2, 5, 15][attempt]) + + listings = extract_listings(resp.text) + log.info('Pagina %d (offset=%d): %d listings de %s', page, (page-1)*STEP, len(listings), url) + return listings + + +def main(): + log.info('=== Iniciando scraper Portal Inmobiliario Chile ===') + sess = get_session() + conn = get_conn() + create_table(conn) + + total_new = 0 + streak = 0 + + for page in range(1, MAX_PAGES + 1): + rows = fetch_page(sess, page) + if rows is None: + log.info('Pagina %d: fin de paginacion', page) + break + if not rows: + streak += 1 + log.info('Pagina %d: sin listings (racha %d/%d)', page, streak, NEW_STREAK) + if streak >= NEW_STREAK: + break + time.sleep(DELAY) + continue + + n = upsert(conn, rows) + streak = 0 if n > 0 else streak + 1 + total_new += n + log.info('Pagina %d: %d upserted', page, n) + + if streak >= NEW_STREAK: + log.info('Deteniendo: %d paginas sin nuevos', streak) + break + time.sleep(DELAY) + + conn.close() + print(f'Finalizado: {total_new} nuevos, 0 actualizados') + log.info('Finalizado: %d nuevos, 0 actualizados', total_new) + + +if __name__ == '__main__': + main()