#!/usr/bin/env python3 """ Backfill scraper Portal Inmobiliario Chile (MercadoLibre) Checkpoint: /opt/scrapers/portal_inmobiliario_cl/backfill_state.json {"page": N, "total_new": M} Para reiniciar desde cero: borrar el archivo de checkpoint. """ import os, re, time, logging, json import requests import psycopg2 from psycopg2.extras import execute_values from datetime import datetime logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', ) log = logging.getLogger('portal_inmobiliario_backfill') PROXY = 'socks5h://127.0.0.1:1090' PROXIES = {'http': PROXY, 'https': PROXY} DB_HOST = '100.75.240.87' DB_NAME = 'portal_inmobiliario_cl' DB_USER = 'pgadmin' DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36' BASE = 'https://www.portalinmobiliario.com' PAGE1_URL = BASE + '/venta/casas' PAGE_URL = BASE + '/venta/casas_Desde_{}_NoIndex_True' STEP = 48 DELAY = 1.2 MAX_PAGES = int(os.environ.get('BF_MAX_PAGES', '400')) MAX_STREAK = int(os.environ.get('BF_STREAK', '8')) STATE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'backfill_state.json') def load_state(): if os.path.exists(STATE_FILE): with open(STATE_FILE) as f: st = json.load(f) log.info('Checkpoint: pagina=%d total_new=%d', st['page'], st.get('total_new', 0)) return st return {'page': 1, 'total_new': 0} def save_state(page, total_new): with open(STATE_FILE, 'w') as f: json.dump({'page': page, 'total_new': total_new, 'ts': datetime.utcnow().isoformat()}, f) def get_session(): s = requests.Session() s.proxies.update(PROXIES) s.headers.update({'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8', 'Accept-Language': 'es-CL,es;q=0.9'}) return s def get_conn(): return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME, user=DB_USER, password=DB_PASS, connect_timeout=10) def create_table(conn): cur = conn.cursor() cur.execute(""" CREATE TABLE IF NOT EXISTS listings ( listing_id TEXT PRIMARY KEY, titulo TEXT, precio BIGINT, moneda TEXT DEFAULT 'CLP', tipo TEXT DEFAULT 'venta', ubicacion TEXT, url TEXT, ambientes INT, superficie_m2 DECIMAL, scraped_at TIMESTAMP DEFAULT NOW() ) """) conn.commit() cur.close() def upsert(conn, rows): if not rows: return 0 deduped = list({r['listing_id']: r for r in rows}.values()) cur = conn.cursor() execute_values(cur, """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at) VALUES %s ON CONFLICT (listing_id) DO UPDATE SET titulo=EXCLUDED.titulo, precio=EXCLUDED.precio, scraped_at=EXCLUDED.scraped_at""", [(r['listing_id'], r['titulo'], r['precio'], r.get('moneda', 'CLP'), 'venta', r.get('ubicacion', 'Chile'), r['url'], datetime.utcnow()) for r in deduped] ) n = cur.rowcount conn.commit() cur.close() return n def extract_listings(html): results = [] mlc_ids = list(dict.fromkeys(re.findall(r'"id"\s*:\s*"(MLC\d+)"', html))) for mlc_id in mlc_ids: pos = html.find(f'"id":"{mlc_id}"') if pos == -1: pos = html.find(f'"id": "{mlc_id}"') if pos == -1: continue start = max(0, pos - 500) end = min(len(html), pos + 1000) ctx = html[start:end] titulo = '' precio = None moneda = 'CLP' ubicacion = 'Chile' url = BASE + '/' + mlc_id m = re.search(r'"title"\s*:\s*"([^"]{5,200})"', ctx) if not m: m = re.search(r'"name"\s*:\s*"([^"]{5,200})"', ctx) if m: titulo = m.group(1) m = re.search(r'"amount"\s*:\s*(\d+)', ctx) if not m: m = re.search(r'"price"\s*:\s*(\d+)', ctx) if m: precio = int(m.group(1)) m = re.search(r'"currency_id"\s*:\s*"([A-Z]{3})"', ctx) if m: moneda = m.group(1) m = re.search(r'"permalink"\s*:\s*"(https://www\.portalinmobiliario\.com/[^"]+)"', ctx) if m: url = m.group(1) m = re.search(r'"city_name"\s*:\s*"([^"]+)"', ctx) if m: ubicacion = m.group(1) results.append({'listing_id': mlc_id, 'titulo': titulo[:300] or 'Sin titulo', 'precio': precio, 'moneda': moneda, 'ubicacion': ubicacion[:200], 'url': url}) return results def fetch_page(sess, page): url = PAGE1_URL if page == 1 else PAGE_URL.format((page - 1) * STEP) for attempt in range(3): try: resp = sess.get(url, timeout=30) if resp.status_code == 404: return None if resp.status_code in (403, 429): log.warning('HTTP %d en %s', resp.status_code, url) time.sleep(60) continue resp.raise_for_status() break except Exception as e: if attempt == 2: log.error('Error %s: %s', url, e) return [] time.sleep([2, 5, 15][attempt]) listings = extract_listings(resp.text) log.info('Pagina %d (offset=%d): %d listings', page, (page - 1) * STEP, len(listings)) return listings def main(): state = load_state() start_page = state['page'] total_new = state.get('total_new', 0) log.info('=== Backfill Portal Inmobiliario CL — desde pagina %d ===', start_page) sess = get_session() conn = get_conn() create_table(conn) streak = 0 last_page = start_page for page in range(start_page, MAX_PAGES + 1): last_page = page rows = fetch_page(sess, page) if rows is None: log.info('Pagina %d: 404 — fin de paginacion', page) save_state(page + 1, total_new) break if not rows: streak += 1 save_state(page + 1, total_new) log.info('Pagina %d: sin listings (racha %d/%d)', page, streak, MAX_STREAK) if streak >= MAX_STREAK: log.info('Deteniendo: %d paginas consecutivas sin listings', streak) break time.sleep(DELAY) continue n = upsert(conn, rows) streak = 0 if n > 0 else streak + 1 total_new += n save_state(page + 1, total_new) log.info('Pagina %d: %d nuevos (acumulado %d)', page, n, total_new) if streak >= MAX_STREAK: log.info('Deteniendo: %d paginas sin nuevos', streak) break time.sleep(DELAY) conn.close() log.info('=== Backfill finalizado: %d nuevos (ultima pagina: %d) ===', total_new, last_page) print(f'Finalizado: {total_new} nuevos') if __name__ == '__main__': main()