From 10cb6a6bc5f8fa507cfbffe2b812f1285b4096b5 Mon Sep 17 00:00:00 2001 From: claude Date: Sun, 2 Aug 2026 18:05:19 +0200 Subject: [PATCH] feat: scraper ArgenProp AR via sitemaps geograficos --- inmuebles/argenprop_ar/scraper.py | 352 ++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 inmuebles/argenprop_ar/scraper.py diff --git a/inmuebles/argenprop_ar/scraper.py b/inmuebles/argenprop_ar/scraper.py new file mode 100644 index 0000000..8f9868e --- /dev/null +++ b/inmuebles/argenprop_ar/scraper.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +""" +Scraper ArgenProp.com.ar – Bienes Raices Argentina +Estrategia: iterar URLs del sitemap de listing (por barrio/tipo) y extraer +listing cards con datos SSR completos en atributos del . +DB: argenprop_ar +Checkpoint: /opt/scrapers/argenprop_ar/state.json + {"sitemap_idx": N, "url_idx": M, "total_new": X} +""" +import os +import re +import time +import json +import gzip +import logging +from datetime import datetime + +import requests +from bs4 import BeautifulSoup +import psycopg2 +from psycopg2.extras import execute_values + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', +) +log = logging.getLogger('argenprop') + +PROXY = 'socks5h://127.0.0.1:1090' +PROXIES = {'http': PROXY, 'https': PROXY} +UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36' +BASE = 'https://www.argenprop.com' +DELAY = 1.5 + +DB_HOST = '100.75.240.87' +DB_PORT = 5432 +DB_NAME = 'argenprop_ar' +DB_USER = 'pgadmin' +DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' + +DAILY = bool(os.environ.get('ARGENPROP_DAILY')) # True en modo daily, False en backfill +MAX_SITEMAPS = int(os.environ.get('MAX_SITEMAPS', '99')) # cuantos sub-sitemaps procesar + +STATE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'state.json') + +# Sub-sitemaps de listing (se descubren dinamicamente desde el índice) +SITEMAP_INDEX = 'https://www.argenprop.com/sitemap.xml' + + +def load_state(): + if os.path.exists(STATE_FILE): + with open(STATE_FILE) as f: + st = json.load(f) + log.info('Checkpoint: sitemap_idx=%d url_idx=%d total_new=%d', + st['sitemap_idx'], st['url_idx'], st.get('total_new', 0)) + return st + return {'sitemap_idx': 0, 'url_idx': 0, 'total_new': 0} + + +def save_state(si, ui, total_new): + with open(STATE_FILE, 'w') as f: + json.dump({'sitemap_idx': si, 'url_idx': ui, 'total_new': total_new, + 'ts': datetime.utcnow().isoformat()}, f) + + +def get_session(): + sess = requests.Session() + sess.proxies.update(PROXIES) + sess.headers.update({ + 'User-Agent': UA, + 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8', + 'Accept-Language': 'es-AR,es;q=0.9', + 'Accept-Encoding': 'gzip, deflate', + 'Connection': 'keep-alive', + }) + return sess + + +def get_conn(): + return psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, + user=DB_USER, password=DB_PASS, connect_timeout=15) + + +def create_db(conn): + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS listings ( + listing_id TEXT PRIMARY KEY, + titulo TEXT, + precio BIGINT, + moneda TEXT DEFAULT 'USD', + tipo TEXT, + ubicacion TEXT, + url TEXT UNIQUE, + superficie_m2 DECIMAL, + ambientes INT, + scraped_at TIMESTAMP DEFAULT NOW() + ) + """) + conn.commit() + cur.close() + + +def upsert(conn, rows): + if not rows: + return 0 + deduped = list({r['listing_id']: r for r in rows}.values()) + cur = conn.cursor() + ids = [r['listing_id'] for r in deduped] + cur.execute('SELECT listing_id FROM listings WHERE listing_id = ANY(%s)', (ids,)) + existing = {rec[0] for rec in cur.fetchall()} + nuevos = sum(1 for r in deduped if r['listing_id'] not in existing) + now = datetime.utcnow() + execute_values(cur, """ + INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, + url, superficie_m2, ambientes, scraped_at) + VALUES %s + ON CONFLICT (listing_id) DO UPDATE SET + titulo = EXCLUDED.titulo, + precio = EXCLUDED.precio, + moneda = EXCLUDED.moneda, + tipo = EXCLUDED.tipo, + ubicacion = EXCLUDED.ubicacion, + url = EXCLUDED.url, + superficie_m2 = EXCLUDED.superficie_m2, + ambientes = EXCLUDED.ambientes, + scraped_at = EXCLUDED.scraped_at + """, [(r['listing_id'], r.get('titulo'), r.get('precio'), r.get('moneda', 'USD'), + r.get('tipo'), r.get('ubicacion'), r.get('url'), + r.get('superficie_m2'), r.get('ambientes'), now) for r in deduped]) + conn.commit() + cur.close() + return nuevos + + +def fetch_gz(sess, url, retries=3): + """Descarga un .xml.gz y retorna lista de URLs.""" + for attempt in range(retries): + try: + r = sess.get(url, timeout=30) + if r.status_code in (404, 410): + return [] + r.raise_for_status() + try: + content = gzip.decompress(r.content).decode('utf-8', errors='replace') + except Exception: + content = r.text + return re.findall(r'(.*?)', content) + except Exception as e: + log.warning('sitemap %s intento %d: %s', url, attempt+1, e) + if attempt < retries - 1: + time.sleep([5, 15][attempt % 2]) + return [] + + +def fetch_page(sess, url, retries=4): + """Descarga una página de listado y extrae listing items.""" + backoffs = [2, 5, 15, 30] + for attempt in range(retries): + try: + r = sess.get(url, timeout=35) + # HTTP 202 = AWS WAF challenge, reintentar + if r.status_code == 202: + log.debug('HTTP 202 en %s (intento %d)', url, attempt+1) + if attempt < retries - 1: + time.sleep(backoffs[attempt]) + continue + return [] + if r.status_code in (403, 429): + log.warning('HTTP %d en %s', r.status_code, url) + time.sleep(45) + continue + if r.status_code in (404, 410): + return [] + if r.status_code != 200: + log.warning('HTTP %d en %s', r.status_code, url) + return [] + break + except requests.RequestException as e: + log.warning('Red %s intento %d: %s', url, attempt+1, e) + if attempt < retries - 1: + time.sleep(backoffs[attempt]) + else: + return [] + + soup = BeautifulSoup(r.text, 'lxml') + items = soup.find_all('div', class_='listing__item') + if not items: + return [] + + results = [] + seen = set() + + for item in items: + try: + card = item.find('a', class_='card') + if not card: + continue + + listing_id = card.get('idaviso', '').strip() + if not listing_id or listing_id in seen: + continue + seen.add(listing_id) + + href = card.get('href', '') + full_url = BASE + href if href.startswith('/') else href + + # Precio en USD desde atributo montonormalizado + try: + precio = int(card.get('montonormalizado', 0) or 0) + except (ValueError, TypeError): + precio = None + moneda = 'USD' # argenprop lista en USD + + # Dormitorios + try: + ambientes = int(card.get('dormitorios', 0) or 0) or None + except (ValueError, TypeError): + ambientes = None + + # Tipo de inmueble desde la URL + tipo = 'venta' + if 'arriendo' in href.lower() or 'alquiler' in href.lower(): + tipo = 'arriendo' + + # Tipo de propiedad + tipo_prop = None + for t in ('casa', 'departamento', 'terreno', 'campo', 'cochera', 'oficina', 'local', 'ph'): + if t in href.lower(): + tipo_prop = t + break + + # Ubicación desde URL: /casa-en-venta-en-palermo-... → palermo + loc_m = re.search(r'-en-(?:venta|alquiler|arriendo)-en-([a-z\-]+?)(?:-\d|-\-|$)', href) + ubicacion = loc_m.group(1).replace('-', ' ').title() if loc_m else None + + # Título y m2 del texto del card + card_text = card.get_text(separator=' ', strip=True) + titulo_m = re.search(r'((?:Casa|Departamento|Terreno|Campo|Oficina|Local|PH|Cochera)[^|]{5,80})', card_text, re.I) + titulo = titulo_m.group(1).strip()[:255] if titulo_m else card_text[:100] + + m2_m = re.search(r'(\d+(?:[,\.]\d+)?)\s*m[2²]', card_text, re.I) + superficie_m2 = None + if m2_m: + try: + superficie_m2 = float(m2_m.group(1).replace(',', '.')) + except ValueError: + pass + + results.append({ + 'listing_id': listing_id, + 'titulo': titulo or 'Sin titulo', + 'precio': precio or None, + 'moneda': moneda, + 'tipo': tipo_prop or tipo, + 'ubicacion': ubicacion, + 'url': full_url, + 'superficie_m2': superficie_m2, + 'ambientes': ambientes, + }) + except Exception as e: + log.debug('Error card: %s', e) + + return results + + +def get_listing_sitemaps(sess): + """Retorna lista de sub-sitemaps de tipo listing.""" + r = sess.get(SITEMAP_INDEX, timeout=20) + all_sms = re.findall(r'(.*?)', r.text) + # Solo los sitemaps de listing (no ficha) + listing_sms = [u for u in all_sms if 'sitemap-listing' in u] + log.info('Sitemaps de listing: %d', len(listing_sms)) + return listing_sms + + +def main(): + state = load_state() + si_start = state['sitemap_idx'] + ui_start = state['url_idx'] + total_new = state.get('total_new', 0) + + log.info('=== Scraper ArgenProp AR — modo %s ===', + 'DAILY' if DAILY else 'BACKFILL') + log.info('Desde sitemap %d url %d', si_start, ui_start) + + sess = get_session() + conn = get_conn() + create_db(conn) + + # Descubrir sitemaps de listing + listing_sms = get_listing_sitemaps(sess) + listing_sms = listing_sms[:MAX_SITEMAPS] + + for si in range(si_start, len(listing_sms)): + sm_url = listing_sms[si] + log.info('== Sitemap %d/%d: %s ==', si+1, len(listing_sms), sm_url) + + # Descargar el índice del sub-sitemap (puede tener sub-sub-sitemaps) + sm_urls = fetch_gz(sess, sm_url) + if not sm_urls: + # Intentar sin gz + sm_urls = [sm_url.replace('.gz', '')] + log.info(' sub-sitemaps/urls: %d', len(sm_urls)) + + # Si son sub-sitemaps (terminan en .xml.gz), expandir + final_urls = [] + for u in sm_urls: + if u.endswith('.xml.gz') or u.endswith('.xml'): + inner = fetch_gz(sess, u) + final_urls.extend(inner) + time.sleep(0.5) + else: + final_urls.append(u) + + log.info(' URLs totales a procesar: %d', len(final_urls)) + + ui_ini = ui_start if si == si_start else 0 + + for ui in range(ui_ini, len(final_urls)): + page_url = final_urls[ui] + rows = fetch_page(sess, page_url) + + if rows: + n = upsert(conn, rows) + total_new += n + if n > 0: + log.info('[%d/%d] %s: %d nuevos (acum %d)', + ui+1, len(final_urls), page_url[-60:], n, total_new) + else: + log.debug('[%d/%d] %s: sin resultados', ui+1, len(final_urls), page_url[-60:]) + + save_state(si, ui + 1, total_new) + time.sleep(DELAY) + + # En modo daily parar temprano si no hay nuevos (ya cubierto) + if DAILY and ui > 200 and total_new == 0: + log.info('Daily: 200 URLs sin nuevos — deteniendo') + break + + # Avanzar al siguiente sitemap + save_state(si + 1, 0, total_new) + time.sleep(DELAY * 2) + + conn.close() + log.info('=== Finalizado: %d nuevos ===', total_new) + print(f'Finalizado: {total_new} nuevos') + + +if __name__ == '__main__': + main()