diff --git a/inmuebles/corotos_do/backfill.py b/inmuebles/corotos_do/backfill.py new file mode 100644 index 0000000..89e44b4 --- /dev/null +++ b/inmuebles/corotos_do/backfill.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +""" +Backfill scraper Corotos.com.do Republica Dominicana +Checkpoint: /opt/scrapers/corotos_do/backfill_state.json + {"page": N, "total_new": M} +Para reiniciar desde cero: borrar el archivo de checkpoint. +""" +import os, re, time, logging, json +import requests +from bs4 import BeautifulSoup +import psycopg2 +from psycopg2.extras import execute_values +from datetime import datetime + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', +) +log = logging.getLogger('corotos_backfill') + +PROXY = 'socks5h://127.0.0.1:1090' +PROXIES = {'http': PROXY, 'https': PROXY} +DB_HOST = '100.75.240.87' +DB_NAME = 'corotos_do' +DB_USER = 'pgadmin' +DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' +UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36' +BASE = 'https://www.corotos.com.do' +LIST_URL = BASE + '/anuncios?categoria=bienes-raices&page={}' +DELAY = 1.0 +MAX_PAGES = int(os.environ.get('BF_MAX_PAGES', '200')) +MAX_STREAK = int(os.environ.get('BF_STREAK', '6')) + +STATE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'backfill_state.json') + + +def load_state(): + if os.path.exists(STATE_FILE): + with open(STATE_FILE) as f: + st = json.load(f) + log.info('Checkpoint: pagina=%d total_new=%d', st['page'], st.get('total_new', 0)) + return st + return {'page': 1, 'total_new': 0} + + +def save_state(page, total_new): + with open(STATE_FILE, 'w') as f: + json.dump({'page': page, 'total_new': total_new, 'ts': datetime.utcnow().isoformat()}, f) + + +def get_session(): + s = requests.Session() + s.proxies.update(PROXIES) + s.headers.update({'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8', + 'Accept-Language': 'es-DO,es;q=0.9'}) + return s + + +def get_conn(): + return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME, + user=DB_USER, password=DB_PASS, connect_timeout=10) + + +def create_table(conn): + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS listings ( + listing_id TEXT PRIMARY KEY, + titulo TEXT, + precio BIGINT, + moneda TEXT DEFAULT 'DOP', + tipo TEXT, + ubicacion TEXT, + url TEXT UNIQUE, + superficie_m2 DECIMAL, + scraped_at TIMESTAMP DEFAULT NOW() + ) + """) + conn.commit() + cur.close() + + +def upsert(conn, rows): + if not rows: + return 0 + deduped = list({r['listing_id']: r for r in rows}.values()) + cur = conn.cursor() + execute_values(cur, + """INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at) + VALUES %s + ON CONFLICT (listing_id) DO UPDATE SET + titulo=EXCLUDED.titulo, precio=EXCLUDED.precio, scraped_at=EXCLUDED.scraped_at""", + [(r['listing_id'], r['titulo'], r['precio'], r.get('moneda', 'DOP'), + r['tipo'], r['ubicacion'], r['url'], datetime.utcnow()) for r in deduped] + ) + n = cur.rowcount + conn.commit() + cur.close() + return n + + +def fetch_page(sess, page): + url = LIST_URL.format(page) + for attempt in range(3): + try: + resp = sess.get(url, timeout=30) + if resp.status_code == 404: + return None + if resp.status_code in (403, 429): + log.warning('HTTP %d en %s', resp.status_code, url) + time.sleep(60) + continue + resp.raise_for_status() + break + except Exception as e: + if attempt == 2: + log.error('Error %s: %s', url, e) + return [] + time.sleep([2, 5, 15][attempt]) + + soup = BeautifulSoup(resp.text, 'lxml') + results = [] + seen = set() + + cards_with_id = soup.find_all(attrs={'data-adid': True}) + for card in cards_with_id: + try: + listing_id = card.get('data-adid', '').strip() + if not listing_id or listing_id in seen: + continue + seen.add(listing_id) + link_tag = card.find('a', href=re.compile(r'/anuncio/')) + href = link_tag['href'] if link_tag else f'/anuncio/{listing_id}' + url_full = BASE + href if href.startswith('/') else href + tipo = 'alquiler' if 'alquiler' in href.lower() or 'alquiler' in card.get_text().lower() else 'venta' + titulo_tag = card.find(['h2', 'h3', 'a', 'p']) + titulo = titulo_tag.get_text(strip=True)[:300] if titulo_tag else 'Sin titulo' + precio = None + moneda = 'DOP' + precio_text = card.get_text() + if 'USD' in precio_text: + moneda = 'USD' + nums = re.findall(r'[\d,]+', precio_text) + for n_str in nums: + try: + val = int(n_str.replace(',', '')) + if val > 999: + precio = val + break + except ValueError: + pass + loc_tag = card.find(string=re.compile( + r'Santo Domingo|Santiago|La Romana|Punta Cana|Puerto Plata|Hig.ey|San Pedro|Bavaro', re.I)) + ubicacion = str(loc_tag).strip()[:200] if loc_tag else 'Rep. Dominicana' + results.append({'listing_id': listing_id, 'titulo': titulo, + 'precio': precio, 'moneda': moneda, + 'tipo': tipo, 'ubicacion': ubicacion, 'url': url_full}) + except Exception as e: + log.debug('Error card: %s', e) + + if not results: + for link in soup.find_all('a', href=re.compile(r'/anuncio/inmuebles')): + href = link.get('href', '') + slug = href.rstrip('/').split('/')[-1] + if slug and slug not in seen: + seen.add(slug) + titulo_tag = link.find(['h2', 'h3', 'span']) + titulo = titulo_tag.get_text(strip=True) if titulo_tag else slug.replace('-', ' ')[:200] + results.append({'listing_id': slug[:50], 'titulo': titulo[:300], + 'precio': None, 'moneda': 'DOP', + 'tipo': 'alquiler' if 'alquiler' in href.lower() else 'venta', + 'ubicacion': 'Rep. Dominicana', + 'url': BASE + href if href.startswith('/') else href}) + + log.info('Pagina %d: %d resultados', page, len(results)) + return results + + +def main(): + state = load_state() + start_page = state['page'] + total_new = state.get('total_new', 0) + log.info('=== Backfill Corotos DO — desde pagina %d ===', start_page) + + sess = get_session() + conn = get_conn() + create_table(conn) + + streak = 0 + last_page = start_page + + for page in range(start_page, MAX_PAGES + 1): + last_page = page + rows = fetch_page(sess, page) + + if rows is None: + log.info('Pagina %d: 404 — fin de paginacion', page) + save_state(page + 1, total_new) + break + + if not rows: + streak += 1 + save_state(page + 1, total_new) + log.info('Pagina %d: sin resultados (racha %d/%d)', page, streak, MAX_STREAK) + if streak >= MAX_STREAK: + log.info('Deteniendo: %d paginas consecutivas sin resultados', streak) + break + time.sleep(DELAY) + continue + + n = upsert(conn, rows) + streak = 0 if n > 0 else streak + 1 + total_new += n + save_state(page + 1, total_new) + log.info('Pagina %d: %d nuevos (acumulado %d)', page, n, total_new) + + if streak >= MAX_STREAK: + log.info('Deteniendo: %d paginas sin nuevos', streak) + break + + time.sleep(DELAY) + + conn.close() + log.info('=== Backfill finalizado: %d nuevos (ultima pagina: %d) ===', total_new, last_page) + print(f'Finalizado: {total_new} nuevos') + + +if __name__ == '__main__': + main()