217 lines
6.9 KiB
Python
217 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Scraper Corotos.com.do Republica Dominicana -> postgres17-central / corotos_do
|
|
URL: /anuncios?categoria=bienes-raices&page={N}
|
|
ID: data-adid="NUMBER" en cada card
|
|
"""
|
|
import os, re, time, logging
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import psycopg2
|
|
from psycopg2.extras import execute_values
|
|
from datetime import datetime
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
|
|
log = logging.getLogger('corotos')
|
|
|
|
PROXY = 'socks5h://127.0.0.1:1090'
|
|
PROXIES = {'http': PROXY, 'https': PROXY}
|
|
DB_HOST = '100.75.240.87'
|
|
DB_NAME = 'corotos_do'
|
|
DB_USER = 'pgadmin'
|
|
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
|
|
BASE = 'https://www.corotos.com.do'
|
|
LIST_URL = BASE + '/anuncios?categoria=bienes-raices&page={}'
|
|
DELAY = 1.0
|
|
MAX_PAGES = int(os.environ.get('CO_MAX_PAGES', '20'))
|
|
NEW_STREAK = 2
|
|
|
|
|
|
def get_session():
|
|
s = requests.Session()
|
|
s.proxies.update(PROXIES)
|
|
s.headers.update({
|
|
'User-Agent': UA,
|
|
'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8',
|
|
'Accept-Language': 'es-DO,es;q=0.9',
|
|
})
|
|
return s
|
|
|
|
|
|
def get_conn():
|
|
return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME,
|
|
user=DB_USER, password=DB_PASS, connect_timeout=10)
|
|
|
|
|
|
def create_table(conn):
|
|
cur = conn.cursor()
|
|
cur.execute("""
|
|
CREATE TABLE IF NOT EXISTS listings (
|
|
listing_id TEXT PRIMARY KEY,
|
|
titulo TEXT,
|
|
precio BIGINT,
|
|
moneda TEXT DEFAULT 'DOP',
|
|
tipo TEXT,
|
|
ubicacion TEXT,
|
|
url TEXT UNIQUE,
|
|
superficie_m2 DECIMAL,
|
|
scraped_at TIMESTAMP DEFAULT NOW()
|
|
)
|
|
""")
|
|
conn.commit()
|
|
cur.close()
|
|
|
|
|
|
def upsert(conn, rows):
|
|
if not rows:
|
|
return 0
|
|
cur = conn.cursor()
|
|
execute_values(cur,
|
|
"""INSERT INTO listings (listing_id, titulo, precio, moneda, tipo, ubicacion, url, scraped_at)
|
|
VALUES %s
|
|
ON CONFLICT (listing_id) DO UPDATE SET
|
|
titulo=EXCLUDED.titulo, precio=EXCLUDED.precio, scraped_at=EXCLUDED.scraped_at""",
|
|
[(r['listing_id'], r['titulo'], r['precio'], r.get('moneda', 'DOP'),
|
|
r['tipo'], r['ubicacion'], r['url'], datetime.utcnow()) for r in rows]
|
|
)
|
|
n = cur.rowcount
|
|
conn.commit()
|
|
cur.close()
|
|
return n
|
|
|
|
|
|
def fetch_page(sess, page):
|
|
url = LIST_URL.format(page)
|
|
for attempt in range(3):
|
|
try:
|
|
resp = sess.get(url, timeout=30)
|
|
if resp.status_code == 404:
|
|
return None
|
|
if resp.status_code in (403, 429):
|
|
log.warning('HTTP %d en %s', resp.status_code, url)
|
|
time.sleep(30)
|
|
continue
|
|
resp.raise_for_status()
|
|
break
|
|
except Exception as e:
|
|
if attempt == 2:
|
|
log.error('Error %s: %s', url, e)
|
|
return []
|
|
time.sleep([2, 5, 15][attempt])
|
|
|
|
soup = BeautifulSoup(resp.text, 'lxml')
|
|
results = []
|
|
seen = set()
|
|
|
|
# Primary: cards con data-adid
|
|
cards_with_id = soup.find_all(attrs={'data-adid': True})
|
|
for card in cards_with_id:
|
|
try:
|
|
listing_id = card.get('data-adid', '').strip()
|
|
if not listing_id or listing_id in seen:
|
|
continue
|
|
seen.add(listing_id)
|
|
|
|
link_tag = card.find('a', href=re.compile(r'/anuncio/'))
|
|
href = link_tag['href'] if link_tag else f'/anuncio/{listing_id}'
|
|
url_full = BASE + href if href.startswith('/') else href
|
|
|
|
tipo = 'alquiler' if 'alquiler' in href.lower() or 'alquiler' in card.get_text().lower() else 'venta'
|
|
|
|
titulo_tag = card.find(['h2', 'h3', 'a', 'p'])
|
|
titulo = titulo_tag.get_text(strip=True)[:300] if titulo_tag else 'Sin titulo'
|
|
|
|
precio = None
|
|
moneda = 'DOP'
|
|
precio_text = card.get_text()
|
|
if 'USD' in precio_text:
|
|
moneda = 'USD'
|
|
nums = re.findall(r'[\d,]+', precio_text)
|
|
for n_str in nums:
|
|
try:
|
|
val = int(n_str.replace(',', ''))
|
|
if val > 999:
|
|
precio = val
|
|
break
|
|
except ValueError:
|
|
pass
|
|
|
|
loc_tag = card.find(string=re.compile(
|
|
r'Santo Domingo|Santiago|La Romana|Punta Cana|Puerto Plata|Hig.ey|San Pedro|Bavaro', re.I))
|
|
ubicacion = str(loc_tag).strip()[:200] if loc_tag else 'Rep. Dominicana'
|
|
|
|
results.append({
|
|
'listing_id': listing_id,
|
|
'titulo': titulo,
|
|
'precio': precio,
|
|
'moneda': moneda,
|
|
'tipo': tipo,
|
|
'ubicacion': ubicacion,
|
|
'url': url_full,
|
|
})
|
|
except Exception as e:
|
|
log.debug('Error card: %s', e)
|
|
|
|
# Fallback si no se encontraron data-adid
|
|
if not results:
|
|
for link in soup.find_all('a', href=re.compile(r'/anuncio/inmuebles')):
|
|
href = link.get('href', '')
|
|
slug = href.rstrip('/').split('/')[-1]
|
|
if slug and slug not in seen:
|
|
seen.add(slug)
|
|
titulo_tag = link.find(['h2', 'h3', 'span'])
|
|
titulo = titulo_tag.get_text(strip=True) if titulo_tag else slug.replace('-', ' ')[:200]
|
|
results.append({
|
|
'listing_id': slug[:50],
|
|
'titulo': titulo[:300],
|
|
'precio': None,
|
|
'moneda': 'DOP',
|
|
'tipo': 'alquiler' if 'alquiler' in href.lower() else 'venta',
|
|
'ubicacion': 'Rep. Dominicana',
|
|
'url': BASE + href if href.startswith('/') else href,
|
|
})
|
|
|
|
log.info('Pagina %d: %d resultados', page, len(results))
|
|
return results
|
|
|
|
|
|
def main():
|
|
log.info('=== Iniciando scraper Corotos Rep. Dominicana ===')
|
|
sess = get_session()
|
|
conn = get_conn()
|
|
create_table(conn)
|
|
|
|
total_new = 0
|
|
streak = 0
|
|
|
|
for page in range(1, MAX_PAGES + 1):
|
|
rows = fetch_page(sess, page)
|
|
if rows is None:
|
|
log.info('Pagina %d: fin', page)
|
|
break
|
|
if not rows:
|
|
streak += 1
|
|
log.info('Pagina %d: sin resultados (racha %d/%d)', page, streak, NEW_STREAK)
|
|
if streak >= NEW_STREAK:
|
|
break
|
|
time.sleep(DELAY)
|
|
continue
|
|
|
|
n = upsert(conn, rows)
|
|
streak = 0 if n > 0 else streak + 1
|
|
total_new += n
|
|
|
|
if streak >= NEW_STREAK:
|
|
log.info('Deteniendo: %d paginas sin nuevos', streak)
|
|
break
|
|
time.sleep(DELAY)
|
|
|
|
conn.close()
|
|
print(f'Finalizado: {total_new} nuevos, 0 actualizados')
|
|
log.info('Finalizado: %d nuevos, 0 actualizados', total_new)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|